Use vendored s3 wkfs

cmd/cashierd/main.go

@@ -35,8 +35,8 @@ import (
 	"github.com/nsheridan/cashier/server/store"
 	"github.com/nsheridan/cashier/server/templates"
 	"github.com/nsheridan/cashier/server/util"
-	"github.com/nsheridan/cashier/server/wkfs/s3fs"
 	"github.com/nsheridan/cashier/server/wkfs/vaultfs"
+	"github.com/nsheridan/wkfs/s3"
 	"github.com/sid77/drop"
 )
 
@@ -313,46 +313,53 @@ func loadCerts(certFile, keyFile string) (tls.Certificate, error) {
 func main() {
 	// Privileged section
 	flag.Parse()
-	config, err := readConfig(*cfg)
+	conf, err := readConfig(*cfg)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	// Register well-known filesystems.
-	s3fs.Register(config.AWS)
-	vaultfs.Register(config.Vault)
+	if conf.AWS == nil {
+		conf.AWS = &config.AWS{}
+	}
+	s3.Register(&s3.Options{
+		Region:    conf.AWS.Region,
+		AccessKey: conf.AWS.AccessKey,
+		SecretKey: conf.AWS.SecretKey,
+	})
+	vaultfs.Register(conf.Vault)
 
-	signer, err := signer.New(config.SSH)
+	signer, err := signer.New(conf.SSH)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	logfile := os.Stderr
-	if config.Server.HTTPLogFile != "" {
-		logfile, err = os.OpenFile(config.Server.HTTPLogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640)
+	if conf.Server.HTTPLogFile != "" {
+		logfile, err = os.OpenFile(conf.Server.HTTPLogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640)
 		if err != nil {
 			log.Fatal(err)
 		}
 	}
 
-	laddr := fmt.Sprintf("%s:%d", config.Server.Addr, config.Server.Port)
+	laddr := fmt.Sprintf("%s:%d", conf.Server.Addr, conf.Server.Port)
 	l, err := net.Listen("tcp", laddr)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	tlsConfig := &tls.Config{}
-	if config.Server.UseTLS {
-		if config.Server.LetsEncryptServername != "" {
+	if conf.Server.UseTLS {
+		if conf.Server.LetsEncryptServername != "" {
 			m := autocert.Manager{
 				Prompt:     autocert.AcceptTOS,
-				Cache:      autocert.DirCache(config.Server.LetsEncryptCache),
-				HostPolicy: autocert.HostWhitelist(config.Server.LetsEncryptServername),
+				Cache:      autocert.DirCache(conf.Server.LetsEncryptCache),
+				HostPolicy: autocert.HostWhitelist(conf.Server.LetsEncryptServername),
 			}
 			tlsConfig.GetCertificate = m.GetCertificate
 		} else {
 			tlsConfig.Certificates = make([]tls.Certificate, 1)
-			tlsConfig.Certificates[0], err = loadCerts(config.Server.TLSCert, config.Server.TLSKey)
+			tlsConfig.Certificates[0], err = loadCerts(conf.Server.TLSCert, conf.Server.TLSKey)
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -360,33 +367,33 @@ func main() {
 		l = tls.NewListener(l, tlsConfig)
 	}
 
-	if config.Server.User != "" {
+	if conf.Server.User != "" {
 		log.Print("Dropping privileges...")
-		if err := drop.DropPrivileges(config.Server.User); err != nil {
+		if err := drop.DropPrivileges(conf.Server.User); err != nil {
 			log.Fatal(err)
 		}
 	}
 
 	// Unprivileged section
 	var authprovider auth.Provider
-	switch config.Auth.Provider {
+	switch conf.Auth.Provider {
 	case "google":
-		authprovider, err = google.New(config.Auth)
+		authprovider, err = google.New(conf.Auth)
 	case "github":
-		authprovider, err = github.New(config.Auth)
+		authprovider, err = github.New(conf.Auth)
 	default:
-		log.Fatalf("Unknown provider %s\n", config.Auth.Provider)
+		log.Fatalf("Unknown provider %s\n", conf.Auth.Provider)
 	}
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	certstore, err := store.New(config.Server.Database)
+	certstore, err := store.New(conf.Server.Database)
 	if err != nil {
 		log.Fatal(err)
 	}
 	ctx := &appContext{
-		cookiestore:  sessions.NewCookieStore([]byte(config.Server.CookieSecret)),
+		cookiestore:  sessions.NewCookieStore([]byte(conf.Server.CookieSecret)),
 		authprovider: authprovider,
 		sshKeySigner: signer,
 		certstore:    certstore,
@@ -394,11 +401,11 @@ func main() {
 	ctx.cookiestore.Options = &sessions.Options{
 		MaxAge:   900,
 		Path:     "/",
-		Secure:   config.Server.UseTLS,
+		Secure:   conf.Server.UseTLS,
 		HttpOnly: true,
 	}
 
-	CSRF := csrf.Protect([]byte(config.Server.CSRFSecret), csrf.Secure(config.Server.UseTLS))
+	CSRF := csrf.Protect([]byte(conf.Server.CSRFSecret), csrf.Secure(conf.Server.UseTLS))
 	r := mux.NewRouter()
 	r.Methods("GET").Path("/").Handler(appHandler{ctx, rootHandler})
 	r.Methods("GET").Path("/auth/login").Handler(appHandler{ctx, loginHandler})

vendor/github.com/nsheridan/wkfs/s3/README.md

+## S3 plugin for WKFS
+
+
+
+Package `s3` registers an AWS S3 filesystem at the well-known `/s3/` filesystem path.
+
+Sample usage:
+
+```go
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+
+	"github.com/nsheridan/wkfs/s3"
+	"go4.org/wkfs"
+)
+
+func main() {
+  opts := &s3.Options{
+    Region: "us-east-1"
+    AccessKey: "abcdef"
+    SecretKey: "secret"
+  }
+  s3.Register(opts)
+  f, err := wkfs.Create("/s3/some-bucket/hello.txt")
+  if err != nil {
+    log.Fatal(err)
+  }
+  _, err := io.WriteString(f, "hello, world")
+  if err != nil {
+    log.Fatal(err)
+  }
+}
+```
+
+
+
+`Options` are completely optional as the AWS SDK will attempt to obtain credentials from a number of locations - see [the documentation for details](http://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) - e.g. if you're using environment variables you can register the filesystem with `s3.Register(nil)`.

server/wkfs/s3fs/s3.go → vendor/github.com/nsheridan/wkfs/s3/s3.go

-package s3fs
+package s3
 
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -16,35 +18,38 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/nsheridan/cashier/server/config"
 )
 
+// Options for registering the S3 wkfs.
+// None of these are required and can be supplied to the aws client by other means.
+type Options struct {
+	Region    string
+	AccessKey string
+	SecretKey string
+}
+
 // Register the /s3/ filesystem as a well-known filesystem.
-func Register(config *config.AWS) {
-	if config == nil {
-		registerBrokenFS(errors.New("aws credentials not found"))
-		return
+func Register(opts *Options) {
+	if opts == nil {
+		opts = &Options{}
 	}
-	ac := &aws.Config{}
+	config := &aws.Config{}
 	// If region is unset the SDK will attempt to read the region from the environment.
-	if config.Region != "" {
-		ac.Region = aws.String(config.Region)
-	}
-	// Attempt to get credentials from the cashier config.
-	// Otherwise check for standard credentials. If neither are present register the fs as broken.
-	// TODO: implement this as a provider.
-	if config.AccessKey != "" && config.SecretKey != "" {
-		ac.Credentials = credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, "")
-	} else {
-		_, err := session.New().Config.Credentials.Get()
+	if opts.Region != "" {
+		config.Region = aws.String(opts.Region)
+	}
+	// Attempt to use supplied credentials, otherwise fall back to the SDK.
+	if opts.AccessKey != "" && opts.SecretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(opts.AccessKey, opts.SecretKey, "")
+	}
+	s, err := session.NewSession(config)
 	if err != nil {
-			registerBrokenFS(errors.New("aws credentials not found"))
+		registerBrokenFS(err)
 		return
 	}
-	}
-	sc := s3.New(session.New(ac))
+	sc := s3.New(s)
 	if aws.StringValue(sc.Config.Region) == "" {
-		registerBrokenFS(errors.New("aws region configuration not found"))
+		registerBrokenFS(errors.New("could not find region configuration"))
 		return
 	}
 	wkfs.RegisterFS("/s3/", &s3FS{
@@ -125,10 +130,28 @@ func (fs *s3FS) Lstat(name string) (os.FileInfo, error) {
 	}, nil
 }
 
-func (fs *s3FS) MkdirAll(path string, perm os.FileMode) error { return nil }
+func (fs *s3FS) MkdirAll(path string, perm os.FileMode) error {
+	_, err := fs.OpenFile(fmt.Sprintf("%s/", filepath.Clean(path)), os.O_CREATE, perm)
+	return err
+}
 
 func (fs *s3FS) OpenFile(name string, flag int, perm os.FileMode) (wkfs.FileWriter, error) {
-	return nil, errors.New("not implemented")
+	bucket, filename, err := fs.parseName(name)
+	if err != nil {
+		return nil, err
+	}
+	switch flag {
+	case os.O_WRONLY | os.O_CREATE | os.O_EXCL:
+	case os.O_WRONLY | os.O_CREATE | os.O_TRUNC:
+	default:
+		return nil, fmt.Errorf("Unsupported OpenFlag flag mode %d on S3", flag)
+	}
+	if flag&os.O_EXCL != 0 {
+		if _, err := fs.Stat(name); err == nil {
+			return nil, os.ErrExist
+		}
+	}
+	return NewS3file(bucket, filename, fs.sc)
 }
 
 type statInfo struct {

vendor/github.com/nsheridan/wkfs/s3/s3_file.go

+package s3
+
+import (
+	"bytes"
+	"errors"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// S3file represents a file in S3.
+type S3file struct {
+	bucket string
+	name   string
+	offset int
+	closed bool
+
+	s3api *s3.S3
+}
+
+// NewS3file initializes an S3file.
+func NewS3file(bucket, name string, s3api *s3.S3) (*S3file, error) {
+	return &S3file{
+		bucket: bucket,
+		name:   name,
+		offset: 0,
+		closed: false,
+		s3api:  s3api,
+	}, nil
+}
+
+// Write len(p) bytes to the file in S3.
+// It returns the number of bytes written and an error, if any.
+func (f *S3file) Write(p []byte) (n int, err error) {
+	if f.closed {
+		panic("read after close")
+	}
+	if f.offset != 0 {
+		return 0, errors.New("Offset cannot be > 0")
+	}
+	readSeeker := bytes.NewReader(p)
+	size := int(readSeeker.Size())
+	obj := &s3.PutObjectInput{
+		Bucket: aws.String(f.bucket),
+		Key:    aws.String(f.name),
+		Body:   readSeeker,
+	}
+	if _, err := f.s3api.PutObject(obj); err != nil {
+		return 0, err
+	}
+	f.offset += size
+	return size, nil
+}
+
+// Close the file, rendering it unusable.
+func (f *S3file) Close() error {
+	f.closed = true
+	return nil
+}

vendor/vendor.json

@@ -392,6 +392,12 @@
 			"revision": "bfdb1a85537d60bc7e954e600c250219ea497417",
 			"revisionTime": "2016-12-11T22:23:15Z"
 		},
+		{
+			"checksumSHA1": "Ywe06VqOCpwDNjipGTMO0oOG/Yg=",
+			"path": "github.com/nsheridan/wkfs/s3",
+			"revision": "60e6f1760f59568e4ce95080d08cd4a90c3c50c7",
+			"revisionTime": "2016-12-29T20:48:42Z"
+		},
 		{
 			"checksumSHA1": "8Y05Pz7onrQPcVWW6JStSsYRh6E=",
 			"path": "github.com/pelletier/go-buffruneio",