implement ghetto file upload for small files

Makefile

@@ -1,6 +1,6 @@
 SHELL = bash
 PROJECT_ROOT := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
-VERSION := 0.4.0
+VERSION := 0.5.0
 GIT_COMMIT := $(shell git rev-parse --short HEAD)
 
 GO_PKGS := $(shell go list ./...)

README.md

@@ -10,10 +10,14 @@ This project is in development phase. You can try it with latest release version
 
 ## Installation
 
+Download and install using go get:
+
 ```sh
 go get -u github.com/romantomjak/b2
 ```
 
+or grab a binary from [releases](https://github.com/romantomjak/b2/releases/latest) section!
+
 ## Usage
 
 ```sh
@@ -33,6 +37,7 @@ Available commands are:
  create Create a new bucket
  get Download files
  list List files and buckets
+ put Upload files
  version Prints the client version
 ```
 
@@ -47,7 +52,7 @@ This is how far I've gotten:
 - [x] List all buckets
 - [ ] Update settings for a bucket
 - [x] List files in a bucket
-- [ ] Upload small files
+- [x] Upload small files (<100 MB)
 - [ ] Upload large files
 - [x] Download a file
 

b2/client.go

@@ -197,15 +197,21 @@ func (c *Client) newRequest(ctx context.Context, method, path string, body inter
 
  u := c.baseURL.ResolveReference(rel)
 
- buf := new(bytes.Buffer)
+ var b io.Reader
  if body != nil {
- err := json.NewEncoder(buf).Encode(body)
- if err != nil {
- return nil, err
+ if r, ok := body.(io.Reader); ok {
+ b = r
+ } else {
+ buf := new(bytes.Buffer)
+ err := json.NewEncoder(buf).Encode(body)
+ if err != nil {
+ return nil, err
+ }
+ b = buf
  }
  }
 
- req, err := http.NewRequestWithContext(ctx, method, u.String(), buf)
+ req, err := http.NewRequestWithContext(ctx, method, u.String(), b)
  if err != nil {
  return nil, err
  }

b2/file.go

@@ -2,12 +2,17 @@ package b2
 
 import (
  "context"
+ "crypto/sha1"
+ "fmt"
  "io"
  "net/http"
+ "net/url"
+ "os"
 )
 
 const (
- listFilesURL = "b2api/v2/b2_list_file_names"
+ listFilesURL = "b2api/v2/b2_list_file_names"
+ fileUploadURL = "b2api/v2/b2_get_upload_url"
 )
 
 // File describes a File or a Folder in a Bucket
@@ -38,6 +43,18 @@ type fileListRoot struct {
  NextFileName string `json:"nextFileName"`
 }
 
+// UploadAuthorizationRequest represents a request to obtain a URL for uploading files
+type UploadAuthorizationRequest struct {
+ BucketID string `json:"bucketId"`
+}
+
+// UploadAuthorization contains the information for uploading a file
+type UploadAuthorization struct {
+ BucketID string `json:"bucketId"`
+ UploadURL string `json:"uploadUrl"`
+ Token string `json:"authorizationToken"`
+}
+
 // FileService handles communication with the File related methods of the
 // B2 API
 type FileService struct {
@@ -74,3 +91,63 @@ func (s *FileService) Download(ctx context.Context, url string, w io.Writer) (*h
 
  return resp, err
 }
+
+// UploadAuthorization returns the information for uploading a file
+func (s *FileService) UploadAuthorization(ctx context.Context, uploadAuthorizationRequest *UploadAuthorizationRequest) (*UploadAuthorization, *http.Response, error) {
+ req, err := s.client.NewRequest(ctx, http.MethodPost, fileUploadURL, uploadAuthorizationRequest)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ auth := new(UploadAuthorization)
+ resp, err := s.client.Do(req, auth)
+ if err != nil {
+ return nil, resp, err
+ }
+
+ return auth, resp, nil
+}
+
+// Upload a file
+func (s *FileService) Upload(ctx context.Context, uploadAuthorization *UploadAuthorization, src, dst string) (*File, *http.Response, error) {
+ f, err := os.Open(src)
+ if err != nil {
+ return nil, nil, err
+ }
+ defer f.Close()
+
+ info, err := f.Stat()
+ if err != nil {
+ return nil, nil, err
+ }
+
+ hash := sha1.New()
+ _, err = io.Copy(hash, f)
+ if err != nil {
+ return nil, nil, err
+ }
+ sha1 := fmt.Sprintf("%x", hash.Sum(nil))
+
+ f.Seek(0, 0)
+
+ req, err := s.client.NewRequest(ctx, http.MethodPost, uploadAuthorization.UploadURL, f)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ req.ContentLength = info.Size()
+
+ req.Header.Set("Authorization", uploadAuthorization.Token)
+ req.Header.Set("X-Bz-File-Name", url.QueryEscape(dst))
+ req.Header.Set("Content-Type", "b2/x-auto")
+ req.Header.Set("X-Bz-Content-Sha1", sha1)
+ req.Header.Set("X-Bz-Info-src_last_modified_millis", fmt.Sprintf("%d", info.ModTime().Unix()*1000))
+
+ file := new(File)
+ resp, err := s.client.Do(req, file)
+ if err != nil {
+ return nil, resp, err
+ }
+
+ return file, resp, nil
+}

command/bucket_create_test.go

@@ -77,6 +77,7 @@ func TestCreateBucketCommand_BucketCreateRequest(t *testing.T) {
  }
 
  _ = cmd.Run([]string{"my-bucket"})
+ // TODO: write bucket response
  // return code is ignored on purpose here.
  // fake b2_create_bucket handler is not writing the response, so
  // the command will fail and return 1

command/commands.go

@@ -27,6 +27,11 @@ func Commands(ui cli.Ui) map[string]cli.CommandFactory {
  baseCommand: baseCommand,
  }, nil
  },
+ "put": func() (cli.Command, error) {
+ return &PutCommand{
+ baseCommand: baseCommand,
+ }, nil
+ },
  "version": func() (cli.Command, error) {
  return &VersionCommand{
  baseCommand: baseCommand,

command/put.go

@@ -0,0 +1,183 @@
+package command
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "path"
+ "strings"
+
+ "github.com/romantomjak/b2/b2"
+)
+
+type PutCommand struct {
+ *baseCommand
+}
+
+func (c *PutCommand) Help() string {
+ helpText := `
+Usage: b2 put <source> <destination>
+
+ Uploads the contents of source to destination. If destination
+ contains a trailing slash it is treated as a directory and
+ file is uploaded keeping the original filename.
+
+General Options:
+
+ ` + c.generalOptions()
+ return strings.TrimSpace(helpText)
+}
+
+func (c *PutCommand) Synopsis() string {
+ return "Upload files"
+}
+
+func (c *PutCommand) Name() string { return "put" }
+
+func (c *PutCommand) Run(args []string) int {
+ flags := c.flagSet()
+ flags.Usage = func() { c.ui.Output(c.Help()) }
+
+ if err := flags.Parse(args); err != nil {
+ return 1
+ }
+
+ // Check that we got both arguments
+ args = flags.Args()
+ numArgs := len(args)
+ if numArgs != 2 {
+ c.ui.Error("This command takes two arguments: <source> and <destination>")
+ return 1
+ }
+
+ // Check that source file exists
+ if !fileExists(args[0]) {
+ c.ui.Error(fmt.Sprintf("File does not exist: %s", args[0]))
+ return 1
+ }
+
+ // FIXME: remove when large file upload is implemented
+ err := checkMaxFileSize(args[0])
+ if err != nil {
+ c.ui.Error("Large file upload is not yet implemented. Maximum file size is 100 MB")
+ return 1
+ }
+
+ bucketName, filePrefix := destinationBucketAndFilename(args[0], args[1])
+
+ // TODO: caching bucket name:id mappings could save this request
+ bucket, err := c.findBucketByName(bucketName)
+ if err != nil {
+ c.ui.Error(fmt.Sprintf("Error: %v", err))
+ return 1
+ }
+
+ // Create a client
+ client, err := c.Client()
+ if err != nil {
+ c.ui.Error(fmt.Sprintf("Error: %v", err))
+ return 1
+ }
+
+ // Request upload url
+ ctx := context.TODO()
+
+ uploadAuthReq := &b2.UploadAuthorizationRequest{
+ BucketID: bucket.ID,
+ }
+ uploadAuth, _, err := client.File.UploadAuthorization(ctx, uploadAuthReq)
+ if err != nil {
+ c.ui.Error(fmt.Sprintf("Error: %v", err))
+ return 1
+ }
+
+ _, _, err = client.File.Upload(ctx, uploadAuth, args[0], filePrefix)
+ if err != nil {
+ c.ui.Error(err.Error())
+ return 1
+ }
+
+ c.ui.Output(fmt.Sprintf("Uploaded %q to %q", args[0], path.Join(bucket.Name, filePrefix)))
+
+ return 0
+}
+
+func (c *PutCommand) findBucketByName(name string) (*b2.Bucket, error) {
+ client, err := c.Client()
+ if err != nil {
+ c.ui.Error(fmt.Sprintf("Error: %v", err))
+ return nil, err
+ }
+
+ req := &b2.BucketListRequest{
+ AccountID: client.AccountID,
+ Name: name,
+ }
+
+ ctx := context.TODO()
+
+ buckets, _, err := client.Bucket.List(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(buckets) == 0 {
+ return nil, fmt.Errorf("bucket with name %q was not found", name)
+ }
+
+ return &buckets[0], nil
+}
+
+// fileExists checks if a file exists and is not a directory
+func fileExists(filename string) bool {
+ info, err := os.Stat(filename)
+ if os.IsNotExist(err) {
+ return false
+ }
+ return !info.IsDir()
+}
+
+// checkMaxFileSize checks that file is a "small" file
+func checkMaxFileSize(filename string) error {
+ var maxFileSize int64 = 100 << (10 * 2) // 100 mb
+
+ info, err := os.Stat(filename)
+ if err != nil {
+ return err
+ }
+
+ if info.Size() > maxFileSize {
+ return errors.New("file is too big")
+ }
+
+ return nil
+}
+
+// destinationBucketAndFilename returns upload bucket and filePrefix
+//
+// b2 does not have a concept of folders, so if destination contains
+// a trailing slash it is treated as a directory and file is uploaded
+// keeping the original filename. If destination is simply a bucket
+// name, it is asumed the destination is "/" and filename is preserved
+func destinationBucketAndFilename(source, destination string) (string, string) {
+ originalFilename := path.Base(source)
+
+ destinationParts := strings.SplitN(destination, "/", 2)
+ bucketName := destinationParts[0]
+ filePrefix := ""
+
+ if len(destinationParts) > 1 {
+ if strings.HasSuffix(destinationParts[1], "/") {
+ filePrefix = path.Join(destinationParts[1], originalFilename)
+ } else {
+ filePrefix = destinationParts[1]
+ }
+ }
+
+ if filePrefix == "" {
+ filePrefix = originalFilename
+ }
+
+ return bucketName, filePrefix
+}