From edcfb95a4d0ccf4f387484d3d86da9d1825ed4f9 Mon Sep 17 00:00:00 2001 From: Miyako Satori <31246794+Mikubill@users.noreply.github.com> Date: Mon, 17 May 2021 13:15:07 +0900 Subject: [PATCH] fix: notion - invalid meta field --- apis/public/notion/client.go | 495 +++++++++++++++++++++++++++++++++++ apis/public/notion/struct.go | 196 ++++++++++++-- apis/public/notion/upload.go | 250 +----------------- go.mod | 11 +- go.sum | 15 ++ 5 files changed, 704 insertions(+), 263 deletions(-) create mode 100644 apis/public/notion/client.go diff --git a/apis/public/notion/client.go b/apis/public/notion/client.go new file mode 100644 index 0000000..a87d97a --- /dev/null +++ b/apis/public/notion/client.go @@ -0,0 +1,495 @@ +package notion + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "mime" + "net/http" + "path" + "strconv" + "strings" + "time" + "transfer/apis" + + "github.com/google/uuid" + "golang.org/x/time/rate" +) + +const ( + s3URLPrefix = "https://s3-us-west-2.amazonaws.com/secure.notion-static.com/" + userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3483.0 Safari/537.36" + acceptLang = "en-US,en;q=0.9" + + CommandSet = "set" + CommandUpdate = "update" + CommandListAfter = "listAfter" + CommandListRemove = "listRemove" + TableSpace = "space" + TableActivity = "activity" + TableBlock = "block" + TableUser = "notion_user" + TableCollection = "collection" + TableCollectionView = "collection_view" + TableComment = "comment" + TableDiscussion = "discussion" + + dashIDLen = len("2131b10c-ebf6-4938-a127-7089ff02dbe4") + noDashIDLen = len("2131b10cebf64938a1277089ff02dbe4") + webAPI = "https://www.notion.so/api/v3" + layout = "2006-01-02T15:04:05.999Z" +) + +func NewWebClient(token string) *webClient { + return &webClient{ + token: token, + limiter: rate.NewLimiter(rate.Every(time.Second*5), 1), + } +} + +// ToDashID convert id in format bb760e2dd6794b64b2a903005b21870a +// to bb760e2d-d679-4b64-b2a9-03005b21870a +// If id is not in that format, we leave it untouched. +func ToDashID(id string) string { + s := strings.Replace(id, "-", "", -1) + if len(s) != noDashIDLen { + return id + } + res := id[:8] + "-" + id[8:12] + "-" + id[12:16] + "-" + id[16:20] + "-" + id[20:] + return res +} + +func (c *webClient) GetPage(pageid string) (res *PageDataResponse, err error) { + pageID := ToDashID(pageid) + + res, err = c.GetPageData(pageID) + if err != nil { + return + } + if res.Spaceid == "" || res.Owneruserid == "" { + return nil, fmt.Errorf("metadata not found, page \"%s\"", pageID) + } + res.Pageid = pageID + + var cur *cursor + var rsp *LoadPageChunkResponse + var last *Block + chunkID := 0 + for { + rsp, err = c.LoadPageChunk(pageID, chunkID, cur) + if err != nil { + return nil, err + } + recordLoc := rsp.RecordMap + if recordLoc == nil { + chunkID++ + cur = &rsp.Cursor + continue + } + + for _, v := range recordLoc.Blocks { + b := v.Block + if b.Alive { + last = b + } + } + break + } + res.Cursor = last + return +} + +func (c *webClient) GetFullPage(pageid string) (last []*Block) { + pageID := ToDashID(pageid) + + var cur *cursor + chunkNo := 0 + for { + rsp, err := c.LoadPageChunk(pageID, chunkNo, cur) + if err != nil { + // log.Warn(err.Error()) + break + } + chunkNo++ + recordLoc := rsp.RecordMap + for _, v := range recordLoc.Blocks { + b := v.Block + if b.Alive && b.Type == "file" { + last = append(last, b) + } + } + cur = &rsp.Cursor + if len(cur.Stack) == 0 { + break + } + } + return +} + +func (c *webClient) insertFile(filename, fileid, fileurl string, + root *PageDataResponse, filesize int64, mod time.Time) (string, error) { + // PrintStruct(root) + + userID := root.Owneruserid + spaceID := root.Spaceid + timeStamp := time.Now().Unix() * 1000 + + newBlockID := uuid.New().String() + + ops := []*Operation{ + buildOp(newBlockID, CommandSet, []string{}, map[string]interface{}{ + "type": "file", + "id": newBlockID, + "version": 1, + }), + buildOp(newBlockID, CommandUpdate, []string{}, map[string]interface{}{ + "parent_id": root.Pageid, + "parent_table": "block", + "alive": true, + }), + buildOp(root.Pageid, CommandListAfter, []string{"content"}, map[string]string{ + "id": newBlockID, + "after": root.Cursor.ID, + }), + buildOp(newBlockID, CommandSet, []string{"created_by_id"}, userID), + buildOp(newBlockID, CommandSet, []string{"created_by_table"}, "notion_user"), + buildOp(newBlockID, CommandSet, []string{"created_time"}, timeStamp), + buildOp(newBlockID, CommandSet, []string{"last_edited_time"}, timeStamp), + buildOp(newBlockID, CommandSet, []string{"last_edited_by_id"}, userID), + buildOp(newBlockID, CommandSet, []string{"last_edited_by_table"}, "notion_user"), + buildOp(newBlockID, CommandUpdate, []string{"properties"}, map[string]interface{}{ + "source": [][]string{{fileurl}}, + "size": [][]string{{ByteCountIEC(filesize)}}, + "title": [][]string{{filename}}, + "actual_size": [][]string{{strconv.FormatInt(filesize, 10)}}, + "actual_modified": [][]string{{strconv.FormatInt(mod.UnixNano(), 10)}}, + }), + buildOp(newBlockID, CommandListAfter, []string{"file_ids"}, map[string]string{ + "id": fileid, + }), + } + err := c.pushTransaction(ops, newBlockID, spaceID, filename) + if err != nil { + return "", err + } + return newBlockID, nil + // lastBlockID = newBlockID +} + +func ByteCountIEC(b int64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", + float64(b)/float64(div), "KMGTPE"[exp]) +} + +// buildOp creates an Operation for this block +func buildOp(blockID, command string, path []string, args interface{}) *Operation { + return &Operation{ + Point: Pointer{ + ID: blockID, + Table: "block", + }, + Path: path, + Command: command, + Args: args, + } +} + +func (c *webClient) pushTransaction(ops []*Operation, blockID, spaceID, filename string) error { + c.limiter.Allow() + err := c.SubmitTransaction(ops, spaceID) + if err != nil { + // log.Println(err) + return err + } + // log.Println("Uploaded.", filename) + return nil +} + +func (c *webClient) SubmitTransaction(ops []*Operation, spaceID string) error { + + reqData := &submitTransactionRequest{ + RequestID: uuid.New().String(), + Transaction: []Transaction{{ + ID: uuid.New().String(), + SpaceID: spaceID, + Operations: ops, + }}, + } + // PrintStruct(reqData) + + js, err := json.Marshal(reqData) + if err != nil { + return err + } + req, err := http.NewRequest("POST", "https://www.notion.so/api/v3/saveTransactions", bytes.NewBuffer(js)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept-Language", acceptLang) + req.Header.Set("cookie", fmt.Sprintf("token_v2=%v", c.token)) + var rsp *http.Response + + // http.DefaultClient.Timeout = time.Second * 30 + rsp, err = http.DefaultClient.Do(req) + + if err != nil { + return err + } + + d, err := ioutil.ReadAll(rsp.Body) + if err != nil { + return err + } + + _ = rsp.Body.Close() + + if rsp.StatusCode != 200 { + return fmt.Errorf("http.Post returned non-200 status code of %d, returns: %s", rsp.StatusCode, d) + } + + if !bytes.Equal(d, []byte("{}")) { + return fmt.Errorf("unknown error: %s", d) + } + return nil +} + +// UploadFile Uploads a file to notion's asset hosting(aws s3) +func (c *webClient) UploadFile(file io.Reader, name string, size int64) (fileID, fileURL string, err error) { + mt := mime.TypeByExtension(path.Ext(name)) + if mt == "" { + mt = "application/octet-stream" + } + + // 1. getUploadFileURL + uploadFileURLResp, err := c.getUploadFileURL(name, mt) + if err != nil { + err = fmt.Errorf("get upload file URL error: %s", err) + return + } + + // 2. Upload file to amazon - PUT + httpClient := http.DefaultClient + + req, err := http.NewRequest(http.MethodPut, uploadFileURLResp.SignedPutURL, file) + if err != nil { + return + } + req.ContentLength = size + req.TransferEncoding = []string{"identity"} // disable chunked (unsupported by aws) + req.Header.Set("Content-Type", mt) + req.Header.Set("User-Agent", userAgent) + + resp, err := httpClient.Do(req) + if err != nil { + return + } + + defer resp.Body.Close() + if resp.StatusCode != 200 { + var contents []byte + contents, err = ioutil.ReadAll(resp.Body) + if err != nil { + contents = []byte(fmt.Sprintf("Error from ReadAll: %s", err)) + } + + err = fmt.Errorf("http PUT '%s' failed with status %s: %s", req.URL, resp.Status, string(contents)) + return + } + + return uploadFileURLResp.FileID, uploadFileURLResp.URL, nil +} + +// getUploadFileURL executes a raw API call: POST /api/v3/getUploadFileUrl +func (c *webClient) getUploadFileURL(name, contentType string) (*GetUploadFileUrlResponse, error) { + req := &getUploadFileUrlRequest{ + Bucket: "secure", + ContentType: contentType, + Name: name, + } + + var rsp GetUploadFileUrlResponse + var err error + rsp.RawJSON, err = c.doNotionAPI("/getUploadFileUrl", req, &rsp) + if err != nil { + return nil, err + } + + rsp.Parse() + + return &rsp, nil +} + +func (r *GetUploadFileUrlResponse) Parse() { + r.FileID = strings.Split(r.URL[len(s3URLPrefix):], "/")[0] +} + +func (c *webClient) doNotionAPI(apiURL string, requestData interface{}, result interface{}) (map[string]interface{}, error) { + var js []byte + var err error + + if requestData != nil { + js, err = json.Marshal(requestData) + if err != nil { + return nil, err + } + } + uri := webAPI + apiURL + if apis.DebugMode { + log.Println(uri) + log.Printf("%s", js) + } + body := bytes.NewBuffer(js) + req, err := http.NewRequest("POST", uri, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept-Language", acceptLang) + req.Header.Set("cookie", fmt.Sprintf("token_v2=%v", c.token)) + var rsp *http.Response + + rsp, err = http.DefaultClient.Do(req) + + if err != nil { + return nil, err + } + defer closeNoError(rsp.Body) + + if rsp.StatusCode != 200 { + _, _ = ioutil.ReadAll(rsp.Body) + return nil, fmt.Errorf("http.Post('%s') returned non-200 status code of %d", uri, rsp.StatusCode) + } + d, err := ioutil.ReadAll(rsp.Body) + + if err != nil { + return nil, err + } + if apis.DebugMode { + log.Printf("%s", d) + } + // log.Prin("%s", d) + err = json.Unmarshal(d, result) + if err != nil { + return nil, err + } + var m map[string]interface{} + err = json.Unmarshal(d, &m) + if err != nil { + return nil, err + } + return m, nil +} + +func closeNoError(c io.Closer) { + _ = c.Close() +} + +// GetRecordValues executes a raw API call /api/v3/getRecordValues +func (c *webClient) GetPageData(pageid string) (*PageDataResponse, error) { + req := &PageDataRequest{ + Type: "block-space", + Blockid: pageid, + Name: "page", + Saveparent: false, + Showmoveto: false, + } + + var rsp PageDataResponse + var err error + if _, err = c.doNotionAPI("/getPublicPageData", req, &rsp); err != nil { + return nil, err + } + + return &rsp, nil +} + +// table is not always present in Record returned by the server +// so must be provided based on what was asked +func parseRecord(table string, r *Record) error { + // it's ok if some records don't return a value + if len(r.Value) == 0 { + return nil + } + if r.Table == "" { + r.Table = table + } else { + // TODO: probably never happens + return fmt.Errorf("%+v != %+v", r.Table, table) + } + + // set Block/Space etc. based on TableView type + var pRawJSON *map[string]interface{} + var obj interface{} + switch table { + case TableBlock: + r.Block = &Block{} + obj = r.Block + pRawJSON = &r.Block.RawJSON + } + if obj == nil { + return fmt.Errorf("ignored table '%s'", r.Table) + } + if false { + if table == TableCollectionView { + s := string(r.Value) + fmt.Printf("collection_view json:\n%s\n\n", s) + } + } + if err := json.Unmarshal(r.Value, pRawJSON); err != nil { + return err + } + id := (*pRawJSON)["id"] + if id != nil { + r.ID = id.(string) + } + if err := json.Unmarshal(r.Value, &obj); err != nil { + return err + } + return nil +} + +// LoadPageChunk executes a raw API call /api/v3/loadPageChunk +func (c *webClient) LoadPageChunk(pageID string, chunkNo int, cur *cursor) (*LoadPageChunkResponse, error) { // emulating notion's website api usage: 50 items on first request, + // 30 on subsequent requests + limit := 30 + if cur == nil { + cur = &cursor{ + // to mimic browser api which sends empty array for this argment + Stack: make([][]stack, 0), + } + limit = 50 + } + req := &loadPageChunkRequest{ + PageID: pageID, + ChunkNumber: chunkNo, + Limit: limit, + Cursor: *cur, + VerticalColumns: false, + } + var rsp LoadPageChunkResponse + var err error + if rsp.RawJSON, err = c.doNotionAPI("/loadPageChunk", req, &rsp); err != nil { + return nil, err + } + for _, r := range rsp.RecordMap.Blocks { + if err := parseRecord(TableBlock, r); err != nil { + return nil, err + } + } + return &rsp, nil +} diff --git a/apis/public/notion/struct.go b/apis/public/notion/struct.go index 5b21d74..1ada92a 100644 --- a/apis/public/notion/struct.go +++ b/apis/public/notion/struct.go @@ -1,29 +1,63 @@ package notion import ( - "github.com/kjk/notionapi" + "encoding/json" + + "golang.org/x/time/rate" ) -// POST /api/v3/getUploadFileUrl request -type getUploadFileUrlRequest struct { - Bucket string `json:"bucket"` - ContentType string `json:"contentType"` - Name string `json:"name"` +type webClient struct { + token string + limiter *rate.Limiter } -// GetUploadFileUrlResponse is a response to POST /api/v3/getUploadFileUrl -type GetUploadFileUrlResponse struct { - URL string `json:"url"` - SignedGetURL string `json:"signedGetUrl"` - SignedPutURL string `json:"signedPutUrl"` +type PageDataRequest struct { + Type string `json:"type"` + Name string `json:"name"` + Blockid string `json:"blockId"` + Showmoveto bool `json:"showMoveTo"` + Saveparent bool `json:"saveParent"` +} - FileID string `json:"-"` +type PageDataResponse struct { + Pageid string + Cursor *Block + Spacename string `json:"spaceName"` + Spaceid string `json:"spaceId"` + Canjoinspace bool `json:"canJoinSpace"` + Icon string `json:"icon"` + Userhasexplicitaccess bool `json:"userHasExplicitAccess"` + Haspublicaccess bool `json:"hasPublicAccess"` + Owneruserid string `json:"ownerUserId"` + Betaenabled bool `json:"betaEnabled"` + Canrequestaccess bool `json:"canRequestAccess"` +} - RawJSON map[string]interface{} `json:"-"` +// /api/v3/loadPageChunk request +type loadPageChunkRequest struct { + PageID string `json:"pageId"` + ChunkNumber int `json:"chunkNumber"` + Limit int `json:"limit"` + Cursor cursor `json:"cursor"` + VerticalColumns bool `json:"verticalColumns"` +} + +type cursor struct { + Stack [][]stack `json:"stack"` +} + +type stack struct { + ID string `json:"id"` + Index int `json:"index"` + Table string `json:"table"` } -type Client struct { - notionapi.Client +// LoadPageChunkResponse is a response to /api/v3/loadPageChunk api +type LoadPageChunkResponse struct { + RecordMap *RecordMap `json:"recordMap"` + Cursor cursor `json:"cursor"` + + RawJSON map[string]interface{} `json:"-"` } type submitTransactionRequest struct { @@ -37,6 +71,11 @@ type Transaction struct { Operations []*Operation `json:"operations"` } +type Pointer struct { + ID string `json:"id"` + Table string `json:"table"` +} + type Operation struct { Point Pointer `json:"pointer"` Path []string `json:"path"` @@ -44,7 +83,128 @@ type Operation struct { Args interface{} `json:"args"` } -type Pointer struct { - ID string `json:"id"` - Table string `json:"table"` +// RecordMap contains a collections of blocks, a space, users, and collections. +type RecordMap struct { + Activities map[string]*Record `json:"activity"` + Blocks map[string]*Record `json:"block"` + Spaces map[string]*Record `json:"space"` + Users map[string]*Record `json:"notion_user"` + Collections map[string]*Record `json:"collection"` + CollectionViews map[string]*Record `json:"collection_view"` + Comments map[string]*Record `json:"comment"` + Discussions map[string]*Record `json:"discussion"` +} + +// POST /api/v3/getUploadFileUrl request +type getUploadFileUrlRequest struct { + Bucket string `json:"bucket"` + ContentType string `json:"contentType"` + Name string `json:"name"` +} + +// GetUploadFileUrlResponse is a response to POST /api/v3/getUploadFileUrl +type GetUploadFileUrlResponse struct { + URL string `json:"url"` + SignedGetURL string `json:"signedGetUrl"` + SignedPutURL string `json:"signedPutUrl"` + FileID string `json:"-"` + RawJSON map[string]interface{} `json:"-"` +} + +// Record represents a polymorphic record +type Record struct { + // fields returned by the server + Role string `json:"role"` + // polymorphic value of the record, which we decode into Block, Space etc. + Value json.RawMessage `json:"value"` + + // fields set from Value based on type + ID string `json:"-"` + Table string `json:"-"` + Block *Block `json:"-"` + // TODO: add more types +} + +// Block describes a block +type Block struct { + // values that come from JSON + // a unique ID of the block + ID string `json:"id"` + // if false, the page is deleted + Alive bool `json:"alive"` + // List of block ids for that make up content of this block + // Use Content to get corresponding block (they are in the same order) + ContentIDs []string `json:"content,omitempty"` + CopiedFrom string `json:"copied_from,omitempty"` + CollectionID string `json:"collection_id,omitempty"` // for BlockCollectionView + // ID of the user who created this block + CreatedBy string `json:"created_by"` + CreatedTime int64 `json:"created_time"` + + CreatedByTable string `json:"created_by_table"` // e.g. "notion_user" + CreatedByID string `json:"created_by_id"` // e.g. "bb760e2d-d679-4b64-b2a9-03005b21870a", + LastEditedByTable string `json:"last_edited_by_table"` // e.g. "notion_user" + LastEditedByID string `json:"last_edited_by_id"` // e.g. "bb760e2d-d679-4b64-b2a9-03005b21870a" + + // List of block ids with discussion content + DiscussionIDs []string `json:"discussion,omitempty"` + // those ids seem to map to storage in s3 + // https://s3-us-west-2.amazonaws.com/secure.notion-static.com/${id}/${name} + FileIDs []string `json:"file_ids,omitempty"` + + // TODO: don't know what this means + IgnoreBlockCount bool `json:"ignore_block_count,omitempty"` + + // ID of the user who last edited this block + LastEditedBy string `json:"last_edited_by"` + LastEditedTime int64 `json:"last_edited_time"` + // ID of parent Block + ParentID string `json:"parent_id"` + ParentTable string `json:"parent_table"` + Properties map[string]interface{} `json:"properties,omitempty"` + // type of the block e.g. TypeText, TypePage etc. + Type string `json:"type"` + // blocks are versioned + Version int64 `json:"version"` + // for BlockCollectionView + ViewIDs []string `json:"view_ids,omitempty"` + + // Parent of this block + Parent *Block `json:"-"` + + // maps ContentIDs array to Block type + Content []*Block `json:"-"` + + // for BlockPage + Title string `json:"-"` + + // For BlockTodo, a checked state + IsChecked bool `json:"-"` + + // for BlockBookmark + Description string `json:"-"` + Link string `json:"-"` + + // for BlockBookmark it's the url of the page + // for BlockGist it's the url for the gist + // fot BlockImage it's url of the image, but use ImageURL instead + // because Source is sometimes not accessible + // for BlockFile it's url of the file + // for BlockEmbed it's url of the embed + Source string `json:"-"` + + // for BlockFile + FileSize string `json:"-"` + + // for BlockImage it's an URL built from Source that is always accessible + ImageURL string `json:"-"` + + // for BlockCode + Code string `json:"-"` + CodeLanguage string `json:"-"` + + // RawJSON represents Block as + RawJSON map[string]interface{} `json:"-"` + + isResolved bool } diff --git a/apis/public/notion/upload.go b/apis/public/notion/upload.go index 44cb944..64b5c14 100644 --- a/apis/public/notion/upload.go +++ b/apis/public/notion/upload.go @@ -1,37 +1,20 @@ package notion import ( - "bytes" "encoding/json" "fmt" "io" - "io/ioutil" "log" - "mime" - "net/http" "net/url" - "path" - "strings" "time" "transfer/apis" "transfer/utils" - - "github.com/google/uuid" - "github.com/kjk/notionapi" ) // Command Types const ( - CommandSet = "set" - CommandUpdate = "update" - CommandListAfter = "listAfter" - CommandListRemove = "listRemove" - signedURLPrefix = "https://www.notion.so/signed" - s3URLPrefix = "https://s3-us-west-2.amazonaws.com/secure.notion-static.com/" notionHost = "https://www.notion.so" - userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3483.0 Safari/537.36" - acceptLang = "en-US,en;q=0.9" ) func PrintStruct(emp interface{}) { @@ -42,253 +25,40 @@ func PrintStruct(emp interface{}) { fmt.Printf("MarshalIndent funnction output\n %s\n", string(empJSON)) } -func ByteCountIEC(b int64) string { - const unit = 1024 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %ciB", - float64(b)/float64(div), "KMGTPE"[exp]) -} - -func SubmitTransaction(ops []*Operation, spaceID string, client *Client) error { - - reqData := &submitTransactionRequest{ - RequestID: uuid.New().String(), - Transaction: []Transaction{{ - ID: uuid.New().String(), - SpaceID: spaceID, - Operations: ops, - }}, - } - if apis.DebugMode { - PrintStruct(reqData) - } - var rsp map[string]interface{} - data, err := doNotionAPI(client, "/api/v3/saveTransactions", reqData, &rsp) - if apis.DebugMode { - PrintStruct(data) - } - return err -} - -func closeNoError(c io.Closer) { - _ = c.Close() -} - -func doNotionAPI(c *Client, apiURL string, requestData interface{}, result interface{}) (map[string]interface{}, error) { - var js []byte - var err error - if requestData != nil { - js, err = json.Marshal(requestData) - if err != nil { - return nil, err - } - } - uri := notionHost + apiURL - body := bytes.NewBuffer(js) - req, err := http.NewRequest("POST", uri, body) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", userAgent) - req.Header.Set("Accept-Language", acceptLang) - if c.AuthToken != "" { - req.Header.Set("cookie", fmt.Sprintf("token_v2=%v", c.AuthToken)) - } - var rsp *http.Response - - rsp, err = http.DefaultClient.Do(req) - - if err != nil { - return nil, err - } - defer closeNoError(rsp.Body) - - if rsp.StatusCode != 200 { - _, _ = ioutil.ReadAll(rsp.Body) - return nil, fmt.Errorf("http.Post('%s') returned non-200 status code of %d", uri, rsp.StatusCode) - } - d, err := ioutil.ReadAll(rsp.Body) - if err != nil { - return nil, err - } - err = json.Unmarshal(d, result) - if err != nil { - return nil, err - } - var m map[string]interface{} - err = json.Unmarshal(d, &m) - if err != nil { - return nil, err - } - return m, nil -} - -// buildOp creates an Operation for this block -func buildOp(blockID, command string, path []string, args interface{}) *Operation { - return &Operation{ - Point: Pointer{ - ID: blockID, - Table: "block", - }, - Path: path, - Command: command, - Args: args, - } -} - func (b *notion) DoUpload(name string, size int64, file io.Reader) error { if b.pageID == "" || b.token == "" { return fmt.Errorf("invalid pageid or token") } - client := &Client{notionapi.Client{AuthToken: b.token}} - page, err := client.DownloadPage(b.pageID) + client := NewWebClient(b.token) + root, err := client.GetPage(b.pageID) if err != nil { - log.Fatalf("DownloadPage() failed with %s\n", err) + log.Fatalf("GetPage() failed with %s\n", err) } - - root := page.BlockByID(page.ID) if apis.DebugMode { PrintStruct(root) } fileID, fileURL, err := client.UploadFile(file, name, size) if err != nil { - log.Fatalf("DownloadPage() failed with %s\n", err) + log.Fatalf("UploadFile() failed with %s\n", err) } - - var lastBlockID string - if len(root.Content) > 0 { - lastBlockID = root.Content[len(root.Content)-1].ID - } - - userID := root.LastEditedByID - spaceID := root.ParentID - if b.spaceID != "" { - spaceID = b.spaceID + if apis.DebugMode { + log.Printf("id: %s, url: %s", fileID, fileURL) } - newBlockID := uuid.New().String() fmt.Printf("syncing blocks..") end := utils.DotTicker() - - ops := []*Operation{ - buildOp(newBlockID, CommandSet, []string{}, map[string]interface{}{ - "type": "file", - "id": newBlockID, - "version": 1, - }), - buildOp(newBlockID, CommandUpdate, []string{}, map[string]interface{}{ - "parent_id": root.ID, - "parent_table": "block", - "alive": true, - }), - buildOp(root.ID, CommandListAfter, []string{"content"}, map[string]string{ - "id": newBlockID, - "after": lastBlockID, - }), - buildOp(newBlockID, CommandSet, []string{"created_by_id"}, userID), - buildOp(newBlockID, CommandSet, []string{"created_by_table"}, "notion_user"), - buildOp(newBlockID, CommandSet, []string{"created_time"}, time.Now().UnixNano()), - buildOp(newBlockID, CommandSet, []string{"last_edited_time"}, time.Now().UnixNano()), - buildOp(newBlockID, CommandSet, []string{"last_edited_by_id"}, userID), - buildOp(newBlockID, CommandSet, []string{"last_edited_by_table"}, "notion_user"), - buildOp(newBlockID, CommandUpdate, []string{"properties"}, map[string]interface{}{ - "source": [][]string{{fileURL}}, - "size": [][]string{{ByteCountIEC(size)}}, - "title": [][]string{{name}}, - }), - buildOp(newBlockID, CommandListAfter, []string{"file_ids"}, map[string]string{ - "id": fileID, - }), + newBlockID, err := client.insertFile(name, fileID, fileURL, root, size, time.Now()) + if err != nil { + log.Fatalf("insertFile() failed with %s\n", err) } - SubmitTransaction(ops, spaceID, client) *end <- struct{}{} fmt.Printf("%s\n", newBlockID) - b.resp = fmt.Sprintf("%s/%s?table=block&id=%s&name=%s&userId=%s&cache=v2", signedURLPrefix, url.QueryEscape(fileURL), newBlockID, name, userID) + b.resp = fmt.Sprintf("%s/%s?table=block&id=%s&name=%s&userId=%s&cache=v2", signedURLPrefix, url.QueryEscape(fileURL), newBlockID, name, root.Owneruserid) return nil } -// getUploadFileURL executes a raw API call: POST /api/v3/getUploadFileUrl -func (c *Client) getUploadFileURL(name, contentType string) (*GetUploadFileUrlResponse, error) { - const apiURL = "/api/v3/getUploadFileUrl" - - req := &getUploadFileUrlRequest{ - Bucket: "secure", - ContentType: contentType, - Name: name, - } - - var rsp GetUploadFileUrlResponse - var err error - rsp.RawJSON, err = doNotionAPI(c, apiURL, req, &rsp) - if err != nil { - return nil, err - } - - rsp.Parse() - - return &rsp, nil -} - -func (r *GetUploadFileUrlResponse) Parse() { - r.FileID = strings.Split(r.URL[len(s3URLPrefix):], "/")[0] -} - -// UploadFile Uploads a file to notion's asset hosting(aws s3) -func (c *Client) UploadFile(file io.Reader, name string, size int64) (fileID, fileURL string, err error) { - ext := path.Ext(name) - mt := mime.TypeByExtension(ext) - if mt == "" { - mt = "application/octet-stream" - } - // 1. getUploadFileURL - uploadFileURLResp, err := c.getUploadFileURL(name, mt) - if err != nil { - err = fmt.Errorf("get upload file URL error: %s", err) - return - } - - // 2. Upload file to amazon - PUT - httpClient := http.DefaultClient - - req, err := http.NewRequest(http.MethodPut, uploadFileURLResp.SignedPutURL, file) - if err != nil { - return - } - req.ContentLength = size - req.TransferEncoding = []string{"identity"} // disable chunked (unsupported by aws) - req.Header.Set("Content-Type", mt) - req.Header.Set("User-Agent", userAgent) - - resp, err := httpClient.Do(req) - if err != nil { - return - } - - defer resp.Body.Close() - if resp.StatusCode != 200 { - var contents []byte - contents, err = ioutil.ReadAll(resp.Body) - if err != nil { - contents = []byte(fmt.Sprintf("Error from ReadAll: %s", err)) - } - - err = fmt.Errorf("http PUT '%s' failed with status %s: %s", req.URL, resp.Status, string(contents)) - return - } - - return uploadFileURLResp.FileID, uploadFileURLResp.URL, nil -} - func (b notion) PostUpload(string, int64) (string, error) { fmt.Printf("Download Link: %s\n", b.resp) return b.resp, nil diff --git a/go.mod b/go.mod index 46d1dbc..cdc483f 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,15 @@ module transfer go 1.14 require ( - github.com/cheggaaa/pb/v3 v3.0.6 - github.com/fatih/color v1.10.0 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/cheggaaa/pb/v3 v3.0.8 + github.com/fatih/color v1.11.0 // indirect github.com/google/uuid v1.2.0 github.com/kjk/notionapi v0.0.0-20210312181036-c1df7a1b08cd - github.com/mattn/go-runewidth v0.0.10 // indirect - github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 + github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/cobra v1.1.3 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 - golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c // indirect + golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba ) diff --git a/go.sum b/go.sum index 035a22e..c78e1da 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= @@ -33,6 +35,8 @@ github.com/cheggaaa/pb/v3 v3.0.5 h1:lmZOti7CraK9RSjzExsY53+WWfub9Qv13B5m4ptEoPE= github.com/cheggaaa/pb/v3 v3.0.5/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw= github.com/cheggaaa/pb/v3 v3.0.6 h1:ULPm1wpzvj60FvmCrX7bIaB80UgbhI+zSaQJKRfCbAs= github.com/cheggaaa/pb/v3 v3.0.6/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw= +github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= +github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -49,6 +53,8 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.11.0 h1:l4iX0RqNnx/pU7rY2DB/I+znuYY0K3x6Ywac6EIr0PA= +github.com/fatih/color v1.11.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -138,6 +144,8 @@ github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+tw github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/minio-go/v6 v6.0.44/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= @@ -156,6 +164,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU= github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= +github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc h1:Ak86L+yDSOzKFa7WM5bf5itSOo1e3Xh8bm5YCMUXIjQ= +github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -295,12 +305,17 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEq golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c h1:coiPEfMv+ThsjULRDygLrJVlNE1gDdL2g65s0LhV2os= golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=