diff --git a/README.md b/README.md index 212cf89..870c6f4 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,13 @@ OR, if you just want to start playing with the library run... `dbox` methods (where dbox is set from requiring the dbox library)... app <-- creates application object - + `app` methods (where app is created from the above `app` call)... requesttoken <-- creates request token for getting request token and authorization url accesstoken <-- creates access token for creating a client object client <-- creates client object with access to users dropbox account - + `client` methods (where client is created from the above `client` call)... account <-- view account @@ -57,6 +57,12 @@ OR, if you just want to start playing with the library run... stream <-- creates readable stream readdir <-- recursively reads directory +extensions to `client` methods added in this fork... + + putChunked <-- upload large files in multiple chunks -- uses + commitChunked <-- complete initiated chunked upload -- uses + uploadChunked <-- automatically performs series of `putChunked` calls then `commitChunked` to upload file via chunks + ## How to Use Creating a functional `dbox` client is a four step process. @@ -70,7 +76,7 @@ Creating a functional `dbox` client is a four step process. var dbox = require("dbox") var app = dbox.app({ "app_key": "umdez34678ck01fx", "app_secret": "tjm89017sci88o6" }) - + ### Step 2 Authorization is a three step process. @@ -106,18 +112,18 @@ Returns account information. client.account(function(status, reply){ console.log(reply) }) - + output of `reply` returns... - { + { uid: 123456789, display_name: 'Brock Whitten', email: 'brock@sintaxi.com', country: 'CA', referral_link: 'https://www.dropbox.com/referrals/NTc0NzYwNDc5', - quota_info: { - shared: 1100727791, - quota: 2415919104, + quota_info: { + shared: 1100727791, + quota: 2415919104, normal: 226168599 } } @@ -175,7 +181,7 @@ Copies a file or directory to a new location. client.cp("bar", "baz", function(status, reply){ console.log(reply) }) - + { "size": "0 bytes", "rev": "irt77dd3728", @@ -213,7 +219,7 @@ output of `reply` returns... "mime_type": "text/plain", "revision": 492341 } - + ### put(path, data, [options,] callback) Creates or modifies a file with given data. `data` may be a string or a buffer. @@ -221,9 +227,9 @@ Creates or modifies a file with given data. `data` may be a string or a buffer. client.put("foo/hello.txt", "here is some text", function(status, reply){ console.log(reply) }) - + output of `reply` returns... - + { "size": "225.4KB", "rev": "35e97029684fe", @@ -247,7 +253,7 @@ Pulls down file (available as a buffer) with its metadata. }) output of `reply.toString()` returns... - + here is some text output of `metadata` returns... @@ -287,7 +293,7 @@ Retrieves file or directory metadata. }) output of `reply` returns... - + { "size": "225.4KB", "rev": "35e97029684fe", @@ -317,7 +323,7 @@ Obtains metadata for the previous revisions of a file. }) output of `reply` returns... - + [ { "is_deleted": true, @@ -355,7 +361,7 @@ Restores a file path to a previous revision. client.revisions("foo/hello.txt", 4, function(status, reply){ console.log(reply) }) - + output of `reply` returns... { @@ -372,7 +378,7 @@ output of `reply` returns... "mime_type": "text/plain", "size": "0 bytes" } - + ### search(path, query, [options,] callback) Returns metadata for all files and directories that match the search query. @@ -466,7 +472,7 @@ output of `metadata` returns... "root": "app_folder", "mime_type": "image/jpeg", "size": "762.5 KB" - } + } ### cpref(path, [options,] callback) @@ -480,7 +486,7 @@ output of `reply` returns... expires: 'Thu, 03 Apr 2042 22:33:49 +0000', copy_ref: 'ALGf72Jrc3A0ZTh5MzA4Mg' } - + ### delta([options,] callback) client.delta(function(status, reply){ @@ -498,19 +504,72 @@ output of `reply` returns... [ '/bar', [Object] ] ] } - + ### readdir(path, callback) Get an array of paths for all files and directories found in the given path. The method calls recursively to dropbox so it can take a long time to evaluate. - + client.readdir('/', function(status, reply){ console.log(reply) }) Output of `readdir` returns... - + ['/','/foo','/bar'] + +### putChunked(buffer, [args,] cb) + +Uploads file in multiple chunks. `buffer` should be a buffer containing the current chunk of data. If it is the first chunk, then `args` should be null. For subsequent chunks, `args` should contain the `upload_id` returned via the callback from the first chunk uploaded, and the current `offset` returned via the callback for the most recent chunk uploaded. + + client.putChunked(buffer, args, function(status, reply) { + if (status != 200) { + // error + return cb(status, reply); // err is status + } + // then send more chunks + // ... + // then `commitChunked` after last chunk uploaded + // ... + }); + +### commitChunked(path, upload_id, cb) + +Completes previously initiated chunked upload. `path` should contain path for saving remote file, and `upload_id` shoould contain the `upload_id` for the chunked upload. + + client.commitChunked(destinationFilename, upload_id, function (status, reply) { + if (status != 200) { + return cb(status, reply); // err is status + } + return cb(null, reply); // err is null + }); + +### uploadChunked: function (localPath, remotePath, options, cb) + +Automatically performs series of `putChunked` calls then `commitChunked` to upload file via chunks + + client.uploadChunked(source, destination, { + "upload_id": null, + "offset": 0, + "onBeginChunkUpload": function(upload_id, chunkSize, chunkOffset) { + /* could inform user here that chunk upload is beginning */ + }, + "onEndChunkUpload": function(status, reply) { + /* could inform user here that chunk finished uploading */ + }, + "chunkSize": chunkSize + }, function(status, reply) { + + if (status !== 200) { + // upload failed + + return cb(status, reply); // err is status + } + + // upload succeeded + cb(null, reply); // err is null + } + ## License Copyright 2011 Chloi Inc. diff --git a/lib/dbox.js b/lib/dbox.js index b2d317b..f110366 100644 --- a/lib/dbox.js +++ b/lib/dbox.js @@ -5,7 +5,7 @@ var path = require("path") exports.app = function(config){ var root = config.root || "sandbox" var helpers = require("./helpers")(config) - + return { root: root, @@ -14,7 +14,7 @@ exports.app = function(config){ var body = qs.stringify(signature) var args = { "method": "POST", - "headers": { + "headers": { "content-type": "application/x-www-form-urlencoded", "content-length": body.length }, @@ -33,7 +33,7 @@ exports.app = function(config){ var body = qs.stringify(signature) var args = { "method": "POST", - "headers": { + "headers": { "content-type": "application/x-www-form-urlencoded", "content-length": body.length }, @@ -62,13 +62,13 @@ exports.app = function(config){ cb(e ? null : r.statusCode, e ? null : helpers.parseJSON(b)) }) }, - + delta: function(args, cb){ if(!cb){ cb = args args = {} } - + if (config.scope) { args.path_prefix = path.join("/", config.scope); } @@ -76,29 +76,29 @@ exports.app = function(config){ var entries = [] var REQUEST_CONCURRENCY_DELAY = 20 var reset; - + var fetch = function(args){ var signature = helpers.sign(options, args) var body = qs.stringify(signature) var opts = { "method": "POST", - "headers": { + "headers": { "content-type": "application/x-www-form-urlencoded", "content-length": body.length }, "url": "https://api.dropbox.com/1/delta", "body": body } - + return request(opts, function(e, r, b){ var status = e ? null : r.statusCode var output = helpers.parseJSON(b) - + if(typeof reset == 'undefined'){ reset = output.reset } - if(output && output.hasOwnProperty("entries")){ + if(output && output.hasOwnProperty("entries")){ output["entries"].forEach(function(entry){ entries.push(entry) }) @@ -110,13 +110,13 @@ exports.app = function(config){ if(output){ output["entries"] = entries output["reset"] = reset - } + } // console.log("MADE IT:", status, output) cb(status, output) } }) } - + fetch(args) }, @@ -126,9 +126,9 @@ exports.app = function(config){ cb = args args = null } - + var signature = helpers.sign(options, args) - + var url = helpers.url({ hostname: "api-content.dropbox.com", action: "files", @@ -141,7 +141,7 @@ exports.app = function(config){ "url": url, "encoding": null } - + return request(args, function(e, r, b) { if (e) { cb(null, null, null); @@ -152,9 +152,9 @@ exports.app = function(config){ }) }, - stream: function(path, args) { + stream: function(path, args) { var signature = helpers.sign(options, args) - + var url = helpers.url({ hostname: "api-content.dropbox.com", action: "files", @@ -169,14 +169,14 @@ exports.app = function(config){ } return request(args); - }, + }, put: function(path, body, args, cb){ if(!cb){ cb = args args = null } - + var signature = helpers.sign(options, args) var url = helpers.url({ @@ -185,41 +185,237 @@ exports.app = function(config){ path: path, query: signature }) - + var args = { "method": "PUT", "headers": { "content-length": body.length }, "url": url } - + // do not send empty body if(body.length > 0) args["body"] = body - + return request(args, function(e, r, b){ cb(e ? null : r.statusCode, e ? null : helpers.parseJSON(b)) }) }, + putChunked: function (buffer, args, cb) { + if (!cb) { + cb = args; + args = { + offset: 0 + }; + } + + var signature = helpers.sign(options, args); + + var url = helpers.url({ + hostname: 'api-content.dropbox.com', + action: 'chunked_upload', + query: signature + }); + + args = { + 'method': 'PUT', + 'url': url.replace(/chunked_upload\/dropbox/i, 'chunked_upload') + }; + + // do not send empty body + if (buffer.length > 0) { + args.body = buffer; + } + + return request(args, function (e, r, b) { + cb(e ? null : r.statusCode, e ? null : helpers.parseJSON(b)); + }); + }, + + commitChunked: function (path, upload_id, cb) { + + + if (!upload_id) { + return cb('upload_id is required'); + } + + var args = { "upload_id": upload_id }; + + var signature = helpers.sign(options, args); + + var url = helpers.url({ + hostname: 'api-content.dropbox.com', + action: 'commit_chunked_upload', + path: path, + query: signature + }); + + args = { + 'method': 'POST', + 'url': url.replace(/commit_chunked_upload\/dropbox/gi, 'commit_chunked_upload/auto') + }; + + return request(args, function (e, r, b) { + cb(e ? null : r.statusCode, e ? null : helpers.parseJSON(b)); + }); + }, + + uploadChunked: function (localPath, remotePath, options, cb) { + + var self = this; + + var defaultOptions = { + "upload_id": null, // default null; `upload_id` of a previously initiated, unfinished chunked upload + "offset": 0, // default 0; `offset` of a previously initiated, unfinished chunked upload + "chunkSize": 5248288, // default 524288; chunkSize in bytes; recommend 524288 (512KB) or 1048576 (1MB) + "onBeginChunkUpload": function(upload_id, chunkSize, offset) {}, // function to be notified before each chunk is uploaded + "onEndChunkUpload": function(status, reply, upload_id, chunkSize, offset) {} // this will get called right after each chunk is uploaded + }; + + if (!options) { + cb = options; + options = defaultOptions; + } + + if (!localPath) { + return cb('localPath is required'); + } + + if (!remotePath) { + return cb('remotePath is required'); + } + + // Assign default options for any not provided + for (var key in defaultOptions) { + if (defaultOptions.hasOwnProperty(key)) { + options[key] = options[key] || defaultOptions[key]; + } + } + + + var fs = require('fs'); + fs.stat(localPath, function (err, stats) { + + var upload_id = options.upload_id || null, + offset = parseInt(options.offset || 0), + chunkSize = parseInt(options.chunkSize || 0); + + if (err) { + return cb({ + 'message': 'unable to get stats for file', + 'err': err + }); + } + + fs.open(localPath, 'r', function (err, fd) { + var buffer, + fileBytesUploaded = 0, + fileBytesRemaining = stats.size; + + if (err) { + return cb({ + 'message': 'unable to open file', + 'err': err + }); + } + + buffer = new Buffer(chunkSize); + + function putChunks() { + fs.read(fd, buffer, 0, chunkSize, offset, function (err, bytesRead, buffer) { + var chunkNumber; + if (err) { + return cb({ + 'message': 'unable to read buffer', + 'err': err + }); + } + // If on last chunk, bytes read might be less than chunk size + if (bytesRead !== chunkSize) { + buffer = buffer.slice(0, bytesRead); + chunkSize = bytesRead; + } + + if (options.onBeginChunkUpload) { + options.onBeginChunkUpload(upload_id, chunkSize, offset); + } + + // Now upload the buffer contents + self.putChunked(buffer, { + 'upload_id': upload_id, + 'offset': offset + }, function (status, reply) { + + if (status === 400) { + // offset was wrong. try again at correct offset + offset = reply.offset; + if (options.onEndChunkUpload) { + options.onEndChunkUpload(status, reply); + } + return putChunks(); + } + + if (status === 404) { + // The upload_id does not exist or has expired. + upload_id = null; + offset = 0; + fileBytesUploaded = 0; + if (options.onEndChunkUpload) { + options.onEndChunkUpload(status, reply); + } + return putChunks(); + } + + if (status !== 200) { + // An unknown error occurred + return cb(status, reply); + } + + offset = reply.offset; + upload_id = reply.upload_id; + + if (options.onEndChunkUpload) { + options.onEndChunkUpload(status, reply); + } + + fileBytesUploaded = offset; + fileBytesRemaining = stats.size - offset; + + if (offset === stats.size) { + return self.commitChunked(remotePath, upload_id, cb); + } + putChunks(); // put next chunk + }); + + }); + } + putChunks(); // put first chunk + + }); + + }); + + }, + metadata: function(path, args, cb){ if(!cb){ cb = args args = null } - + var signature = helpers.sign(options, args) - + var url = helpers.url({ hostname: "api.dropbox.com", action: "metadata", path: path, query: signature }) - + var args = { "method": "GET", "url": url } - + return request(args, function(e, r, b){ // this is a special case, since the dropbox api returns a // 304 response with an empty body when the 'hash' option @@ -299,9 +495,9 @@ exports.app = function(config){ cb = args args = null } - + var signature = helpers.sign(options, args) - + var url = helpers.url({ hostname: "api.dropbox.com", action: "revisions", @@ -323,10 +519,10 @@ exports.app = function(config){ cb = args args = null } - + var signature = helpers.sign(options, args) signature["rev"] = rev - + var url = helpers.url({ hostname: "api.dropbox.com", action: "restore", @@ -334,7 +530,7 @@ exports.app = function(config){ }) var body = qs.stringify(signature) - + var args = { "method": "POST", "headers": { @@ -344,22 +540,22 @@ exports.app = function(config){ "url": url, "body": body } - + return request(args, function(e, r, b){ cb(e ? null : r.statusCode, e ? null : helpers.parseJSON(b)) }) }, search: function(path, query, args, cb){ - + if(!cb){ cb = args args = null } - + var signature = helpers.sign(options, args) signature["query"] = query - + var url = helpers.url({ hostname: "api.dropbox.com", action: "search", @@ -371,7 +567,7 @@ exports.app = function(config){ "method": "POST", "headers": { "content-type": "application/x-www-form-urlencoded", - "content-length": body.length + "content-length": body.length }, "url": url, "body": body @@ -386,24 +582,24 @@ exports.app = function(config){ cb = args args = null } - + var signature = helpers.sign(options, args) - + var url = helpers.url({ hostname: "api.dropbox.com", action: "shares", path: path }) - + var body = qs.stringify(signature) - + var args = { "method": "POST", "headers": { "content-type": "application/x-www-form-urlencoded", - "content-length": body.length + "content-length": body.length }, - "url": url, + "url": url, "body": body } return request(args, function(e, r, b){ @@ -416,22 +612,22 @@ exports.app = function(config){ cb = args args = null } - + var signature = helpers.sign(options, args) - + var url = helpers.url({ hostname: "api.dropbox.com", action: "media", path: path }) - + var body = qs.stringify(signature) - + var args = { "method": "POST", "headers": { "content-type": "application/x-www-form-urlencoded", - "content-length": body.length + "content-length": body.length }, "url": url, "body": body @@ -446,16 +642,16 @@ exports.app = function(config){ cb = args args = null } - + var signature = helpers.sign(options, args) - + var url = helpers.url({ hostname: "api.dropbox.com", action: "copy_ref", path: path, query: signature }) - + var args = { "method": "GET", "url": url @@ -470,9 +666,9 @@ exports.app = function(config){ cb = args args = null } - + var signature = helpers.sign(options, args) - + var url = helpers.url({ hostname: "api-content.dropbox.com", action: "thumbnails", @@ -485,7 +681,7 @@ exports.app = function(config){ "url": url, "encoding": null } - + return request(args, function(e, r, b){ if (e) { cb(null, null, null) @@ -503,28 +699,28 @@ exports.app = function(config){ } var signature = helpers.sign(options, args) - + // check for copy ref if(from_path.hasOwnProperty("copy_ref")){ signature['from_copy_ref'] = from_path["copy_ref"] }else{ signature['from_path'] = helpers.filePath(from_path) } - + signature["root"] = root // API quirk that this is reqired for this call signature["to_path"] = helpers.filePath(to_path) - - + + var url = helpers.url({ hostname: "api.dropbox.com", action: "fileops/copy" }) - + var body = qs.stringify(signature) var args = { "method": "POST", - "headers": { + "headers": { "content-type": "application/x-www-form-urlencoded", "content-length": body.length }, @@ -543,21 +739,21 @@ exports.app = function(config){ } var signature = helpers.sign(options, args) - + signature["root"] = root // API quirk that this is reqired for this call signature["from_path"] = helpers.filePath(from_path) signature["to_path"] = helpers.filePath(to_path) - + var url = helpers.url({ hostname: "api.dropbox.com", action: "fileops/move" }) var body = qs.stringify(signature) - + var args = { "method": "POST", - "headers": { + "headers": { "content-type": "application/x-www-form-urlencoded", "content-length": body.length }, @@ -577,20 +773,20 @@ exports.app = function(config){ } var signature = helpers.sign(options, args) - + signature["root"] = root signature["path"] = helpers.filePath(path) - + var url = helpers.url({ hostname: "api.dropbox.com", action: "fileops/delete" }) - + var body = qs.stringify(signature) - + var args = { "method": "POST", - "headers": { + "headers": { "content-type": "application/x-www-form-urlencoded", "content-length": body.length }, @@ -609,20 +805,20 @@ exports.app = function(config){ } var signature = helpers.sign(options, args) - + signature["root"] = root signature["path"] = helpers.filePath(path) - + var url = helpers.url({ hostname: "api.dropbox.com", action: "fileops/create_folder" }) - + var body = qs.stringify(signature) - + var args = { "method": "POST", - "headers": { + "headers": { "content-type": "application/x-www-form-urlencoded", "content-length": body.length }, @@ -635,7 +831,7 @@ exports.app = function(config){ } } } - } + } }