From 080572fdad2e7decd415c1002d3bb55f153bc198 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Thu, 26 Jul 2018 15:48:01 -0700 Subject: [PATCH 1/5] Use r2 API client using open-api derived methods Its important we can upload using node streams, because of the potential for very large files in deploys. --- README.md | 4 +- docs/deploy.md | 6 +- docs/sites.md | 6 +- docs/whoami.md | 4 +- package.json | 6 +- src/base.js | 4 +- src/commands/deploy/index.js | 6 +- src/commands/link/index.js | 6 +- src/commands/open/index.js | 2 +- src/commands/sites/list/index.js | 2 +- src/commands/status/index.js | 7 +- src/commands/whoami/index.js | 2 +- src/utils/api/README.md | 110 +++++++++++++++++++++++ src/utils/api/access-token.js | 28 ------ src/utils/api/browser/generate-method.js | 92 +++++++++++++++++++ src/utils/api/browser/index.js | 85 ++++++++++++++++++ src/utils/api/browser/index.test.js | 79 ++++++++++++++++ src/utils/api/browser/shape-swagger.js | 28 ++++++ src/utils/api/browser/util.js | 31 +++++++ src/utils/api/deploy/file-hasher.js | 4 +- src/utils/api/deploy/index.js | 51 ++++------- src/utils/api/index.js | 38 ++------ 22 files changed, 478 insertions(+), 123 deletions(-) create mode 100644 src/utils/api/README.md delete mode 100644 src/utils/api/access-token.js create mode 100644 src/utils/api/browser/generate-method.js create mode 100644 src/utils/api/browser/index.js create mode 100644 src/utils/api/browser/index.test.js create mode 100644 src/utils/api/browser/shape-swagger.js create mode 100644 src/utils/api/browser/util.js diff --git a/README.md b/README.md index 9bfff12d863..4eed0d17213 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ USAGE # Command Topics -* [`netlify-cli deploy`](docs/deploy.md) - Create a new deploy from the contents of a folder +* [`netlify-cli deploy`](docs/deploy.md) - Create a new deploy from the contents of a folder. * [`netlify-cli forms`](docs/forms.md) - Handle form operations * [`netlify-cli functions`](docs/functions.md) - Manage netlify functions * [`netlify-cli link`](docs/link.md) - Link a local repo or project folder to an existing site on Netlify @@ -35,7 +35,7 @@ USAGE * [`netlify-cli logout`](docs/logout.md) - Logout of account * [`netlify-cli sites`](docs/sites.md) - Handle site operations * [`netlify-cli status`](docs/status.md) - Print currently logged in use -* [`netlify-cli whoami`](docs/whoami.md) - Print currently logged in use +* [`netlify-cli whoami`](docs/whoami.md) - Print currently logged in user and account info diff --git a/docs/deploy.md b/docs/deploy.md index 35b10a48fb0..4a71a0cc458 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -1,20 +1,20 @@ `netlify-cli deploy` ==================== -Create a new deploy from the contents of a folder +Create a new deploy from the contents of a folder. * [`netlify-cli deploy [PUBLISHFOLDER]`](#netlify-cli-deploy-publishfolder) ## `netlify-cli deploy [PUBLISHFOLDER]` -Create a new deploy from the contents of a folder +Create a new deploy from the contents of a folder. ``` USAGE $ netlify-cli deploy [PUBLISHFOLDER] ARGUMENTS - PUBLISHFOLDER folder to deploy + PUBLISHFOLDER folder to deploy (optional) ``` _See code: [src/commands/deploy/index.js](https://github.com/netlify/cli/blob/v0.0.0/src/commands/deploy/index.js)_ diff --git a/docs/sites.md b/docs/sites.md index c0e050c72df..686baf028ba 100644 --- a/docs/sites.md +++ b/docs/sites.md @@ -22,8 +22,10 @@ DESCRIPTION The sites command will help you manage all your sites EXAMPLES - $ netlify sites:create -name my-new-site - $ netlify sites:update -name my-new-site + $ netlify sites:create --name my-new-site + $ netlify sites:update --name my-new-site + $ netlify sites:delete --name my-new-site + $ netlify sites:list ``` _See code: [src/commands/sites/index.js](https://github.com/netlify/cli/blob/v0.0.0/src/commands/sites/index.js)_ diff --git a/docs/whoami.md b/docs/whoami.md index 15f6857e25c..b0debcb9c11 100644 --- a/docs/whoami.md +++ b/docs/whoami.md @@ -1,13 +1,13 @@ `netlify-cli whoami` ==================== -Print currently logged in use +Print currently logged in user and account info * [`netlify-cli whoami`](#netlify-cli-whoami) ## `netlify-cli whoami` -Print currently logged in use +Print currently logged in user and account info ``` USAGE diff --git a/package.json b/package.json index cdb14158e32..5f58ada8714 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "bugs": "https://github.com/netlify/cli/issues", "dependencies": { "@iarna/toml": "^1.7.1", + "@netlify/open-api": "^0.1.0", "@oclif/command": "^1", "@oclif/config": "^1", "@oclif/errors": "^1.1.2", @@ -41,12 +42,12 @@ "folder-walker": "^3.1.0", "hasha": "^3.0.0", "is-docker": "^1.1.0", + "lodash.camelcase": "^4.3.0", "lodash.flatten": "^4.4.0", "lodash.get": "^4.4.2", "lodash.set": "^4.3.2", "lodash.snakecase": "^4.1.1", "make-dir": "^1.3.0", - "netlifys_api_definition": "^0.2.0", "open": "0.0.5", "opn": "^5.3.0", "p-all": "^1.0.0", @@ -57,6 +58,8 @@ "parallel-transform": "^1.1.0", "prettyjson": "^1.2.1", "pump": "^3.0.0", + "qs": "^6.5.2", + "r2": "^2.0.1", "request": "^2.87.0", "through2-filter": "^3.0.0", "through2-map": "^3.0.0", @@ -67,6 +70,7 @@ "devDependencies": { "@oclif/dev-cli": "^1.13.31", "ava": "1.0.0-beta.6", + "body": "^5.1.0", "eslint": "^4.19.1", "eslint-config-prettier": "^2.9.0", "eslint-plugin-import": "^2.13.0", diff --git a/src/base.js b/src/base.js index 00bed90d630..d6a146bb789 100644 --- a/src/base.js +++ b/src/base.js @@ -18,9 +18,9 @@ class BaseCommand extends Command { } this.log(`Logging into your Netlify account...`) const client = this.netlify - const ticket = await client.api.createTicket(this.global.get('clientId')) + const ticket = await client.createTicket({ clientId: this.global.get('clientId') }) openBrowser(`https://app.netlify.com/authorize?response_type=ticket&ticket=${ticket.id}`) - const accessToken = await client.waitForAccessToken(ticket) + const accessToken = await client.getAccessToken(ticket) this.global.set('accessToken', accessToken) this.log('Logged in...') } diff --git a/src/commands/deploy/index.js b/src/commands/deploy/index.js index 44e997efe26..bf94decc5d2 100644 --- a/src/commands/deploy/index.js +++ b/src/commands/deploy/index.js @@ -46,7 +46,7 @@ class DeployCommand extends Command { this.exit() } else { try { - await this.netlify.api.getSite(siteId) + await this.netlify.getSite({ siteId }) } catch (e) { this.error(e.message) } @@ -56,7 +56,7 @@ class DeployCommand extends Command { const deployFolder = args.publishFolder || get(this.site.toml, 'build.publish') || - get(await this.netlify.api.getSite(siteId), 'build_settings.dir') + get(await this.netlify.getSite({ siteId }), 'build_settings.dir') if (!deployFolder) { this.error( @@ -72,7 +72,7 @@ class DeployCommand extends Command { try { results = await this.netlify.deploy(siteId, resolvedDeployPath) } catch (e) { - this.error(JSON.stringify(e, null, ' ')) + this.error(e) } cliUx.action.stop(`Finished deploy ${results.deployId}`) this.log( diff --git a/src/commands/link/index.js b/src/commands/link/index.js index d8aa2c5b120..33ec4d59a40 100644 --- a/src/commands/link/index.js +++ b/src/commands/link/index.js @@ -10,7 +10,7 @@ class LinkCommand extends Command { const siteId = this.site.get('siteId') if (siteId && !flags.force) { - const site = await this.netlify.api.getSite(siteId) + const site = await this.netlify.getSite(siteId) this.log(`Site already linked to ${site.name}`) this.log(`Link: ${site.admin_url}`) return this.exit() @@ -19,7 +19,7 @@ class LinkCommand extends Command { if (flags.id) { let site try { - site = await this.netlify.api.getSite(flags.id) + site = await this.netlify.getSite(flags.id) } catch (e) { if (e.status === 404) throw new CLIError(`Site id ${flags.id} not found`) else throw new CLIError(e) @@ -32,7 +32,7 @@ class LinkCommand extends Command { if (flags.name) { let results try { - results = await this.netlify.api.listSites({ + results = await this.netlify.listSites({ name: flags.name, filter: 'all' }) diff --git a/src/commands/open/index.js b/src/commands/open/index.js index c2b424d6ec8..8af1bf1d072 100644 --- a/src/commands/open/index.js +++ b/src/commands/open/index.js @@ -18,7 +18,7 @@ class OpenCommand extends Command { let site try { - site = await this.netlify.api.getSite(siteId) + site = await this.netlify.getSite(siteId) } catch (e) { if (e.status === 401 /* unauthorized*/) { this.warn(`Log in with a different account or re-link to a site you have permission for`) diff --git a/src/commands/sites/list/index.js b/src/commands/sites/list/index.js index c3374428311..46f05245086 100644 --- a/src/commands/sites/list/index.js +++ b/src/commands/sites/list/index.js @@ -9,7 +9,7 @@ class SitesListCommand extends Command { const client = this.netlify // Fetch all sites! - client.api.listSites(null, (err, sites) => { + client.listSites(null, (err, sites) => { if (err) { throw new CLIError(err) } diff --git a/src/commands/status/index.js b/src/commands/status/index.js index 7bac0af874b..c8638dfe2a9 100644 --- a/src/commands/status/index.js +++ b/src/commands/status/index.js @@ -1,6 +1,5 @@ const Command = require('../../base') const renderShortDesc = require('../../utils/renderShortDescription') -const { CLIError } = require('@oclif/errors') const prettyjson = require('prettyjson') class StatusCommand extends Command { @@ -9,7 +8,7 @@ class StatusCommand extends Command { const siteId = this.site.get('siteId') let personal if (accessToken) { - const accounts = await this.netlify.api.listAccountsForUser() + const accounts = await this.netlify.listAccountsForUser() personal = accounts.find(account => account.type === 'PERSONAL') // TODO: use users endpoint } else { @@ -23,13 +22,13 @@ class StatusCommand extends Command { let site try { - site = await this.netlify.api.getSite(siteId) + site = await this.netlify.getSite({ siteId }) } catch (e) { if (e.status === 401 /* unauthorized*/) { this.warn(`Log in with a different account or re-link to a site you have permission for`) this.error(`Not authorized to view the currently linked site (${siteId})`) } - throw new CLIError(e) + this.error(e) } const statusData = { diff --git a/src/commands/whoami/index.js b/src/commands/whoami/index.js index d318e96124d..b635981af08 100644 --- a/src/commands/whoami/index.js +++ b/src/commands/whoami/index.js @@ -8,7 +8,7 @@ class WhoamiCommand extends Command { async run() { const accessToken = this.global.get('accessToken') if (accessToken) { - const accounts = await this.netlify.api.listAccountsForUser() + const accounts = await this.netlify.listAccountsForUser() const personal = accounts.find(account => account.type === 'PERSONAL') const teams = accounts.filter(account => account.type !== 'PERSONAL') const data = { diff --git a/src/utils/api/README.md b/src/utils/api/README.md new file mode 100644 index 00000000000..a7e24464b62 --- /dev/null +++ b/src/utils/api/README.md @@ -0,0 +1,110 @@ +# @netlify/js-api + +A Netlify [open-api](https://github.com/netlify/open-api) client that works in the browser and Node.js. + +## Usage + +```js +const NetlifyAPI = require('../utils/api') +const client = new NetlifyAPI('1234myAccessToken') +const sites = await client.listSites() +``` + +## API + +### `client = new NetlifyAPI([accessToken], [opts])` + +Create a new instance of the Netlify API client with the provided `accessToken`. + +`accessToken` is optional. Without it, you can't make authorized requests. + +`opts` includes: + +```js +{ + userAgent: 'netlify-js-client', + scheme: 'https', + host: 'api.netlify.com', + pathPrefix: '/api/v1' +} +``` + +### `client.accessToken` + +A setter/getter that returns the `accessToken` that the client is configured to use. You can set this after the class is instantiated, and all subsequent calls will use the new `accessToken`. + +### `client.basePath` + +A getter that returns the formatted base URL of the endpoint the client is configured to use. + +### Open API Client methods + +The client is dynamically generated from the [open-api](https://github.com/netlify/open-api) definition file. Each method is is named after the `operationId` name of each endpoint action. **To see list of available operation see the [open-api website](https://open-api.netlify.com/)**. + +Every open-api method has the following signature: + +#### `promise(response) = client.operationId([params], [opts])` + +Perform a call to the given endpoint corresponding with the `operationId`. Returns promise that will resolve with the body of the response, or reject with an error with details about the request attached. Rejects if the `status` > 400. Successful response objects have `status` and `statusText` properties on their prototype. + +`params` is an object that includes any of the required or optional endpoint parameters. `params.body` should be an object which gets serialized to JSON automatically. If the endpoint accepts `binary`, `params.body` can be a Node.js readable stream. + +```js +// example params +{ + any_param_needed, + paramsCanAlsoBeCamelCase, + body: { + an: 'arbitrary js object' + } +} +``` + +Optional `opts` can include any property you want passed to `node-fetch`. The `headers` propety is merged with some `defaultHeaders`. + +```js +// example opts +{ + headers + // any other properties for node-fetch +} +``` + +```js +// default headers +{ + 'User-agent': 'netlify-js-client', + accept: 'application/json' +} +``` + +### Convenience Methods + +Some methods have been added in addition to the open API methods that make certain actions simpler to perform. + +#### `promise(accessToken) = client.getAccessToken(ticket, [opts])` + +Pass in a [`ticket`](open-api.netlify.com#model-ticket) and get back an `accessToken`. Call this with the response from a `client.createTicket({ client_id })` call. Automatically sets the `accessToken` to `this.accessToken` and returns `accessToken` for the consumer to save for later. + +Optional `opts` include: + +```js +{ + poll: 1000, // number of ms to wait between polling + timeout: 3.6e6 // number of ms to wait before timing out +} +``` + +### `promise(deploy) = client.deploy(siteId, buildDir, [opts])` + +**Node.js Only**: Pass in a `siteId` and a path to a folder you wan't to deploy. This creates a new deploy for the `siteId`, scans the folder and begins an upload of changed files. + +Optional `opts` include: + +```js +{ + deployTimeout: 1.2e6, // 20 mins + parallelHash: 100, // number of parallel hashing calls + parallelUpload: 4 // number of files to upload in parallel +} +``` diff --git a/src/utils/api/access-token.js b/src/utils/api/access-token.js deleted file mode 100644 index 6db4f655368..00000000000 --- a/src/utils/api/access-token.js +++ /dev/null @@ -1,28 +0,0 @@ -const pWaitFor = require('p-wait-for') -const pTimeout = require('p-timeout') - -// exchange a ticket for an access token -module.exports = async function getAccessToken(api, ticket, opts) { - opts = Object.assign( - { - poll: 1000, - timeout: 3.6e6 - }, - opts - ) - - const { id } = ticket - let authorizedTicket - await pTimeout( - pWaitFor(async () => { - const t = await api.showTicket(id) - if (t.authorized) authorizedTicket = t - return !!t.authorized - }, opts.poll), - opts.timeout, - 'Timeout while waiting for ticket grant' - ) - - const accessToken = await api.exchangeTicket(authorizedTicket.id) - return accessToken.access_token -} diff --git a/src/utils/api/browser/generate-method.js b/src/utils/api/browser/generate-method.js new file mode 100644 index 00000000000..85ada623a69 --- /dev/null +++ b/src/utils/api/browser/generate-method.js @@ -0,0 +1,92 @@ +const get = require('lodash.get') +const set = require('lodash.set') +const queryString = require('qs') +const r2 = require('r2') +const camelCase = require('lodash.camelcase') + +function existy(val) { + return val != null +} + +// open-api 2.0 +module.exports = method => { + return async function(params, opts) { + opts = Object.assign({}, opts) + params = Object.assign({}, this.globalParams, params) + + let path = this.basePath + method.path + // Path parameters + Object.values(method.parameters.path).forEach(param => { + const val = params[param.name] || params[camelCase(param.name)] + if (existy(val)) { + path = path.replace(`{${param.name}}`, val) + } else if (param.required) { + throw new Error(`Missing required param ${param.name}`) + } + }) + // qs parameters + let qs + Object.values(method.parameters.query).forEach(param => { + const val = params[param.name] || params[camelCase(param.name)] + if (existy(val)) { + if (!qs) qs = {} + qs[param.name] = val + } else if (param.required) { + throw new Error(`Missing required param ${param.name}`) + } + }) + if (qs) path = path += `?${queryString.stringify(qs)}` + // body parameters + let body + let bodyType = 'json' + if (params.body) { + body = params.body + Object.values(method.parameters.body).forEach(param => { + const type = get(param, 'schema.format') + if (type === 'binary') { + bodyType = 'binary' + } + }) + } + + const discoveredHeaders = {} + if (body) { + switch (bodyType) { + case 'binary': { + opts.body = body + set(discoveredHeaders, 'Content-Type', 'application/octet-stream') + break + } + case 'json': + default: { + opts.json = body + break + } + } + } + + opts.headers = Object.assign({}, this.defaultHeaders, discoveredHeaders, opts.headers) + + const req = await r2[method.verb](path, opts) + const response = await req.response + + if (response.status >= 400) { + const err = new Error(response.statusText) + err.status = response.status + err.response = response + err.path = path + err.opts = opts + throw err + } + // Put the status on the prototype to prevent it from serializing + const status = { + status: response.status, + statusText: response.statusText + } + const json = await req.json + // inject prototype props + Object.setPrototypeOf(status, Object.getPrototypeOf(json)) + Object.setPrototypeOf(json, status) + return json + } +} diff --git a/src/utils/api/browser/index.js b/src/utils/api/browser/index.js new file mode 100644 index 00000000000..bcf91ca1e14 --- /dev/null +++ b/src/utils/api/browser/index.js @@ -0,0 +1,85 @@ +const set = require('lodash.set') +const get = require('lodash.get') +const { methods } = require('./shape-swagger') +const dfn = require('@netlify/open-api') +const generateMethod = require('./generate-method') +const pWaitFor = require('p-wait-for') +const pTimeout = require('p-timeout') + +// Browser compatible Open API Client +class NetlifyAPI { + constructor(accessToken, opts) { + if (typeof accessToken === 'object') { + opts = accessToken + accessToken = null + } + opts = Object.assign( + { + userAgent: 'netlify-js-client', + scheme: dfn.schemes[0], + host: dfn.host, + pathPrefix: dfn.basePath + }, + opts + ) + this.defaultHeaders = { + 'User-agent': opts.userAgent, + accept: 'application/json' + } + this.scheme = opts.scheme + this.host = opts.host + this.pathPrefix = opts.pathPrefix + this.globalParams = Object.assign({}, opts.globalParams) + if (accessToken) this.accessToken = accessToken + } + + get accessToken() { + return (get(this, 'defaultHeaders.Authorization') || '').replace('Bearer ', '') + } + + set accessToken(token) { + if (token) { + set(this, 'defaultHeaders.Authorization', 'Bearer ' + token) + } else { + delete this.defaultHeaders.Authorization + } + } + + get basePath() { + return `${this.scheme}://${this.host}${this.pathPrefix}` + } + + // Attach generic browser compatible methods here + async getAccessToken(ticket, opts) { + opts = Object.assign( + { + poll: 1000, + timeout: 3.6e6 + }, + opts + ) + + const { id } = ticket + let authorizedTicket + const api = this + + const checkTicket = async () => { + const t = await api.showTicket({ ticketId: id }) + if (t.authorized) authorizedTicket = t + return !!t.authorized + } + + await pTimeout(pWaitFor(checkTicket, opts.poll), opts.timeout, 'Timeout while waiting for ticket grant') + + const accessToken = await api.exchangeTicket({ ticketId: authorizedTicket.id }) + this.accessToken = accessToken.access_token + return accessToken.access_token + } +} + +methods.forEach(method => { + /* {param1, param2, body, ... }, [opts] */ + NetlifyAPI.prototype[method.operationId] = generateMethod(method) +}) + +module.exports = NetlifyAPI diff --git a/src/utils/api/browser/index.test.js b/src/utils/api/browser/index.test.js new file mode 100644 index 00000000000..a320a4118c9 --- /dev/null +++ b/src/utils/api/browser/index.test.js @@ -0,0 +1,79 @@ +const test = require('ava') +const http = require('http') +const promisify = require('util.promisify') +const NetlifyAPI = require('./index') +const body = promisify(require('body')) + +const createServer = handler => { + const server = http.createServer(handler) + server._close = server.close + server.close = promisify(cb => server._close(cb)) + server._listen = server.listen + server.listen = promisify((port, cb) => server._listen(port, cb)) + return server +} + +const port = 1123 + +const client = new NetlifyAPI('1234', { + scheme: 'http', + host: `localhost:${port}`, + pathPrefix: '/v1', + globalParams: { clientId: '1234' } +}) + +test.serial('can make basic requests', async t => { + const server = createServer((req, res) => { + t.is(req.url, '/v1/oauth/tickets?client_id=1234') + res.end('{"foo": "bar"}') + }) + + await server.listen(port) + + const body = await client.createTicket() + t.is(body.status, 200) + t.deepEqual(body, { foo: 'bar' }) + + await server.close() +}) + +test.serial('can make requests with a body', async t => { + const server = createServer(async (req, res) => { + t.is(req.url, '/v1/hooks?site_id=Site123') + t.is(await body(req), '{"some":"bodyParams","another":"one"}') + res.end('{"foo": "bar"}') + }) + + await server.listen(port) + + const response = await client.createHookBySiteId({ + site_id: 'Site123', + body: { + some: 'bodyParams', + another: 'one' + } + }) + t.is(response.status, 200) + t.deepEqual(response, { foo: 'bar' }) + + await server.close() +}) + +test.serial('basic api exists', async t => { + t.is(client.basePath, `http://localhost:1123/v1`, 'basePath getter works') + t.is(client.accessToken, '1234', 'accessToken is set') + t.deepEqual( + client.defaultHeaders, + { + Authorization: 'Bearer 1234', + 'User-agent': 'netlify-js-client', + accept: 'application/json' + }, + 'Default headers are set' + ) + client.accessToken = undefined + t.falsy(client.accessToken, 'deleting access token works fine') + client.accessToken = 5678 + t.is(client.accessToken, '5678', 'accessToken is set') + t.is(client.defaultHeaders.Authorization, 'Bearer 5678', 'Bearer token is updated correctly') +}) diff --git a/src/utils/api/browser/shape-swagger.js b/src/utils/api/browser/shape-swagger.js new file mode 100644 index 00000000000..1249edd9542 --- /dev/null +++ b/src/utils/api/browser/shape-swagger.js @@ -0,0 +1,28 @@ +const dfn = require('@netlify/open-api') +const { sortParams, mergeParams } = require('./util') +const methods = [] + +Object.entries(dfn.paths).forEach(([apiPath, verbs]) => { + const topParams = sortParams(verbs.parameters) + delete verbs.parameters + + Object.entries(verbs).forEach(([verb, props]) => { + const verbParams = sortParams(props.parameters) + delete props.parameters + + const opSpec = Object.assign( + {}, + props, + { + verb, + path: apiPath + }, + { + parameters: mergeParams(topParams, verbParams) + } + ) + methods.push(opSpec) + }) +}) + +exports.methods = methods diff --git a/src/utils/api/browser/util.js b/src/utils/api/browser/util.js new file mode 100644 index 00000000000..08a9232f8b7 --- /dev/null +++ b/src/utils/api/browser/util.js @@ -0,0 +1,31 @@ +const set = require('lodash.set') + +exports.sortParams = (parameters = []) => { + const paramSet = { + // minimum param set + path: {}, + query: {}, + body: {} + } + + parameters.forEach(param => { + set(paramSet, `${param.in}.${param.name}`, param) + }) + + return paramSet +} + +exports.mergeParams = (...params) => { + const merged = {} + + params.forEach(paramSet => { + Object.entries(paramSet).forEach(([type, params]) => { + if (!merged[type]) merged[type] = {} + Object.values(params).forEach((param, index) => { + set(merged, `${param.in}.${param.name}`, Object.assign(param, { index })) + }) + }) + }) + + return merged +} diff --git a/src/utils/api/deploy/file-hasher.js b/src/utils/api/deploy/file-hasher.js index 57f42b5594d..502268f18b2 100644 --- a/src/utils/api/deploy/file-hasher.js +++ b/src/utils/api/deploy/file-hasher.js @@ -11,7 +11,7 @@ module.exports = fileHasher async function fileHasher(dir, opts) { opts = Object.assign( { - parallel: 100 + parallelHash: 100 }, opts ) @@ -26,7 +26,7 @@ async function fileHasher(dir, opts) { fileObj => fileObj.type === 'file' && (fileObj.relname.match(/(\/__MACOSX|\/\.)/) ? false : true) ) - const hasher = transform(opts.parallel, { objectMode: true }, (fileObj, cb) => { + const hasher = transform(opts.parallelHash, { objectMode: true }, (fileObj, cb) => { hasha .fromFile(fileObj.filepath, { algorithm: 'sha1' }) .then(sha1 => cb(null, Object.assign({}, fileObj, { sha1 }))) diff --git a/src/utils/api/deploy/index.js b/src/utils/api/deploy/index.js index 918c731ec1a..c717ced86b7 100644 --- a/src/utils/api/deploy/index.js +++ b/src/utils/api/deploy/index.js @@ -4,25 +4,26 @@ const fileHasher = require('./file-hasher') const pWaitFor = require('p-wait-for') const pTimeout = require('p-timeout') const flatten = require('lodash.flatten') -const request = require('request') -const get = require('lodash.get') module.exports = async (api, siteId, dir, opts) => { opts = Object.assign( { - deployTimeout: 1.2e6 // 20 mins + deployTimeout: 1.2e6, // 20 mins + parallelHash: 100, // Queue up 100 file hashes at a time + parallelUpload: 4 // Number of concurrent uploads }, opts ) + // TODO Implement progress function const { files, shaMap } = await fileHasher(dir, opts) console.log(`Hashed ${Object.keys(files).length} files`) - let deploy = await api.createSiteDeploy(siteId, { files }, {}) + let deploy = await api.createSiteDeploy({ siteId, body: { files } }) const { id: deployId, required } = deploy const uploadList = getUploadList(required, shaMap) console.log(`Deploy requested ${uploadList.length} files`) - await uploadFiles(api, deployId, uploadList) + await uploadFiles(api, deployId, uploadList, opts) console.log(`Done uploading files. Waiting on deploy...`) // Update deploy object deploy = await waitForDeploy(api, deployId, opts.deployTimeout) @@ -40,41 +41,21 @@ function getUploadList(required, shaMap) { return flatten(required.map(sha => shaMap[sha])) } -async function uploadFiles(api, deployId, uploadList) { - const uploadFile = fileObj => { +async function uploadFiles(api, deployId, uploadList, opts) { + const uploadFile = async fileObj => { const { normalizedPath } = fileObj const readStream = fs.createReadStream(fileObj.filepath) - const reqOpts = { - url: `https://api.netlify.com/api/v1/deploys/${deployId}/files/${normalizedPath}`, - headers: { - 'User-agent': 'Netlify CLI (oclif)', - Authorization: 'Bearer ' + get(api, 'apiClient.authentications.netlifyAuth.accessToken'), - 'Content-Type': 'application/octet-stream' - }, - body: readStream - } - console.log(`uploading ${normalizedPath}`) - return new Promise((resolve, reject) => { - request.put(reqOpts, (err, httpResponse, body) => { - if (err) return reject(err) - if (httpResponse.statusCode >= 400) { - const apiError = new Error('There was an error with one of the file uploads') - apiError.response = httpResponse - return reject(apiError) - } - try { - body = JSON.parse(body) - } catch (_) { - // Ignore if body can't parse - } - console.log(`done uploading ${normalizedPath}`) - resolve({ httpResponse, body }) - }) + const response = await api.uploadDeployFile({ + body: readStream, + deployId, + path: normalizedPath }) + console.log(`done uploading ${normalizedPath}`) + return response } - const results = await pMap(uploadList, uploadFile, { concurrency: 4 }) + const results = await pMap(uploadList, uploadFile, { concurrency: opts.parallelUpload }) return results } @@ -91,7 +72,7 @@ async function waitForDeploy(api, deployId, timeout) { return deploy async function loadDeploy() { - const d = await api.getDeploy(deployId) + const d = await api.getDeploy({ deployId }) if (d.state === 'ready') { deploy = d return true diff --git a/src/utils/api/index.js b/src/utils/api/index.js index 3444f034ff5..71fa179e9a6 100644 --- a/src/utils/api/index.js +++ b/src/utils/api/index.js @@ -1,40 +1,12 @@ -const NetlifysApiDefinition = require('netlifys_api_definition') -const promisifyAll = require('util.promisify-all') const deploy = require('./deploy') -const getAccessToken = require('./access-token') -const get = require('lodash.get') -const set = require('lodash.set') - -class Netlify { - constructor(accessToken) { - if (accessToken) { - const defaultClient = NetlifysApiDefinition.ApiClient.instance - const netlifyAuth = defaultClient.authentications['netlifyAuth'] - netlifyAuth.accessToken = accessToken - } - this.api = promisifyAll(new NetlifysApiDefinition.DefaultApi()) - } - - get accessToken() { - const api = this.api - return get(api, 'apiClient.authentications.netlifyAuth.accessToken') - } - - set accessToken(token) { - const api = this.api - set(api, 'apiClient.authentications.netlifyAuth.accessToken', token) - } +const NetlifyAPI = require('./browser') +class NetlifyNodeAPI extends NetlifyAPI { + // Attach node specific methods to this class async deploy(siteId, buildDir, opts) { if (!this.accessToken) throw new Error('Missing access token') - return await deploy(this.api, siteId, buildDir, opts) - } - - async waitForAccessToken(ticket) { - const accessToken = await getAccessToken(this.api, ticket) - this.accessToken = accessToken - return accessToken + return await deploy(this, siteId, buildDir, opts) } } -module.exports = Netlify +module.exports = NetlifyNodeAPI From 9f08debde8f40431680e45bb49e1822f3d22828a Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Thu, 26 Jul 2018 15:51:02 -0700 Subject: [PATCH 2/5] Readme tweak --- src/utils/api/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/api/README.md b/src/utils/api/README.md index a7e24464b62..0bae13f5b03 100644 --- a/src/utils/api/README.md +++ b/src/utils/api/README.md @@ -25,7 +25,8 @@ Create a new instance of the Netlify API client with the provided `accessToken`. userAgent: 'netlify-js-client', scheme: 'https', host: 'api.netlify.com', - pathPrefix: '/api/v1' + pathPrefix: '/api/v1', + globalParams: {} // parameters you want available for every request } ``` From ebf82b9d39fdcb42886eafbed77376fe0444fc62 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Thu, 26 Jul 2018 15:57:14 -0700 Subject: [PATCH 3/5] Fix heading --- src/utils/api/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/api/README.md b/src/utils/api/README.md index 0bae13f5b03..119d0e4cff9 100644 --- a/src/utils/api/README.md +++ b/src/utils/api/README.md @@ -96,7 +96,7 @@ Optional `opts` include: } ``` -### `promise(deploy) = client.deploy(siteId, buildDir, [opts])` +#### `promise(deploy) = client.deploy(siteId, buildDir, [opts])` **Node.js Only**: Pass in a `siteId` and a path to a folder you wan't to deploy. This creates a new deploy for the `siteId`, scans the folder and begins an upload of changed files. From 2e75dce10082935769e7c71dbb5d999044b01ef8 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Thu, 26 Jul 2018 18:18:14 -0700 Subject: [PATCH 4/5] Tests --- .gitignore | 3 +- package.json | 11 +- src/utils/api/browser/generate-method.js | 16 ++- src/utils/api/browser/index.test.js | 123 +++++++++++++++++++++++ 4 files changed, 143 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 1ea3fd6b7b1..1dd03019fe2 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ vendor .DS_STORE .vscode site -.netlify \ No newline at end of file +.netlify +coverage \ No newline at end of file diff --git a/package.json b/package.json index 5f58ada8714..6802a36cdd0 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "failFast": false, "failWithoutAssertions": false, "tap": false, - "compileEnhancements": true + "babel": false, + "compileEnhancements": false }, "bin": { "ntl": "./bin/run", @@ -25,7 +26,7 @@ "bugs": "https://github.com/netlify/cli/issues", "dependencies": { "@iarna/toml": "^1.7.1", - "@netlify/open-api": "^0.1.0", + "@netlify/open-api": "^0.1.1", "@oclif/command": "^1", "@oclif/config": "^1", "@oclif/errors": "^1.1.2", @@ -76,8 +77,10 @@ "eslint-plugin-import": "^2.13.0", "eslint-plugin-node": "^6.0.1", "eslint-plugin-prettier": "^2.6.1", + "from2-string": "^1.1.0", "mkdirp": "^0.5.1", "npm-run-all": "^4.1.3", + "nyc": "^12.0.2", "prettier": "^1.13.7", "rimraf": "^2.6.2", "sitedown": "^3.3.0" @@ -119,9 +122,9 @@ "clean": "rimraf site && mkdirp site", "start": "node ./bin/run", "test": "run-s test:*", - "test:ava": "ava -vvvv", + "test:ava": "nyc --reporter=lcov ava --verbose && nyc report", "test:lint": "eslint src", "watch": "run-p watch:*", - "watch:ava": "npm run test:ava -- --watch" + "watch:ava": "nyc --reporter=lcov ava --watch" } } diff --git a/src/utils/api/browser/generate-method.js b/src/utils/api/browser/generate-method.js index 85ada623a69..aaa3a6e964f 100644 --- a/src/utils/api/browser/generate-method.js +++ b/src/utils/api/browser/generate-method.js @@ -73,6 +73,7 @@ module.exports = method => { if (response.status >= 400) { const err = new Error(response.statusText) err.status = response.status + err.statusText = response.statusText err.response = response err.path = path err.opts = opts @@ -83,10 +84,15 @@ module.exports = method => { status: response.status, statusText: response.statusText } - const json = await req.json - // inject prototype props - Object.setPrototypeOf(status, Object.getPrototypeOf(json)) - Object.setPrototypeOf(json, status) - return json + + try { + const json = await req.json + // inject prototype props + Object.setPrototypeOf(status, Object.getPrototypeOf(json)) + Object.setPrototypeOf(json, status) + return json + } catch (e) { + return await req.text + } } } diff --git a/src/utils/api/browser/index.test.js b/src/utils/api/browser/index.test.js index a320a4118c9..8c201d22b68 100644 --- a/src/utils/api/browser/index.test.js +++ b/src/utils/api/browser/index.test.js @@ -3,6 +3,7 @@ const http = require('http') const promisify = require('util.promisify') const NetlifyAPI = require('./index') const body = promisify(require('body')) +const fromString = require('from2-string') const createServer = handler => { const server = http.createServer(handler) @@ -59,6 +60,50 @@ test.serial('can make requests with a body', async t => { await server.close() }) +test.serial('path parameter assignment', async t => { + const server = createServer(async (req, res) => { + t.is(req.url, '/v1/hooks?site_id=Site123') + res.end() + }) + + await server.listen(port) + + const error = await t.throws(client.createHookBySiteId(/* missing args */)) + t.is(error.message, 'Missing required param site_id') + const response = await client.createHookBySiteId({ siteId: 'Site123' }) + t.is(response, '', 'Testing other path branch') + + await server.close() +}) + +test.serial('handles errors from API', async t => { + const server = createServer(async (req, res) => { + res.statusCode = 404 + res.statusMessage = 'Test not found' + res.end() + }) + + await server.listen(port) + + const error = await t.throws(client.createHookBySiteId({ siteId: 'Site123' })) + t.is(error.status, 404, 'status code is captures on error') + t.is(error.statusText, 'Test not found', 'status text is captures on error') + t.truthy(error.response, 'Error has response object') + t.truthy(error.path, 'Error has response object') + t.deepEqual( + error.opts, + { + headers: { + Authorization: 'Bearer 1234', + 'User-agent': 'netlify-js-client', + accept: 'application/json' + } + }, + 'Opts look correct' + ) + await server.close() +}) + test.serial('basic api exists', async t => { t.is(client.basePath, `http://localhost:1123/v1`, 'basePath getter works') t.is(client.accessToken, '1234', 'accessToken is set') @@ -77,3 +122,81 @@ test.serial('basic api exists', async t => { t.is(client.accessToken, '5678', 'accessToken is set') t.is(client.defaultHeaders.Authorization, 'Bearer 5678', 'Bearer token is updated correctly') }) + +test.serial('binary uploads', async t => { + const server = createServer(async (req, res) => { + t.is(await body(req), 'hello world') + res.statusCode = 200 + res.statusMessage = 'OK' + res.end('{"ok": true}') + }) + + await server.listen(port) + + const readStream = fromString('hello world') + const response = await client.uploadDeployFile({ + body: readStream, + deployId: '123', + path: 'normalizedPath' + }) + + t.deepEqual(response, { ok: true }) + t.is(response.status, 200) + + await server.close() +}) + +test('variadic api', async t => { + const newClient = new NetlifyAPI({ + scheme: 'http', + host: `localhost:${port}`, + pathPrefix: '/v1', + globalParams: { clientId: '1234' } + }) + + t.falsy(newClient.accessToken, 'can instantiate with just options') + t.falsy(newClient.defaultHeaders.Authorization, 'headers are falsy when not set') + + newClient.accessToken = '123' + + t.is(newClient.accessToken, '123', 'can set the access token and get it back') + t.is(newClient.defaultHeaders.Authorization, 'Bearer 123', 'headers are set') +}) + +test.serial('access token can poll', async t => { + let okayToResponse = false + setTimeout(() => { + okayToResponse = true + }, 1000) + const server = createServer(async (req, res) => { + if (req.url == '/v1/oauth/tickets/ticket-id') { + if (!okayToResponse) { + res.end('{}') + } else { + res.end( + JSON.stringify({ + authorized: true, + id: 'ticket-id' + }) + ) + } + } else if (req.url == '/v1/oauth/tickets/ticket-id/exchange') { + res.end( + JSON.stringify({ + access_token: 'open-sesame' + }) + ) + } else { + res.statusCode = 500 + res.end(JSON.stringify({ path: req.url })) + } + }) + + await server.listen(port) + + const accessToken = await client.getAccessToken({ id: 'ticket-id' }, { poll: 200, timeout: 5000 }) + + t.is(accessToken, 'open-sesame') + + await server.close() +}) From e6b2789fc79e9c4dd3308ee77bfdf434d8c43410 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Thu, 26 Jul 2018 18:21:52 -0700 Subject: [PATCH 5/5] Add more test coverage --- src/utils/global-config/util.test.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utils/global-config/util.test.js b/src/utils/global-config/util.test.js index e40e1964c50..126df04d6d9 100644 --- a/src/utils/global-config/util.test.js +++ b/src/utils/global-config/util.test.js @@ -1,8 +1,14 @@ const test = require('ava') -const { toEnvCase } = require('./util') +const { toEnvCase, isDotProp } = require('./util') test('camelCase to NETLIFY_ENV_CASE', t => { const envCase = toEnvCase('fooBar') t.is(envCase, 'NETLIFY_FOO_BAR', 'env case conversion works') t.pass() }) + +test('isDotProp detects doPropPaths', t => { + t.true(isDotProp('foo.bar.baz')) + t.false(isDotProp('foo_baz')) + t.true(isDotProp('foo_baz[biz]')) +})