From d399dc99f31b5918b6b707e07c7ccb4391d43c3f Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 31 May 2019 12:06:37 +0200 Subject: [PATCH] fix(gatsby-dev-cli): refactor, wait for package installation before copying files, handle packages that are not direct deps of local project (#14347) * tmp * more tmp * more tmp * finalize tests * some cleanup * cleanup js files installed from npm to get rid of any stale files, they will be copied over anyway * don't delete nested node_modules js files * assert that files are copied over --- packages/gatsby-dev-cli/package.json | 1 + .../gatsby-dev-cli/src/__tests__/watch.js | 564 +++++++++++++++++- packages/gatsby-dev-cli/src/index.js | 22 +- .../src/local-npm-registry/index.js | 68 +++ .../local-npm-registry/install-packages.js | 30 + .../{verdaccio.js => publish-package.js} | 116 +--- .../local-npm-registry/verdaccio-config.js | 30 + .../src/utils/traverse-package-deps.js | 2 +- packages/gatsby-dev-cli/src/watch.js | 69 ++- 9 files changed, 753 insertions(+), 149 deletions(-) create mode 100644 packages/gatsby-dev-cli/src/local-npm-registry/index.js create mode 100644 packages/gatsby-dev-cli/src/local-npm-registry/install-packages.js rename packages/gatsby-dev-cli/src/local-npm-registry/{verdaccio.js => publish-package.js} (61%) create mode 100644 packages/gatsby-dev-cli/src/local-npm-registry/verdaccio-config.js diff --git a/packages/gatsby-dev-cli/package.json b/packages/gatsby-dev-cli/package.json index ed4747402bc37..a7a00a1b2351a 100644 --- a/packages/gatsby-dev-cli/package.json +++ b/packages/gatsby-dev-cli/package.json @@ -13,6 +13,7 @@ "@babel/runtime": "^7.0.0", "chokidar": "^1.7.0", "configstore": "^3.1.0", + "del": "^3.0.0", "execa": "^1.0.0", "fs-extra": "^4.0.1", "is-absolute": "^0.2.6", diff --git a/packages/gatsby-dev-cli/src/__tests__/watch.js b/packages/gatsby-dev-cli/src/__tests__/watch.js index c3790880a0075..f1af97de07e88 100644 --- a/packages/gatsby-dev-cli/src/__tests__/watch.js +++ b/packages/gatsby-dev-cli/src/__tests__/watch.js @@ -7,8 +7,30 @@ jest.mock(`fs-extra`, () => { return { copy: jest.fn(), existsSync: jest.fn(), + removeSync: jest.fn(), } }) +jest.mock(`del`, () => jest.fn()) + +jest.mock(`verdaccio`, () => { + return { + default: (_, _2, _3, _4, _5, callback) => { + callback( + { + listen: (_, _2, cb2) => { + cb2() + }, + }, + { + port: `aa`, + path: `bb`, + host: `cc,`, + } + ) + }, + } +}) + const chokidar = require(`chokidar`) const fs = require(`fs-extra`) const path = require(`path`) @@ -27,14 +49,20 @@ beforeEach(() => { }) }) -describe(`watching`, () => { - const callEventCallback = (...args) => - on.mock.calls[0].slice(-1).pop()(...args) - const callReadyCallback = (...args) => - on.mock.calls[1].slice(-1).pop()(...args) +const args = [ + process.cwd(), + [`gatsby`], + { + localPackages: [`gatsby`], + }, +] - const args = [process.cwd(), [`gatsby`], {}] +const callEventCallback = (...args) => + on && on.mock.calls[0].slice(-1).pop()(...args) +const callReadyCallback = (...args) => + on && on.mock.calls[1].slice(-1).pop()(...args) +describe(`watching`, () => { it(`watches files`, () => { watch(...args) expect(chokidar.watch).toHaveBeenCalledTimes(1) @@ -110,7 +138,9 @@ describe(`watching`, () => { }) it(`filters duplicate directories`, () => { - watch(process.cwd(), [`gatsby`, `gatsby`], {}) + watch(process.cwd(), [`gatsby`, `gatsby`], { + localPackages: [`gatsby`], + }) expect(chokidar.watch).toHaveBeenCalledWith( [expect.stringContaining(`gatsby`)], @@ -142,7 +172,10 @@ describe(`watching`, () => { }) it(`exits if scanOnce is defined`, async () => { - watch(process.cwd(), [`gatsby`], { scanOnce: true }) + watch(process.cwd(), [`gatsby`], { + scanOnce: true, + localPackages: [`gatsby`], + }) await callReadyCallback() @@ -150,3 +183,518 @@ describe(`watching`, () => { }) }) }) + +const monoRepoPackages = [ + `babel-plugin-remove-graphql-queries`, + `babel-preset-gatsby`, + `babel-preset-gatsby-package`, + `cypress-gatsby`, + `gatsby`, + `gatsby-cli`, + `gatsby-codemods`, + `gatsby-cypress`, + `gatsby-dev-cli`, + `gatsby-image`, + `gatsby-link`, + `gatsby-plugin-canonical-urls`, + `gatsby-plugin-catch-links`, + `gatsby-plugin-coffeescript`, + `gatsby-plugin-create-client-paths`, + `gatsby-plugin-cxs`, + `gatsby-plugin-emotion`, + `gatsby-plugin-facebook-analytics`, + `gatsby-plugin-feed`, + `gatsby-plugin-flow`, + `gatsby-plugin-fullstory`, + `gatsby-plugin-glamor`, + `gatsby-plugin-google-analytics`, + `gatsby-plugin-google-gtag`, + `gatsby-plugin-google-tagmanager`, + `gatsby-plugin-guess-js`, + `gatsby-plugin-jss`, + `gatsby-plugin-layout`, + `gatsby-plugin-less`, + `gatsby-plugin-lodash`, + `gatsby-plugin-manifest`, + `gatsby-plugin-netlify`, + `gatsby-plugin-netlify-cms`, + `gatsby-plugin-no-sourcemaps`, + `gatsby-plugin-nprogress`, + `gatsby-plugin-offline`, + `gatsby-plugin-page-creator`, + `gatsby-plugin-postcss`, + `gatsby-plugin-preact`, + `gatsby-plugin-react-css-modules`, + `gatsby-plugin-react-helmet`, + `gatsby-plugin-remove-trailing-slashes`, + `gatsby-plugin-sass`, + `gatsby-plugin-sharp`, + `gatsby-plugin-sitemap`, + `gatsby-plugin-styled-components`, + `gatsby-plugin-styled-jsx`, + `gatsby-plugin-styletron`, + `gatsby-plugin-stylus`, + `gatsby-plugin-subfont`, + `gatsby-plugin-twitter`, + `gatsby-plugin-typescript`, + `gatsby-plugin-typography`, + `gatsby-react-router-scroll`, + `gatsby-remark-autolink-headers`, + `gatsby-remark-code-repls`, + `gatsby-remark-copy-linked-files`, + `gatsby-remark-custom-blocks`, + `gatsby-remark-embed-snippet`, + `gatsby-remark-graphviz`, + `gatsby-remark-images`, + `gatsby-remark-images-contentful`, + `gatsby-remark-katex`, + `gatsby-remark-prismjs`, + `gatsby-remark-responsive-iframe`, + `gatsby-remark-smartypants`, + `gatsby-source-contentful`, + `gatsby-source-drupal`, + `gatsby-source-faker`, + `gatsby-source-filesystem`, + `gatsby-source-graphql`, + `gatsby-source-hacker-news`, + `gatsby-source-lever`, + `gatsby-source-medium`, + `gatsby-source-mongodb`, + `gatsby-source-npm-package-search`, + `gatsby-source-shopify`, + `gatsby-source-wikipedia`, + `gatsby-source-wordpress`, + `gatsby-telemetry`, + `gatsby-transformer-asciidoc`, + `gatsby-transformer-csv`, + `gatsby-transformer-documentationjs`, + `gatsby-transformer-excel`, + `gatsby-transformer-hjson`, + `gatsby-transformer-javascript-frontmatter`, + `gatsby-transformer-javascript-static-exports`, + `gatsby-transformer-json`, + `gatsby-transformer-pdf`, + `gatsby-transformer-react-docgen`, + `gatsby-transformer-remark`, + `gatsby-transformer-screenshot`, + `gatsby-transformer-sharp`, + `gatsby-transformer-sqip`, + `gatsby-transformer-toml`, + `gatsby-transformer-xml`, + `gatsby-transformer-yaml`, + `graphql-skip-limit`, +] + +const mockDepsChanges = packagesWithChangedDeps => ({ packageName }) => + Promise.resolve({ + didDepsChanged: packagesWithChangedDeps.includes(packageName), + }) + +jest.mock(`../utils/check-deps-changes`, () => { + return { + checkDepsChanges: jest.fn(), + } +}) + +jest.mock(`../local-npm-registry/publish-package`, () => { + return { + publishPackage: jest.fn(), + } +}) + +jest.mock(`../local-npm-registry/install-packages`, () => { + return { + installPackages: jest.fn(), + } +}) + +jest.mock(`../utils/promisified-spawn`, () => { + return { + promisifiedSpawn: jest.fn(() => Promise.resolve()), + } +}) + +describe(`dependency changs`, () => { + const { publishPackage } = require(`../local-npm-registry/publish-package`) + const { installPackages } = require(`../local-npm-registry/install-packages`) + const { checkDepsChanges } = require(`../utils/check-deps-changes`) + const { promisifiedSpawn } = require(`../utils/promisified-spawn`) + + const assertPublish = ({ include = [], exclude = [] }) => { + include.forEach(includedPackage => { + expect(publishPackage).toBeCalledWith( + expect.objectContaining({ + packageName: includedPackage, + }) + ) + }) + + exclude.forEach(excludedPackage => { + expect(publishPackage).not.toBeCalledWith( + expect.objectContaining({ + packageName: excludedPackage, + }) + ) + }) + } + + const assertInstall = ({ include = [], exclude = [] }) => { + include.forEach(includedPackage => { + expect(installPackages).toBeCalledWith( + expect.objectContaining({ + packagesToInstall: expect.arrayContaining([includedPackage]), + }) + ) + }) + + exclude.forEach(excludedPackage => { + expect(installPackages).not.toBeCalledWith( + expect.objectContaining({ + packagesToInstall: expect.arrayContaining([excludedPackage]), + }) + ) + }) + } + + const assertCopy = packages => { + packages.forEach(pkgName => { + expect(fs.copy).toBeCalledWith( + expect.stringContaining(path.join(`packages`, pkgName)), + expect.stringContaining(path.join(`node_modules`, pkgName)), + expect.anything() + ) + }) + } + + let realProcess + beforeAll(() => { + realProcess = global.process + + global.process = { + ...realProcess, + exit: jest.fn(), + } + }) + + beforeEach(() => { + publishPackage.mockClear() + installPackages.mockClear() + checkDepsChanges.mockClear() + promisifiedSpawn.mockClear() + }) + + afterAll(() => { + global.process = realProcess + }) + + describe(`publishing and installing packages`, () => { + it(`watching gatsby installs gatsby`, async () => { + checkDepsChanges.mockImplementationOnce(mockDepsChanges([`gatsby`])) + + watch(process.cwd(), [`gatsby`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [`gatsby`, `gatsby-plugin-sharp`], + }) + + const filePath = path.join(process.cwd(), `packages/gatsby/package.json`) + await callEventCallback(`add`, filePath) + await callReadyCallback() + + assertPublish({ + include: [`gatsby`], + exclude: [`gatsby-plugin-sharp`], + }) + + assertInstall({ + include: [`gatsby`], + exclude: [`gatsby-cli`, `gatsby-plugin-sharp`], + }) + }) + + it(`watching gatsby and gatsby-plugin-sharp installs gatsby and copies gatsby-plugin-sharp`, async () => { + checkDepsChanges.mockImplementationOnce(mockDepsChanges([`gatsby`])) + + watch(process.cwd(), [`gatsby`, `gatsby-plugin-sharp`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [`gatsby`, `gatsby-plugin-sharp`], + }) + + const filePath = path.join(process.cwd(), `packages/gatsby/package.json`) + await callEventCallback(`add`, filePath) + // no deps changes in gatsby-plugin-sharp, just copy files over + await callEventCallback( + `add`, + path.join(process.cwd(), `packages/gatsby-plugin-sharp/index.js`) + ) + await callReadyCallback() + + assertPublish({ + include: [`gatsby`], + exclude: [`gatsby-plugin-sharp`], + }) + + assertInstall({ + include: [`gatsby`], + exclude: [`gatsby-cli`, `gatsby-plugin-sharp`], + }) + + assertCopy([`gatsby`, `gatsby-plugin-sharp`]) + }) + + it(`watching gatsby-cli installs gatsby`, async () => { + checkDepsChanges.mockImplementationOnce(mockDepsChanges([`gatsby-cli`])) + + watch(process.cwd(), [`gatsby-cli`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [`gatsby`, `gatsby-plugin-sharp`], + }) + + const filePath = path.join( + process.cwd(), + `packages/gatsby-cli/package.json` + ) + await callEventCallback(`add`, filePath) + await callReadyCallback() + + assertPublish({ + include: [`gatsby`, `gatsby-cli`], + exclude: [`gatsby-plugin-sharp`], + }) + + assertInstall({ + include: [`gatsby`], + exclude: [`gatsby-cli`, `gatsby-plugin-sharp`], + }) + }) + + it(`watching gatsby-source-filesytem and having gatsby-source-wordpress installs gatsby-source-wordpress`, async () => { + checkDepsChanges.mockImplementationOnce( + mockDepsChanges([`gatsby-source-filesystem`]) + ) + + watch(process.cwd(), [`gatsby-source-filesystem`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [ + `gatsby`, + `gatsby-source-wordpress`, + `gatsby-plugin-sharp`, + ], + }) + + const filePath = path.join( + process.cwd(), + `packages/gatsby-source-filesystem/package.json` + ) + await callEventCallback(`add`, filePath) + await callReadyCallback() + + assertPublish({ + include: [`gatsby-source-wordpress`, `gatsby-source-filesystem`], + exclude: [`gatsby`, `gatsby-plugin-sharp`], + }) + + assertInstall({ + include: [`gatsby-source-wordpress`], + exclude: [`gatsby`, `gatsby-source-filesystem`, `gatsby-plugin-sharp`], + }) + }) + + it(`watching gatsby-source-filesytem and having gatsby-source-filesystem installs gatsby-source-filesystem`, async () => { + checkDepsChanges.mockImplementationOnce( + mockDepsChanges([`gatsby-source-filesystem`]) + ) + + watch(process.cwd(), [`gatsby-source-filesystem`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [ + `gatsby`, + `gatsby-source-filesystem`, + `gatsby-plugin-sharp`, + ], + }) + + const filePath = path.join( + process.cwd(), + `packages/gatsby-source-filesystem/package.json` + ) + await callEventCallback(`add`, filePath) + await callReadyCallback() + + assertPublish({ + include: [`gatsby-source-filesystem`], + exclude: [`gatsby`, `gatsby-plugin-sharp`], + }) + + assertInstall({ + include: [`gatsby-source-filesystem`], + exclude: [`gatsby`, `gatsby-plugin-sharp`], + }) + }) + + it(`watching gatsby-source-filesystem and not having gatsby-source-filesystem or gatsby-source-wordpress, installs nothing`, async () => { + checkDepsChanges.mockImplementationOnce( + mockDepsChanges([`gatsby-source-filesystem`]) + ) + + watch(process.cwd(), [`gatsby-source-filesystem`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [`gatsby`, `gatsby-plugin-sharp`], + }) + + const filePath = path.join( + process.cwd(), + `packages/gatsby-source-filesystem/package.json` + ) + await callEventCallback(`add`, filePath) + await callReadyCallback() + + assertPublish({ + include: [], + exclude: [ + `gatsby`, + `gatsby-source-filesystem`, + `gatsby-source-wordpress`, + `gatsby-plugin-sharp`, + ], + }) + + assertInstall({ + include: [], + exclude: [ + `gatsby`, + `gatsby-source-filesystem`, + `gatsby-source-wordpress`, + `gatsby-plugin-sharp`, + ], + }) + }) + + it(`watching gatsby-source-filesytem and both having gatsby-source-filesystem and gatsby-source-wordpress, should install both`, async () => { + checkDepsChanges.mockImplementationOnce( + mockDepsChanges([`gatsby-source-filesystem`]) + ) + + watch(process.cwd(), [`gatsby-source-filesystem`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [ + `gatsby`, + `gatsby-source-wordpress`, + `gatsby-source-filesystem`, + `gatsby-plugin-sharp`, + ], + }) + + const filePath = path.join( + process.cwd(), + `packages/gatsby-source-filesystem/package.json` + ) + await callEventCallback(`add`, filePath) + await callReadyCallback() + + assertPublish({ + include: [`gatsby-source-filesystem`, `gatsby-source-wordpress`], + exclude: [`gatsby`, `gatsby-plugin-sharp`], + }) + + assertInstall({ + include: [`gatsby-source-filesystem`, `gatsby-source-wordpress`], + exclude: [`gatsby`, `gatsby-plugin-sharp`], + }) + }) + }) + + describe(`order of operation`, () => { + it(`publish and installs from verdaccio before copying files`, async () => { + let lastOp = null + let installWasCalledAfterFsCopy = false + installPackages.mockImplementation(() => { + if (lastOp === fs.copy) { + installWasCalledAfterFsCopy = true + } + lastOp = installPackages + }) + + fs.copy.mockImplementation(() => { + lastOp = fs.copy + }) + + checkDepsChanges.mockImplementationOnce(mockDepsChanges([`gatsby`])) + + watch(process.cwd(), [`gatsby`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [`gatsby`, `gatsby-plugin-sharp`], + }) + + const filePath = path.join(process.cwd(), `packages/gatsby/package.json`) + await callEventCallback( + `add`, + path.join(process.cwd(), `packages/gatsby/not-package.json`) + ) + await callEventCallback(`add`, filePath) + + await callReadyCallback() + + expect(installWasCalledAfterFsCopy).toBe(false) + expect(installPackages).toBeCalled() + expect(fs.copy).toBeCalled() + }) + + it(`installs from npm before copying files`, async () => { + let lastOp = null + let installWasCalledAfterFsCopy = false + promisifiedSpawn.mockImplementation(() => { + if (lastOp === fs.copy) { + installWasCalledAfterFsCopy = true + } + lastOp = promisifiedSpawn + }) + + fs.copy.mockImplementation(() => { + lastOp = fs.copy + }) + + checkDepsChanges.mockImplementationOnce(() => + Promise.resolve({ + didDepsChanged: false, + packageNotInstalled: true, + }) + ) + + watch(process.cwd(), [`gatsby`], { + scanOnce: true, + quiet: true, + monoRepoPackages, + localPackages: [`gatsby`, `gatsby-plugin-sharp`], + }) + + const filePath = path.join(process.cwd(), `packages/gatsby/package.json`) + await callEventCallback( + `add`, + path.join(process.cwd(), `packages/gatsby/not-package.json`) + ) + await callEventCallback(`add`, filePath) + + await callReadyCallback() + + expect(installWasCalledAfterFsCopy).toBe(false) + expect(promisifiedSpawn).toBeCalled() + expect(fs.copy).toBeCalled() + }) + }) +}) diff --git a/packages/gatsby-dev-cli/src/index.js b/packages/gatsby-dev-cli/src/index.js index e1a0f87d23b26..99a1b67f1a307 100644 --- a/packages/gatsby-dev-cli/src/index.js +++ b/packages/gatsby-dev-cli/src/index.js @@ -68,22 +68,17 @@ gatsby-dev --set-path-to-repo /path/to/my/cloned/version/gatsby process.exit() } -const localPkg = JSON.parse(fs.readFileSync(`package.json`)) -let packages = Object.keys( - _.merge({}, localPkg.dependencies, localPkg.devDependencies) -) - // get list of packages from monorepo const monoRepoPackages = fs.readdirSync(path.join(gatsbyLocation, `packages`)) -if (argv.copyAll) { - packages = monoRepoPackages -} else { - // intersect dependencies with monoRepoPackags to get list of packages to watch - packages = _.intersection(monoRepoPackages, packages) -} +const localPkg = JSON.parse(fs.readFileSync(`package.json`)) +// intersect dependencies with monoRepoPackags to get list of packages that are used +let localPackages = _.intersection( + monoRepoPackages, + Object.keys(_.merge({}, localPkg.dependencies, localPkg.devDependencies)) +) -if (!argv.packages && _.isEmpty(packages)) { +if (!argv.packages && _.isEmpty(localPackages)) { console.error( ` You haven't got any gatsby dependencies into your current package.json @@ -100,7 +95,8 @@ gatsby-dev will pick them up. process.exit() } -watch(gatsbyLocation, argv.packages || packages, { +watch(gatsbyLocation, argv.packages, { + localPackages, quiet: argv.quiet, scanOnce: argv.scanOnce, monoRepoPackages, diff --git a/packages/gatsby-dev-cli/src/local-npm-registry/index.js b/packages/gatsby-dev-cli/src/local-npm-registry/index.js new file mode 100644 index 0000000000000..0922b3f265c95 --- /dev/null +++ b/packages/gatsby-dev-cli/src/local-npm-registry/index.js @@ -0,0 +1,68 @@ +const startVerdaccio = require(`verdaccio`).default + +const fs = require(`fs-extra`) +const _ = require(`lodash`) + +let VerdaccioInitPromise = null + +const { verdaccioConfig } = require(`./verdaccio-config`) +const { publishPackage } = require(`./publish-package`) +const { installPackages } = require(`./install-packages`) + +const startServer = () => { + if (VerdaccioInitPromise) { + return VerdaccioInitPromise + } + + console.log(`Starting local verdaccio server`) + + // clear storage + fs.removeSync(verdaccioConfig.storage) + + VerdaccioInitPromise = new Promise(resolve => { + startVerdaccio( + verdaccioConfig, + verdaccioConfig.port, + verdaccioConfig.storage, + `1.0.0`, + `gatsby-dev`, + (webServer, addr, pkgName, pkgVersion) => { + // console.log(webServer) + webServer.listen(addr.port || addr.path, addr.host, () => { + console.log(`Started local verdaccio server`) + + resolve() + }) + } + ) + }) + + return VerdaccioInitPromise +} + +exports.startVerdaccio = startServer + +exports.publishPackagesLocallyAndInstall = async ({ + packagesToPublish, + localPackages, + root, + ignorePackageJSONChanges, +}) => { + await startServer() + + const versionPostFix = Date.now() + + for (let packageName of packagesToPublish) { + await publishPackage({ + packageName, + packagesToPublish, + root, + versionPostFix, + ignorePackageJSONChanges, + }) + } + + const packagesToInstall = _.intersection(packagesToPublish, localPackages) + + installPackages({ packagesToInstall }) +} diff --git a/packages/gatsby-dev-cli/src/local-npm-registry/install-packages.js b/packages/gatsby-dev-cli/src/local-npm-registry/install-packages.js new file mode 100644 index 0000000000000..fb62127810109 --- /dev/null +++ b/packages/gatsby-dev-cli/src/local-npm-registry/install-packages.js @@ -0,0 +1,30 @@ +const { promisifiedSpawn } = require(`../utils/promisified-spawn`) +const { registryUrl } = require(`./verdaccio-config`) + +const installPackages = async ({ packagesToInstall }) => { + const installCmd = [ + `yarn`, + [ + `add`, + ...packagesToInstall.map(packageName => `${packageName}@gatsby-dev`), + `--registry=${registryUrl}`, + `--exact`, + ], + ] + + console.log( + `Installing packages from local registry:\n${packagesToInstall + .map(packageAndVersion => ` - ${packageAndVersion}`) + .join(`\n`)}` + ) + + try { + await promisifiedSpawn(installCmd) + + console.log(`Installation complete`) + } catch { + console.error(`Installation failed`) + } +} + +exports.installPackages = installPackages diff --git a/packages/gatsby-dev-cli/src/local-npm-registry/verdaccio.js b/packages/gatsby-dev-cli/src/local-npm-registry/publish-package.js similarity index 61% rename from packages/gatsby-dev-cli/src/local-npm-registry/verdaccio.js rename to packages/gatsby-dev-cli/src/local-npm-registry/publish-package.js index 6614a1833dc2b..b6c08510dd514 100644 --- a/packages/gatsby-dev-cli/src/local-npm-registry/verdaccio.js +++ b/packages/gatsby-dev-cli/src/local-npm-registry/publish-package.js @@ -1,79 +1,18 @@ -const startVerdaccio = require(`verdaccio`).default -const path = require(`path`) const fs = require(`fs-extra`) -const _ = require(`lodash`) -const os = require(`os`) +const path = require(`path`) -const { - getMonorepoPackageJsonPath, -} = require(`../utils/get-monorepo-package-json-path`) const { promisifiedSpawn } = require(`../utils/promisified-spawn`) -const { registerCleanupTask } = require(`./cleanup-tasks`) - -let VerdaccioInitPromise = null - -const verdaccioConfig = { - storage: path.join(os.tmpdir(), `verdaccio`, `storage`), - port: 4873, // default - web: { - enable: true, - title: `gatsby-dev`, - }, - logs: [{ type: `stdout`, format: `pretty-timestamped`, level: `warn` }], - packages: { - "**": { - access: `$all`, - publish: `$all`, - proxy: `npmjs`, - }, - }, - uplinks: { - npmjs: { - url: `https://registry.npmjs.org/`, - }, - }, -} +const { registryUrl } = require(`./verdaccio-config`) -const registryUrl = `http://localhost:${verdaccioConfig.port}` const NPMRCContent = `${registryUrl.replace( /https?:/g, `` )}/:_authToken="gatsby-dev"` -exports.registryUrl = registryUrl - -const startServer = () => { - if (VerdaccioInitPromise) { - return VerdaccioInitPromise - } - - console.log(`Starting local verdaccio server`) - - // clear storage - fs.removeSync(verdaccioConfig.storage) - - VerdaccioInitPromise = new Promise(resolve => { - startVerdaccio( - verdaccioConfig, - verdaccioConfig.port, - verdaccioConfig.storage, - `1.0.0`, - `gatsby-dev`, - (webServer, addr, pkgName, pkgVersion) => { - // console.log(webServer) - webServer.listen(addr.port || addr.path, addr.host, () => { - console.log(`Started local verdaccio server`) - - resolve() - }) - } - ) - }) - - return VerdaccioInitPromise -} - -exports.startVerdaccio = startServer +const { + getMonorepoPackageJsonPath, +} = require(`../utils/get-monorepo-package-json-path`) +const { registerCleanupTask } = require(`./cleanup-tasks`) /** * Edit package.json to: @@ -196,45 +135,4 @@ const publishPackage = async ({ unadjustPackageJson() } -exports.publishPackagesLocallyAndInstall = async ({ - packagesToPublish, - packages, - root, - ignorePackageJSONChanges, -}) => { - await startServer() - - const versionPostFix = Date.now() - - for (let packageName of packagesToPublish) { - await publishPackage({ - packageName, - packagesToPublish, - root, - versionPostFix, - ignorePackageJSONChanges, - }) - } - - const packagesToInstall = _.intersection(packagesToPublish, packages).map( - packageName => `${packageName}@gatsby-dev` - ) - const installCmd = [ - `yarn`, - [`add`, ...packagesToInstall, `--registry=${registryUrl}`, `--exact`], - ] - - console.log( - `Installing packages from local registry:\n${packagesToInstall - .map(packageAndVersion => ` - ${packageAndVersion}`) - .join(`\n`)}` - ) - - try { - await promisifiedSpawn(installCmd) - - console.log(`Installation complete`) - } catch { - console.error(`Installation failed`) - } -} +exports.publishPackage = publishPackage diff --git a/packages/gatsby-dev-cli/src/local-npm-registry/verdaccio-config.js b/packages/gatsby-dev-cli/src/local-npm-registry/verdaccio-config.js new file mode 100644 index 0000000000000..c95574e547d72 --- /dev/null +++ b/packages/gatsby-dev-cli/src/local-npm-registry/verdaccio-config.js @@ -0,0 +1,30 @@ +const path = require(`path`) +const os = require(`os`) + +const verdaccioConfig = { + storage: path.join(os.tmpdir(), `verdaccio`, `storage`), + port: 4873, // default + web: { + enable: true, + title: `gatsby-dev`, + }, + logs: [{ type: `stdout`, format: `pretty-timestamped`, level: `warn` }], + packages: { + "**": { + access: `$all`, + publish: `$all`, + proxy: `npmjs`, + }, + }, + uplinks: { + npmjs: { + url: `https://registry.npmjs.org/`, + }, + }, +} + +exports.verdaccioConfig = verdaccioConfig + +const registryUrl = `http://localhost:${verdaccioConfig.port}` + +exports.registryUrl = registryUrl diff --git a/packages/gatsby-dev-cli/src/utils/traverse-package-deps.js b/packages/gatsby-dev-cli/src/utils/traverse-package-deps.js index 964a5c3597059..0e035d509e48e 100644 --- a/packages/gatsby-dev-cli/src/utils/traverse-package-deps.js +++ b/packages/gatsby-dev-cli/src/utils/traverse-package-deps.js @@ -9,7 +9,7 @@ const path = require(`path`) * ``` * { * "gatsby-cli": Set(["gatsby"]), - * "gatsby-telemtry": Set(["gatsby", "gatsby-cli"]), + * "gatsby-telemetry": Set(["gatsby", "gatsby-cli"]), * "gatsby-source-filesystem": Set(["gatsby-source-contentful", "gatsby-source-drupal", "gatsby-source-wordpress", etc]) * // no package have remark plugin in dependencies - so dependant list is empty * "gatsby-transformer-remark": Set([]) diff --git a/packages/gatsby-dev-cli/src/watch.js b/packages/gatsby-dev-cli/src/watch.js index d1ae85718312d..0e35364da9c50 100644 --- a/packages/gatsby-dev-cli/src/watch.js +++ b/packages/gatsby-dev-cli/src/watch.js @@ -1,11 +1,10 @@ const chokidar = require(`chokidar`) const _ = require(`lodash`) +const del = require(`del`) const fs = require(`fs-extra`) const path = require(`path`) -const { - publishPackagesLocallyAndInstall, -} = require(`./local-npm-registry/verdaccio`) +const { publishPackagesLocallyAndInstall } = require(`./local-npm-registry`) const { checkDepsChanges } = require(`./utils/check-deps-changes`) const { getDependantPackages } = require(`./utils/get-dependant-packages`) const { promisifiedSpawn } = require(`./utils/promisified-spawn`) @@ -22,7 +21,11 @@ const quit = () => { * non-existant packages break on('ready') * See: https://github.com/paulmillr/chokidar/issues/449 */ -function watch(root, packages, { scanOnce, quiet, monoRepoPackages }) { +function watch( + root, + packages, + { scanOnce, quiet, monoRepoPackages, localPackages } +) { let afterPackageInstallation = false let queuedCopies = [] @@ -41,9 +44,9 @@ function watch(root, packages, { scanOnce, quiet, monoRepoPackages }) { }) } - const copyPath = (oldPath, newPath, quiet) => + const copyPath = (oldPath, newPath, quiet, packageName) => new Promise((resolve, reject) => { - const argObj = { oldPath, newPath, quiet, resolve, reject } + const argObj = { oldPath, newPath, quiet, packageName, resolve, reject } if (afterPackageInstallation) { realCopyPath(argObj) } else { @@ -56,14 +59,50 @@ function watch(root, packages, { scanOnce, quiet, monoRepoPackages }) { queuedCopies.forEach(argObj => realCopyPath(argObj)) queuedCopies = [] } + + const clearJSFilesFromNodeModules = async () => { + const packagesToClear = queuedCopies.reduce((acc, { packageName }) => { + if (packageName) { + acc.add(packageName) + } + return acc + }, new Set()) + + await Promise.all( + [...packagesToClear].map(async packageToClear => { + await del([ + `node_modules/${packageToClear}/**/*.{js,js.map}`, + `!node_modules/${packageToClear}/node_modules/**/*.{js,js.map}`, + ]) + }) + ) + } // check packages deps and if they depend on other packages from monorepo // add them to packages list - const { seenPackages: allPackagesToWatch, depTree } = traversePackagesDeps({ + const { seenPackages, depTree } = traversePackagesDeps({ root, - packages, + packages: _.uniq(localPackages), monoRepoPackages, }) + const allPackagesToWatch = packages + ? _.intersection(packages, seenPackages) + : seenPackages + + let packagesToInstall = [] + + const getPackagesToInstall = packages => + packages.forEach(pkg => { + if (localPackages.includes(pkg)) { + packagesToInstall.push(pkg) + } + if (depTree[pkg]) { + getPackagesToInstall([...depTree[pkg]]) + } + }) + + getPackagesToInstall(allPackagesToWatch) + if (allPackagesToWatch.length === 0) { console.error(`There are no packages to watch.`) return @@ -189,13 +228,7 @@ function watch(root, packages, { scanOnce, quiet, monoRepoPackages }) { return } - if (packagesToPublish.has(packageName)) { - // we are in middle of publishing to localy registry, - // so we don't need to copy files as yarn will handle this - return - } - - let localCopies = [copyPath(filePath, newPath, quiet)] + let localCopies = [copyPath(filePath, newPath, quiet, packageName)] // If this is from "cache-dir" also copy it into the site's .cache if (_.includes(filePath, `cache-dir`)) { @@ -216,14 +249,13 @@ function watch(root, packages, { scanOnce, quiet, monoRepoPackages }) { if (isInitialScan) { isInitialScan = false if (packagesToPublish.size > 0) { - const publishAndInstallPromise = publishPackagesLocallyAndInstall({ + await publishPackagesLocallyAndInstall({ packagesToPublish: Array.from(packagesToPublish), root, - packages, + localPackages, ignorePackageJSONChanges, }) packagesToPublish.clear() - allCopies.push(publishAndInstallPromise) } else if (anyPackageNotInstalled) { // run `yarn` const yarnInstallCmd = [`yarn`] @@ -233,6 +265,7 @@ function watch(root, packages, { scanOnce, quiet, monoRepoPackages }) { console.log(`Installation complete`) } + await clearJSFilesFromNodeModules() runQueuedCopies() }