From 0410f323c584920d9663955914ef21ca4d7d4955 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 16 Dec 2019 11:59:40 +1100 Subject: [PATCH 01/16] v5.0.7: bump version and update changelog Republish of v5.0.6 but with node-gyp-v5.0.6.tar.gz removed from pack file PR-URL: https://github.com/nodejs/node-gyp/pull/1972 --- CHANGELOG.md | 5 +++++ package.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9179f7b5df..28f5237798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v5.0.7 2019-12-16 +================= + +Republish of v5.0.6 with unnecessary tarball removed from pack file. + v5.0.6 2019-12-16 ================= diff --git a/package.json b/package.json index 0b2edd85ca..ebd8fa697e 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "bindings", "gyp" ], - "version": "5.0.6", + "version": "5.0.7", "installVersion": 9, "author": "Nathan Rajlich (http://tootallnate.net)", "repository": { From 0d5a415a148cf1d45d0968e268106604bed9329b Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 18 Nov 2019 13:37:59 +1100 Subject: [PATCH 02/16] doc: add travis badge PR-URL: https://github.com/nodejs/node-gyp/pull/1971 Reviewed-By: Richard Lau Reviewed-By: Jiawen Geng Reviewed-By: Jiawen Geng --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e39c5de8ea..29a49104bc 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # `node-gyp` - Node.js native addon build tool +[![Build Status](https://travis-ci.com/nodejs/node-gyp.svg?branch=master)](https://travis-ci.com/nodejs/node-gyp) + `node-gyp` is a cross-platform command-line tool written in Node.js for compiling native addon modules for Node.js. It contains a fork of the [gyp](https://gyp.gsrc.io) project that was previously used by the Chromium From 0607596a4c65a43bca7f8ead73200978e5fbe8d1 Mon Sep 17 00:00:00 2001 From: Suraneti Rodsuwan Date: Fri, 13 Dec 2019 13:19:46 +0700 Subject: [PATCH 03/16] doc: fix typo in README.md (#1985) PR-URL: https://github.com/nodejs/node-gyp/pull/1985 Reviewed-By: Christian Clauss Reviewed-By: Jiawen Geng --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29a49104bc..01f5d4e1f1 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ etc.), regardless of what version of Node.js is actually installed on your syste ## Features * The same build commands work on any of the supported platforms - * Supports the targetting of different versions of Node.js + * Supports the targeting of different versions of Node.js ## Installation From fd4b1351e490564ebe54a560e6bc2de37f545e80 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 2 Dec 2019 17:55:45 +0100 Subject: [PATCH 04/16] test: initial Github Actions with Ubuntu & macOS Running Python standalone tests on multiple OSes would free up Travis CI for tests of various combinations of Node.js and Python as well as tests on other [CPU architectures](https://docs.travis-ci.com/user/multi-cpu-architectures). __arch: amd64, arm64, ppc64le, s390x__ PR-URL: https://github.com/nodejs/node-gyp/pull/1985 Reviewed-By: Rod Vagg Reviewed-By: Jiawen Geng --- .github/workflows/Python_tests.yml | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/Python_tests.yml diff --git a/.github/workflows/Python_tests.yml b/.github/workflows/Python_tests.yml new file mode 100644 index 0000000000..235127f21a --- /dev/null +++ b/.github/workflows/Python_tests.yml @@ -0,0 +1,36 @@ +# TODO: Line 14, enable os: windows-latest +# TODO: Line 15, enable python-version: 3.5 +# TODO: Line 36, enable pytest --doctest-modules + +name: Python_tests +on: [push, pull_request] +jobs: + Python_tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + max-parallel: 15 + matrix: + os: [macos-latest, ubuntu-latest] # , windows-latest] + python-version: [2.7, 3.6, 3.7, 3.8] # 3.5, + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest # -r requirements.txt + - name: Lint with flake8 + if: matrix.os == 'ubuntu-latest' + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: pytest + # - name: Run doctests with pytest + # run: pytest --doctest-modules From 32c8744b34f4b63aa33b49f48d410f568317c100 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 28 Nov 2019 18:49:45 +0100 Subject: [PATCH 05/16] test: fix macOS Travis on Python 2.7 & 3.7 Uses `pyenv` to manage MacOS python versions since its not included in the environment. rvagg: landing this from #1979 even though it wasn't from the original author. Treating approval there as approval of this commit too. PR-URL: https://github.com/nodejs/node-gyp/pull/1979 Reviewed-By: Rod Vagg --- .travis.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e43ba548fb..998669cdc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,13 @@ dist: xenial language: python cache: pip -matrix: +addons: + homebrew: + update: true + packages: + - npm + - pyenv +jobs: include: - name: "Python 2.7 on Linux" env: NODE_GYP_FORCE_PYTHON=python2 @@ -11,7 +17,9 @@ matrix: osx_image: xcode11.2 language: shell # 'language: python' is not yet supported on macOS env: NODE_GYP_FORCE_PYTHON=python2 - before_install: HOMEBREW_NO_AUTO_UPDATE=1 brew install npm + before_install: + - pyenv install 2.7 + - pyenv global 2.7 - name: "Node.js 6 & Python 2.7 on Windows" os: windows language: node_js @@ -63,7 +71,6 @@ matrix: osx_image: xcode11.2 language: shell # 'language: python' is not yet supported on macOS env: NODE_GYP_FORCE_PYTHON=python3 - before_install: HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade npm || true - name: "Node.js 12 & Python 3.7 on Windows" os: windows language: node_js From 0923f344c91920cb619eff2a60d549f1f39204ae Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Thu, 28 Nov 2019 13:10:55 -0500 Subject: [PATCH 06/16] test: direct python invocation & simpler pyenv Reorder Travis builds by OS. Replace `pyenv global` calls with properly set `PATH` and `PYENV_VERSION` env vars. Does not assume python modules are in the `PATH` so all python modules are prefixed with `python -m`. PR-URL: https://github.com/nodejs/node-gyp/pull/1979 Reviewed-By: Christian Clauss Reviewed-By: Rod Vagg --- .travis.yml | 64 ++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 998669cdc3..22f7bc12fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,30 +12,6 @@ jobs: - name: "Python 2.7 on Linux" env: NODE_GYP_FORCE_PYTHON=python2 python: 2.7 - - name: "Python 2.7 on macOS" - os: osx - osx_image: xcode11.2 - language: shell # 'language: python' is not yet supported on macOS - env: NODE_GYP_FORCE_PYTHON=python2 - before_install: - - pyenv install 2.7 - - pyenv global 2.7 - - name: "Node.js 6 & Python 2.7 on Windows" - os: windows - language: node_js - node_js: 6 # node - env: >- - PATH=/c/Python27:/c/Python27/Scripts:$PATH - NODE_GYP_FORCE_PYTHON=/c/Python27/python.exe - before_install: choco install python2 - - name: "Node.js 12 & Python 2.7 on Windows" - os: windows - language: node_js - node_js: 12 # node - env: >- - PATH=/c/Python27:/c/Python27/Scripts:$PATH - NODE_GYP_FORCE_PYTHON=/c/Python27/python.exe - before_install: choco install python2 - name: "Node.js 6 & Python 3.8 on Linux" python: 3.8 @@ -66,11 +42,36 @@ jobs: env: NODE_GYP_FORCE_PYTHON=python3 before_install: nvm install 12 - - name: "Python 3.7 on macOS" + - name: "Python 2.7 on macOS" os: osx osx_image: xcode11.2 language: shell # 'language: python' is not yet supported on macOS - env: NODE_GYP_FORCE_PYTHON=python3 + env: NODE_GYP_FORCE_PYTHON=python2 PATH=$HOME/.pyenv/shims:$PATH PYENV_VERSION=2.7.17 + before_install: pyenv install $PYENV_VERSION + - name: "Python 3.8 on macOS" + os: osx + osx_image: xcode11.2 + language: shell # 'language: python' is not yet supported on macOS + env: NODE_GYP_FORCE_PYTHON=python3 PATH=$HOME/.pyenv/shims:$PATH PYENV_VERSION=3.8.0 + before_install: pyenv install $PYENV_VERSION + + - name: "Node.js 6 & Python 2.7 on Windows" + os: windows + language: node_js + node_js: 6 # node + env: >- + PATH=/c/Python27:/c/Python27/Scripts:$PATH + NODE_GYP_FORCE_PYTHON=/c/Python27/python.exe + before_install: choco install python2 + - name: "Node.js 12 & Python 2.7 on Windows" + os: windows + language: node_js + node_js: 12 # node + env: >- + PATH=/c/Python27:/c/Python27/Scripts:$PATH + NODE_GYP_FORCE_PYTHON=/c/Python27/python.exe + before_install: choco install python2 + - name: "Node.js 12 & Python 3.7 on Windows" os: windows language: node_js @@ -89,20 +90,19 @@ jobs: before_install: choco install python install: - #- pip install -r requirements.txt - - pip install --upgrade flake8 pytest==4.6.6 # pytest 5 no longer supports legacy Python + - python -m pip install --upgrade flake8 pytest==4.6.6 # pytest 5 no longer supports legacy Python before_script: - - flake8 --version + - python -m flake8 --version # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + - python -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. Two space indentation is OK. The GitHub editor is 127 chars wide - - flake8 . --count --exit-zero --ignore=E111,E114,W503 --max-complexity=10 --max-line-length=127 --statistics + - python -m flake8 . --count --exit-zero --ignore=E111,E114,W503 --max-complexity=10 --max-line-length=127 --statistics - npm install - npm list script: - node -e 'require("npmlog").level="verbose"; require("./lib/find-python")(null,()=>{})' - npm test - - GYP_MSVS_VERSION=2015 GYP_MSVS_OVERRIDE_PATH="C:\\Dummy" pytest + - GYP_MSVS_VERSION=2015 GYP_MSVS_OVERRIDE_PATH="C:\\Dummy" python -m pytest notifications: on_success: change on_failure: change # `always` will be the setting once code changes slow down From a3f1143514c9398769e3f6193f2135aff2e1df85 Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Wed, 27 Nov 2019 19:40:00 -0500 Subject: [PATCH 07/16] lib: noproxy support, match proxy detection to `request` PR-URL: https://github.com/nodejs/node-gyp/pull/1978 Reviewed-By: Rod Vagg --- README.md | 3 +- lib/install.js | 6 +-- lib/node-gyp.js | 1 + lib/proxy.js | 92 +++++++++++++++++++++++++++++++++++++++++ test/test-download.js | 95 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 lib/proxy.js diff --git a/README.md b/README.md index 01f5d4e1f1..81d90a650b 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,8 @@ Some additional resources for Node.js native addons and writing `gyp` configurat | `--devdir=$path` | SDK download directory (default is OS cache directory) | `--ensure` | Don't reinstall headers if already present | `--dist-url=$url` | Download header tarball from custom URL -| `--proxy=$url` | Set HTTP proxy for downloading header tarball +| `--proxy=$url` | Set HTTP(S) proxy for downloading header tarball +| `--noproxy=$urls` | Set urls to ignore proxies when downloading header tarball | `--cafile=$cafile` | Override default CA chain (to download tarball) | `--nodedir=$path` | Set the path to the node source code | `--python=$path` | Set path to the Python binary diff --git a/lib/install.js b/lib/install.js index f68cd7fd6d..c919c10588 100644 --- a/lib/install.js +++ b/lib/install.js @@ -11,6 +11,7 @@ const request = require('request') const mkdir = require('mkdirp') const processRelease = require('./process-release') const win = process.platform === 'win32' +const getProxyFromURI = require('./proxy') function install (fs, gyp, argv, callback) { var release = processRelease(argv, gyp, process.version, process.release) @@ -410,10 +411,7 @@ function download (gyp, env, url) { } // basic support for a proxy server - var proxyUrl = gyp.opts.proxy || - env.http_proxy || - env.HTTP_PROXY || - env.npm_config_proxy + var proxyUrl = getProxyFromURI(gyp, env, url) if (proxyUrl) { if (/^https?:\/\//i.test(proxyUrl)) { log.verbose('download', 'using proxy url: "%s"', proxyUrl) diff --git a/lib/node-gyp.js b/lib/node-gyp.js index 9d24103900..81fc590919 100644 --- a/lib/node-gyp.js +++ b/lib/node-gyp.js @@ -67,6 +67,7 @@ proto.configDefs = { ensure: Boolean, // 'install' solution: String, // 'build' (windows only) proxy: String, // 'install' + noproxy: String, // 'install' devdir: String, // everywhere nodedir: String, // 'configure' loglevel: String, // everywhere diff --git a/lib/proxy.js b/lib/proxy.js new file mode 100644 index 0000000000..92d9ed2f7f --- /dev/null +++ b/lib/proxy.js @@ -0,0 +1,92 @@ +'use strict' +// Taken from https://github.com/request/request/blob/212570b/lib/getProxyFromURI.js + +const url = require('url') + +function formatHostname (hostname) { + // canonicalize the hostname, so that 'oogle.com' won't match 'google.com' + return hostname.replace(/^\.*/, '.').toLowerCase() +} + +function parseNoProxyZone (zone) { + zone = zone.trim().toLowerCase() + + var zoneParts = zone.split(':', 2) + var zoneHost = formatHostname(zoneParts[0]) + var zonePort = zoneParts[1] + var hasPort = zone.indexOf(':') > -1 + + return { hostname: zoneHost, port: zonePort, hasPort: hasPort } +} + +function uriInNoProxy (uri, noProxy) { + var port = uri.port || (uri.protocol === 'https:' ? '443' : '80') + var hostname = formatHostname(uri.hostname) + var noProxyList = noProxy.split(',') + + // iterate through the noProxyList until it finds a match. + return noProxyList.map(parseNoProxyZone).some(function (noProxyZone) { + var isMatchedAt = hostname.indexOf(noProxyZone.hostname) + var hostnameMatched = ( + isMatchedAt > -1 && + (isMatchedAt === hostname.length - noProxyZone.hostname.length) + ) + + if (noProxyZone.hasPort) { + return (port === noProxyZone.port) && hostnameMatched + } + + return hostnameMatched + }) +} + +function getProxyFromURI (gyp, env, uri) { + // If a string URI/URL was given, parse it into a URL object + if (typeof uri === 'string') { + // eslint-disable-next-line + uri = url.parse(uri) + } + + // Decide the proper request proxy to use based on the request URI object and the + // environmental variables (NO_PROXY, HTTP_PROXY, etc.) + // respect NO_PROXY environment variables (see: https://lynx.invisible-island.net/lynx2.8.7/breakout/lynx_help/keystrokes/environments.html) + + var noProxy = gyp.opts.noproxy || env.NO_PROXY || env.no_proxy || env.npm_config_noproxy || '' + + // if the noProxy is a wildcard then return null + + if (noProxy === '*') { + return null + } + + // if the noProxy is not empty and the uri is found return null + + if (noProxy !== '' && uriInNoProxy(uri, noProxy)) { + return null + } + + // Check for HTTP or HTTPS Proxy in environment Else default to null + + if (uri.protocol === 'http:') { + return gyp.opts.proxy || + env.HTTP_PROXY || + env.http_proxy || + env.npm_config_proxy || null + } + + if (uri.protocol === 'https:') { + return gyp.opts.proxy || + env.HTTPS_PROXY || + env.https_proxy || + env.HTTP_PROXY || + env.http_proxy || + env.npm_config_proxy || null + } + + // if none of that works, return null + // (What uri protocol are you using then?) + + return null +} + +module.exports = getProxyFromURI diff --git a/test/test-download.js b/test/test-download.js index 738a43f276..c88f0c612c 100644 --- a/test/test-download.js +++ b/test/test-download.js @@ -90,6 +90,101 @@ test('download over https with custom ca', function (t) { }) }) +test('download over http with proxy', function (t) { + t.plan(2) + + var server = http.createServer(function (req, res) { + t.strictEqual(req.headers['user-agent'], + 'node-gyp v42 (node ' + process.version + ')') + res.end('ok') + pserver.close(function () { + server.close() + }) + }) + + var pserver = http.createServer(function (req, res) { + t.strictEqual(req.headers['user-agent'], + 'node-gyp v42 (node ' + process.version + ')') + res.end('proxy ok') + server.close(function () { + pserver.close() + }) + }) + + var host = 'localhost' + server.listen(0, host, function () { + var port = this.address().port + pserver.listen(port + 1, host, function () { + var gyp = { + opts: { + proxy: 'http://' + host + ':' + (port + 1) + }, + version: '42' + } + var url = 'http://' + host + ':' + port + var req = install.test.download(gyp, {}, url) + req.on('response', function (res) { + var body = '' + res.setEncoding('utf8') + res.on('data', function (data) { + body += data + }) + res.on('end', function () { + t.strictEqual(body, 'proxy ok') + }) + }) + }) + }) +}) + +test('download over http with noproxy', function (t) { + t.plan(2) + + var server = http.createServer(function (req, res) { + t.strictEqual(req.headers['user-agent'], + 'node-gyp v42 (node ' + process.version + ')') + res.end('ok') + pserver.close(function () { + server.close() + }) + }) + + var pserver = http.createServer(function (req, res) { + t.strictEqual(req.headers['user-agent'], + 'node-gyp v42 (node ' + process.version + ')') + res.end('proxy ok') + server.close(function () { + pserver.close() + }) + }) + + var host = 'localhost' + server.listen(0, host, function () { + var port = this.address().port + pserver.listen(port + 1, host, function () { + var gyp = { + opts: { + proxy: 'http://' + host + ':' + (port + 1), + noproxy: 'localhost' + }, + version: '42' + } + var url = 'http://' + host + ':' + port + var req = install.test.download(gyp, {}, url) + req.on('response', function (res) { + var body = '' + res.setEncoding('utf8') + res.on('data', function (data) { + body += data + }) + res.on('end', function () { + t.strictEqual(body, 'ok') + }) + }) + }) + }) +}) + test('download with missing cafile', function (t) { t.plan(1) var gyp = { From 91ee26dd4837de15658a82b70476f74ad0e54981 Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Mon, 23 Dec 2019 02:58:15 +0000 Subject: [PATCH 08/16] test: fix typo in header download test (#2001) PR-URL: https://github.com/nodejs/node-gyp/pull/2001 Reviewed-By: Christian Clauss Reviewed-By: Rod Vagg --- test/test-download.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-download.js b/test/test-download.js index c88f0c612c..fe373e3280 100644 --- a/test/test-download.js +++ b/test/test-download.js @@ -211,7 +211,7 @@ test('download headers (actual)', function (t) { process.release.name !== 'node' || semver.prerelease(process.version) !== null || semver.satisfies(process.version, '<10')) { - return t.skip('Skipping acutal download of headers due to test environment configuration') + return t.skip('Skipping actual download of headers due to test environment configuration') } t.plan(17) From bc509c511d47bf434c9612b545a113b3f253fb6b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 29 Dec 2019 08:43:40 +0100 Subject: [PATCH 09/16] test: add Windows to GitHub Actions testing (#1996) PR-URL: https://github.com/nodejs/node-gyp/pull/1996 Reviewed-By: Jiawen Geng --- .github/workflows/Python_tests.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Python_tests.yml b/.github/workflows/Python_tests.yml index 235127f21a..dba9d10933 100644 --- a/.github/workflows/Python_tests.yml +++ b/.github/workflows/Python_tests.yml @@ -1,4 +1,3 @@ -# TODO: Line 14, enable os: windows-latest # TODO: Line 15, enable python-version: 3.5 # TODO: Line 36, enable pytest --doctest-modules @@ -11,7 +10,7 @@ jobs: fail-fast: false max-parallel: 15 matrix: - os: [macos-latest, ubuntu-latest] # , windows-latest] + os: [macos-latest, ubuntu-latest, windows-latest] python-version: [2.7, 3.6, 3.7, 3.8] # 3.5, steps: - uses: actions/checkout@v1 @@ -30,7 +29,12 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest + - name: Test with pytest (Linux and macOS) + if: matrix.os != 'windows-latest' run: pytest + - name: Test with pytest (Windows) + if: matrix.os == 'windows-latest' + shell: bash + run: GYP_MSVS_VERSION=2015 GYP_MSVS_OVERRIDE_PATH="C:\\Dummy" pytest # - name: Run doctests with pytest # run: pytest --doctest-modules From cb3f6aae5e74f1ebfae0bb0fa8d40f1d9d7fcb6e Mon Sep 17 00:00:00 2001 From: James Home Date: Sun, 29 Dec 2019 01:47:45 -0600 Subject: [PATCH 10/16] doc: update macOS_Catalina.md (#1992) PR-URL: https://github.com/nodejs/node-gyp/pull/1992 Reviewed-By: Christian Clauss Reviewed-By: Jiawen Geng --- macOS_Catalina.md | 49 ++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/macOS_Catalina.md b/macOS_Catalina.md index 5f02face57..530cbf25c3 100644 --- a/macOS_Catalina.md +++ b/macOS_Catalina.md @@ -1,39 +1,37 @@ # Installation notes for macOS Catalina (v10.15) -_This document specifically refers to upgrades from previous versions of macOS to Catalina (10.15). It should be removed from the source repository when Catalina ceases to be the latest macOS version or updated to deal with challenges involved in upgrades to the next version of macOS._ +_This document specifically refers to upgrades from previous versions of macOS to Catalina (10.15). It should be removed from the source repository when Catalina ceases to be the latest macOS version or when future Catalina versions no longer raise these issues._ -Lessons learned from: -* https://github.com/nodejs/node-gyp/issues/1779 -* https://github.com/nodejs/node-gyp/issues/1861 -* https://github.com/nodejs/node-gyp/issues/1927 and elsewhere - -Installing `node-gyp` on macOS can be found at https://github.com/nodejs/node-gyp#on-macos - -However, upgrading to macOS Catalina changes some settings that may cause normal `node-gyp` installations to fail. +**Upgrading to macOS Catalina may cause normal `node-gyp` installations to fail.** ### Is my Mac running macOS Catalina? -Let's make first make sure that your Mac is currently running Catalina: -% `sw_vers` +Let's first make sure that your Mac is running Catalina: +``` +% sw_vers ProductName: Mac OS X ProductVersion: 10.15 BuildVersion: 19A602 +``` +If `ProductVersion` is less then `10.15` then this document is not for you. Normal install docs for `node-gyp` on macOS can be found at https://github.com/nodejs/node-gyp#on-macos -If `ProductVersion` is less then `10.15` then this document is not really for you. ### The acid test -Next, lets see if `Xcode Command Line Tools` are installed: +To see if `Xcode Command Line Tools` is installed in a way that will work with `node-gyp`, run: 1. `/usr/sbin/pkgutil --packages | grep CL` - * If nothing is listed, then [skip to the next section](#Two-roads). - * If `com.apple.pkg.CLTools_Executables` is listed then try: + * `com.apple.pkg.CLTools_Executables` should be listed. If it isn't, this test failed. 2. `/usr/sbin/pkgutil --pkg-info com.apple.pkg.CLTools_Executables` - * If `version: 11.0.0` or later is listed then _you are done_! Your Mac should be ready to install `node-gyp`. Doing `clang -v` should show `Apple clang version 11.0.0` or later. + * `version: 11.0.0` (or later) should be listed. If it isn't, this test failed. + +If both tests succeeded, _you are done_! You should be ready to install `node-gyp`. + +If either test failed, there is a problem with your Xcode Command Line Tools installation. [Continue to Solutions](#Solutions). -As you go through the remainder of this document, at anytime you can try these `acid test` commands. If they pass then your Mac should be ready to install `node-gyp`. +### Solutions +There are three ways to install the Xcode libraries `node-gyp` needs on macOS. People running Catalina have had success with some but not others in a way that has been unpredictable. -### Two roads -There are two main ways to install `node-gyp` on macOS: 1. With the full Xcode (~7.6 GB download) from the `App Store` app. 2. With the _much_ smaller Xcode Command Line Tools via `xcode-select --install` +3. With the _much_ smaller Xcode Command Line Tools via manual download. **For people running the latest version of Catalina (10.15.2 at the time of this writing), this has worked when the other two solutions haven't.** ### Installing `node-gyp` using the full Xcode 1. `xcodebuild -version` should show `Xcode 11.1` or later. @@ -52,7 +50,7 @@ There are two main ways to install `node-gyp` on macOS: 10. `sudo xcode-select --reset` # Enter root password. No output is normal. 11. Repeat step 7 above. Is the path different this time? Repeat the _acid test_. -### Installing `node-gyp` using the Xcode Command Line Tools +### Installing `node-gyp` using the Xcode Command Line Tools via `xcode-select --install` 1. If the _acid test_ has not succeeded, then try `xcode-select --install` 2. Wait until the install process is _complete_. 3. `softwareupdate -l` # No listing is a good sign. @@ -64,6 +62,11 @@ There are two main ways to install `node-gyp` on macOS: 8. `sudo xcode-select --reset` # Enter root password. No output is normal. 9. Repeat step 5 above. Is the path different this time? Repeat the _acid test_. +### Installing `node-gyp` using the Xcode Command Line Tools via manual download +1. Download the appropriate version of the "Command Line Tools for Xcode" for your version of Catalina from developer.apple.com/download. As of MacOS 10.15.2, that's Command_Line_Tools_for_Xcode_11.3.dmg +2. Install the package. +3. Run the _acid test_. + ### I did all that and the acid test still does not pass :-( 1. `sudo rm -rf $(xcode-select -print-path)` # Enter root password. No output is normal. 2. `xcode-select --install` @@ -72,3 +75,9 @@ There are two main ways to install `node-gyp` on macOS: 5. `npm explore npm -g -- npm explore npm-lifecycle -- npm install node-gyp@latest` 6. If the _acid test_ still does _not_ pass then... 7. Add a comment to https://github.com/nodejs/node-gyp/issues/1927 so we can improve. + +Lessons learned from: +* https://github.com/nodejs/node-gyp/issues/1779 +* https://github.com/nodejs/node-gyp/issues/1861 +* https://github.com/nodejs/node-gyp/issues/1927 and elsewhere +* Thanks to @rrrix for discovering Solution 3 From 52365819c726520654ef914cead0c7bb686c8820 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sun, 29 Dec 2019 18:49:13 +1100 Subject: [PATCH 11/16] test: remove old docker test harness (#1993) PR-URL: https://github.com/nodejs/node-gyp/pull/1993 Reviewed-By: Richard Lau Reviewed-By: Jiawen Geng --- test/docker.sh | 112 ------------------------------------------------- 1 file changed, 112 deletions(-) delete mode 100755 test/docker.sh diff --git a/test/docker.sh b/test/docker.sh deleted file mode 100755 index 67014209d0..0000000000 --- a/test/docker.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -#set -e - -test_node_versions="6.17.0 8.15.1 10.15.3 11.12.0" - -myuid=$(id -u) -mygid=$(id -g) -__dirname="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -dot_node_gyp=${__dirname}/.node-gyp/ - -# borrows from https://github.com/rvagg/dnt/ - -# Simple setup function for a container: -# setup_container(image id, base image, commands to run to set up) -setup_container() { - local container_id="$1" - local base_container="$2" - local run_cmd="$3" - - # Does this image exist? If yes, ignore - docker inspect "$container_id" &> /dev/null - if [[ $? -eq 0 ]]; then - echo "Found existing container [$container_id]" - else - # No such image, so make it - echo "Did not find container [$container_id], creating..." - docker run -i $base_container /bin/bash -c "$run_cmd" - sleep 2 - docker commit $(docker ps -l -q) $container_id - fi -} - -# Run tests inside each of the versioned containers, copy cwd into npm's copy of node-gyp -# so it'll be invoked by npm when a compile is needed -# run_tests(version, test-commands) -run_tests() { - local version="$1" - local run_cmd="$2" - - run_cmd="rsync -aAXx --delete --exclude .git --exclude build /node-gyp-src/ /usr/lib/node_modules/npm/node_modules/node-gyp/; - /bin/su -s /bin/bash node-gyp -c 'cd && ${run_cmd}'" - - rm -rf $dot_node_gyp - mkdir $dot_node_gyp - - docker run \ - --rm -i \ - -v ~/.npm/:/node-gyp/.npm/ \ - -v ${dot_node_gyp}:/node-gyp/.node-gyp/ \ - -v $(pwd):/node-gyp-src/:ro \ - node-gyp-test/${version} /bin/bash -c "${run_cmd}" -} - -# A base image with build tools and a user account -setup_container "node-gyp-test/base" "ubuntu:14.04" " - adduser --gecos node-gyp --home /node-gyp/ --disabled-login node-gyp --uid $myuid && - echo "node-gyp:node-gyp" | chpasswd && - apt-get update && - apt-get install -y build-essential python git rsync curl -" - -# An image on top of the base containing clones of repos we want to use for testing -setup_container "node-gyp-test/clones" "node-gyp-test/base" " - cd /node-gyp/ && git clone https://github.com/justmoon/node-bignum.git && - cd /node-gyp/ && git clone https://github.com/bnoordhuis/node-buffertools.git && - chown -R node-gyp.node-gyp /node-gyp/ -" - -# An image for each of the node versions we want to test with that version installed and the latest npm -for v in $test_node_versions; do - setup_container "node-gyp-test/${v}" "node-gyp-test/clones" " - curl -sL https://nodejs.org/dist/v${v}/node-v${v}-linux-x64.tar.gz | tar -zxv --strip-components=1 -C /usr/ && - npm install npm@latest -g && - node -v && npm -v - " -done - -# Test use of --target=x.y.z to compile against alternate versions -test_download_node_version() { - local run_with_ver="$1" - local expected_dir="$2" - local expected_ver="$3" - run_tests $run_with_ver "cd node-buffertools && npm install --loglevel=info --target=${expected_ver}" - local node_ver=$(cat "${dot_node_gyp}${expected_dir}/node_version.h" | grep '#define NODE_\w*_VERSION [0-9]*$') - node_ver=$(echo $node_ver | sed 's/#define NODE_[A-Z]*_VERSION //g' | sed 's/ /./g') - if [ "X$(echo $node_ver)" != "X${expected_ver}" ]; then - echo "Did not download v${expected_ver} using --target, instead got: $(echo $node_ver)" - exit 1 - fi - echo "Verified correct download of [v${node_ver}]" -} - -test_download_node_version "0.12.7" "0.10.30/src" "0.10.30" -# should download the headers file -test_download_node_version "4.3.0" "4.3.0/include/node" "4.3.0" -test_download_node_version "5.6.0" "5.6.0/include/node" "5.6.0" - -# TODO: test --dist-url by starting up a localhost server and serving up tarballs - -# testing --dist-url, using simple-proxy.js to make localhost work as a distribution -# point for tarballs -# we can test whether it uses the proxy because after 2 connections the proxy will -# die and therefore should not be running at the end of the test, `nc` can tell us this -run_tests "0.12.7" " - (node /node-gyp-src/test/simple-proxy.js 8080 /boombar/ https://nodejs.org/dist/ &) && - cd node-buffertools && - NODEJS_ORG_MIRROR=http://localhost:8080/boombar/ /node-gyp-src/bin/node-gyp.js --loglevel=info rebuild && - nc -z localhost 8080 && echo -e \"\\n\\n\\033[31mFAILED TO USE LOCAL PROXY\\033[39m\\n\\n\" -" - -rm -rf $dot_node_gyp From f37a8b40d09e6beeccdf0905941c8e45407f6a37 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sun, 29 Dec 2019 18:50:15 +1100 Subject: [PATCH 12/16] doc: add GitHub Actions badge (#1994) PR-URL: https://github.com/nodejs/node-gyp/pull/1994 Reviewed-By: Richard Lau Reviewed-By: Christian Clauss Reviewed-By: Jiawen Geng --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 81d90a650b..79abf4ba71 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # `node-gyp` - Node.js native addon build tool -[![Build Status](https://travis-ci.com/nodejs/node-gyp.svg?branch=master)](https://travis-ci.com/nodejs/node-gyp) +[![Travis CI](https://travis-ci.com/nodejs/node-gyp.svg?branch=master)](https://travis-ci.com/nodejs/node-gyp) +[![Build Status](https://github.com/nodejs/node-gyp/workflows/Python_tests/badge.svg)](https://github.com/nodejs/node-gyp/actions?workflow=Python_tests) `node-gyp` is a cross-platform command-line tool written in Node.js for compiling native addon modules for Node.js. It contains a fork of the From 103740cd957a7d875a12dcfbe072421751ca2b9f Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 31 Dec 2019 21:39:18 +0100 Subject: [PATCH 13/16] gyp: list(dict) so we can del dict(key) while iterating Fixes #1998 Reviewed-By: Rod Vagg PR-URL: https://github.com/nodejs/node-gyp/pull/2009 --- gyp/pylib/gyp/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gyp/pylib/gyp/input.py b/gyp/pylib/gyp/input.py index 2973c078fc..d1742800ac 100644 --- a/gyp/pylib/gyp/input.py +++ b/gyp/pylib/gyp/input.py @@ -2286,7 +2286,7 @@ def SetUpConfigurations(target, target_dict): merged_configurations[configuration]) # Now drop all the abstract ones. - for configuration in target_dict['configurations'].keys(): + for configuration in list(target_dict['configurations']): old_configuration_dict = target_dict['configurations'][configuration] if old_configuration_dict.get('abstract'): del target_dict['configurations'][configuration] From 1694907bbf51d9ca0343897bd7ac961d76acbace Mon Sep 17 00:00:00 2001 From: Xavier Guimard Date: Sun, 29 Dec 2019 09:15:08 +0100 Subject: [PATCH 14/16] =?UTF-8?q?lib:=20compatibility=20with=20semver=20?= =?UTF-8?q?=E2=89=A5=207=20(`new`=20for=20semver.Range)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: #2005 Reviewed-By: Rod Vagg PR-URL: https://github.com/nodejs/node-gyp/pull/2006 --- lib/find-python.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/find-python.js b/lib/find-python.js index e79bc50d5a..fabc4630f1 100644 --- a/lib/find-python.js +++ b/lib/find-python.js @@ -226,7 +226,7 @@ PythonFinder.prototype = { } this.addLog(`- version is "${version}"`) - const range = semver.Range(this.semverRange) + const range = new semver.Range(this.semverRange) var valid = false try { valid = range.test(version) From 278dcddbddee25761d7a010b3b5e9f10e21af707 Mon Sep 17 00:00:00 2001 From: Andrew Casey Date: Wed, 8 Jan 2020 13:49:39 -0800 Subject: [PATCH 15/16] lib: ignore VS instances that cause COMExceptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I have quite a few instances of VS installed and it looks like Find-VisualStudio.cs enumerates all of them, even when find-visualstudio.js already knows which one it wants (from the environment variable in the developer command prompt). One of them (from 15.7.2, if that's interesting) causes a COMException on the ISetupInstance2.GetPackages call. Ignoring such packages seems harmless and unblocks the rest of the run. PR-URL: https://github.com/nodejs/node-gyp/pull/2018 Reviewed-By: João Reis --- lib/Find-VisualStudio.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/Find-VisualStudio.cs b/lib/Find-VisualStudio.cs index 0a533f42d6..d2e45a7627 100644 --- a/lib/Find-VisualStudio.cs +++ b/lib/Find-VisualStudio.cs @@ -205,7 +205,14 @@ public static void PrintJson() return; } - instances.Add(InstanceJson(rgelt[0])); + try + { + instances.Add(InstanceJson(rgelt[0])); + } + catch (COMException) + { + // Ignore instances that can't be queried. + } } } From 85684e5868eecf21487ef74ee235ada85c6aeac0 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 3 Feb 2020 14:48:20 +1100 Subject: [PATCH 16/16] v5.1.0: bump version and update changelog PR-URL: https://github.com/nodejs/node-gyp/pull/2012 --- CHANGELOG.md | 18 ++++++++++++++++++ package.json | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28f5237798..35cade3d82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +v5.1.0 2020-02-03 +================= + +* [[`f37a8b40d0`](https://github.com/nodejs/node-gyp/commit/f37a8b40d0)] - **doc**: add GitHub Actions badge (#1994) (Rod Vagg) [#1994](https://github.com/nodejs/node-gyp/pull/1994) +* [[`cb3f6aae5e`](https://github.com/nodejs/node-gyp/commit/cb3f6aae5e)] - **doc**: update macOS\_Catalina.md (#1992) (James Home) [#1992](https://github.com/nodejs/node-gyp/pull/1992) +* [[`0607596a4c`](https://github.com/nodejs/node-gyp/commit/0607596a4c)] - **doc**: fix typo in README.md (#1985) (Suraneti Rodsuwan) [#1985](https://github.com/nodejs/node-gyp/pull/1985) +* [[`0d5a415a14`](https://github.com/nodejs/node-gyp/commit/0d5a415a14)] - **doc**: add travis badge (Rod Vagg) [#1971](https://github.com/nodejs/node-gyp/pull/1971) +* [[`103740cd95`](https://github.com/nodejs/node-gyp/commit/103740cd95)] - **gyp**: list(dict) so we can del dict(key) while iterating (Christian Clauss) [#2009](https://github.com/nodejs/node-gyp/pull/2009) +* [[`278dcddbdd`](https://github.com/nodejs/node-gyp/commit/278dcddbdd)] - **lib**: ignore VS instances that cause COMExceptions (Andrew Casey) [#2018](https://github.com/nodejs/node-gyp/pull/2018) +* [[`1694907bbf`](https://github.com/nodejs/node-gyp/commit/1694907bbf)] - **lib**: compatibility with semver ≥ 7 (`new` for semver.Range) (Xavier Guimard) [#2006](https://github.com/nodejs/node-gyp/pull/2006) +* [[`a3f1143514`](https://github.com/nodejs/node-gyp/commit/a3f1143514)] - **(SEMVER-MINOR)** **lib**: noproxy support, match proxy detection to `request` (Matias Lopez) [#1978](https://github.com/nodejs/node-gyp/pull/1978) +* [[`52365819c7`](https://github.com/nodejs/node-gyp/commit/52365819c7)] - **test**: remove old docker test harness (#1993) (Rod Vagg) [#1993](https://github.com/nodejs/node-gyp/pull/1993) +* [[`bc509c511d`](https://github.com/nodejs/node-gyp/commit/bc509c511d)] - **test**: add Windows to GitHub Actions testing (#1996) (Christian Clauss) [#1996](https://github.com/nodejs/node-gyp/pull/1996) +* [[`91ee26dd48`](https://github.com/nodejs/node-gyp/commit/91ee26dd48)] - **test**: fix typo in header download test (#2001) (Richard Lau) [#2001](https://github.com/nodejs/node-gyp/pull/2001) +* [[`0923f344c9`](https://github.com/nodejs/node-gyp/commit/0923f344c9)] - **test**: direct python invocation & simpler pyenv (Matias Lopez) [#1979](https://github.com/nodejs/node-gyp/pull/1979) +* [[`32c8744b34`](https://github.com/nodejs/node-gyp/commit/32c8744b34)] - **test**: fix macOS Travis on Python 2.7 & 3.7 (Christian Clauss) [#1979](https://github.com/nodejs/node-gyp/pull/1979) +* [[`fd4b1351e4`](https://github.com/nodejs/node-gyp/commit/fd4b1351e4)] - **test**: initial Github Actions with Ubuntu & macOS (Christian Clauss) [#1985](https://github.com/nodejs/node-gyp/pull/1985) + v5.0.7 2019-12-16 ================= diff --git a/package.json b/package.json index ebd8fa697e..29cfc11cc0 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "bindings", "gyp" ], - "version": "5.0.7", + "version": "5.1.0", "installVersion": 9, "author": "Nathan Rajlich (http://tootallnate.net)", "repository": {