From d9c23d9edcc4e65ea70fdff171df9d08fb7b5203 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Thu, 6 Jun 2024 23:07:39 +0800 Subject: [PATCH 1/3] feat(gyp): update gyp to v0.18.1 --- gyp/.github/dependabot.yml | 20 + gyp/.github/workflows/Python_tests.yml | 11 +- gyp/.github/workflows/node-gyp.yml | 14 +- gyp/.github/workflows/release-please.yml | 93 +- gyp/.release-please-manifest.json | 3 + gyp/CHANGELOG.md | 41 + gyp/CONTRIBUTING.md | 4 + gyp/README.md | 2 +- gyp/data/ninja/build.ninja | 4 + gyp/docs/GypVsCMake.md | 116 ++ gyp/docs/Hacking.md | 46 + gyp/docs/InputFormatReference.md | 1080 +++++++++++++++++ gyp/docs/LanguageSpecification.md | 430 +++++++ gyp/docs/README.md | 27 + gyp/docs/Testing.md | 450 +++++++ gyp/docs/UserDocumentation.md | 965 +++++++++++++++ gyp/pylib/gyp/MSVSSettings.py | 2 + gyp/pylib/gyp/common.py | 63 +- gyp/pylib/gyp/common_test.py | 103 +- gyp/pylib/gyp/generator/android.py | 4 +- .../gyp/generator/compile_commands_json.py | 10 +- gyp/pylib/gyp/generator/gypsh.py | 2 +- gyp/pylib/gyp/generator/make.py | 80 +- gyp/pylib/gyp/generator/msvs.py | 9 +- gyp/pylib/gyp/generator/ninja.py | 31 + gyp/pylib/gyp/generator/ninja_test.py | 12 + gyp/pylib/gyp/input.py | 10 +- gyp/pylib/gyp/msvs_emulation.py | 5 +- gyp/pylib/gyp/xcode_emulation.py | 29 +- gyp/pylib/gyp/xcode_emulation_test.py | 53 + gyp/pyproject.toml | 19 +- gyp/release-please-config.json | 11 + 32 files changed, 3663 insertions(+), 86 deletions(-) create mode 100644 gyp/.github/dependabot.yml create mode 100644 gyp/.release-please-manifest.json create mode 100644 gyp/data/ninja/build.ninja create mode 100644 gyp/docs/GypVsCMake.md create mode 100644 gyp/docs/Hacking.md create mode 100644 gyp/docs/InputFormatReference.md create mode 100644 gyp/docs/LanguageSpecification.md create mode 100644 gyp/docs/README.md create mode 100644 gyp/docs/Testing.md create mode 100644 gyp/docs/UserDocumentation.md create mode 100644 gyp/pylib/gyp/xcode_emulation_test.py create mode 100644 gyp/release-please-config.json diff --git a/gyp/.github/dependabot.yml b/gyp/.github/dependabot.yml new file mode 100644 index 0000000000..58d68276fc --- /dev/null +++ b/gyp/.github/dependabot.yml @@ -0,0 +1,20 @@ +# Keep GitHub Actions up to date with Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + groups: + GitHub_Actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly + - package-ecosystem: "pip" + directory: "/" + groups: + pip: + patterns: + - "*" # Group all pip updates into a single larger pull request + schedule: + interval: weekly diff --git a/gyp/.github/workflows/Python_tests.yml b/gyp/.github/workflows/Python_tests.yml index 049d5fe50c..5c86466378 100644 --- a/gyp/.github/workflows/Python_tests.yml +++ b/gyp/.github/workflows/Python_tests.yml @@ -15,15 +15,16 @@ jobs: fail-fast: false max-parallel: 5 matrix: - os: [macos-latest, ubuntu-latest] # , windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: [macos-13, macos-14, ubuntu-latest] # , windows-latest] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true + - uses: seanmiddleditch/gha-setup-ninja@v4 - name: Install dependencies run: | python -m pip install --upgrade pip setuptools @@ -35,3 +36,7 @@ jobs: run: pytest # - name: Run doctests with pytest # run: pytest --doctest-modules + - name: Test CLI commands on a pipx install + run: | + pipx run --no-cache --spec ./ gyp --help + pipx run --no-cache --spec ./ gyp --version diff --git a/gyp/.github/workflows/node-gyp.yml b/gyp/.github/workflows/node-gyp.yml index ebe7497521..b960fcf2b7 100644 --- a/gyp/.github/workflows/node-gyp.yml +++ b/gyp/.github/workflows/node-gyp.yml @@ -10,9 +10,9 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python: ["3.8", "3.10", "3.12"] - + node-version: ["22"] + os: [macos-13, macos-14, ubuntu-latest, windows-latest] + python-version: ["3.8", "3.10", "3.12", "3.13"] runs-on: ${{ matrix.os }} steps: - name: Clone gyp-next @@ -24,12 +24,12 @@ jobs: with: repository: nodejs/node-gyp path: node-gyp - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: - node-version: 18.x - - uses: actions/setup-python@v4 + node-version: ${{ matrix.node-version }} + - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python }} + python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Install Python dependencies run: | diff --git a/gyp/.github/workflows/release-please.yml b/gyp/.github/workflows/release-please.yml index 665c4c48fe..95dadd53bb 100644 --- a/gyp/.github/workflows/release-please.yml +++ b/gyp/.github/workflows/release-please.yml @@ -7,10 +7,91 @@ name: release-please jobs: release-please: runs-on: ubuntu-latest + outputs: + release_created: ${{ steps.release.outputs.release_created }} + tag_name: ${{ steps.release.outputs.tag_name }} + permissions: + contents: write + pull-requests: write steps: - - uses: google-github-actions/release-please-action@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - release-type: python - package-name: gyp-next - bump-minor-pre-major: true + - uses: google-github-actions/release-please-action@v4 + id: release + + build: + name: Build distribution + needs: + - release-please + if: ${{ needs.release-please.outputs.release_created }} # only publish on release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install pypa/build + run: >- + python3 -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python distribution to PyPI + needs: + - release-please + - build + if: ${{ needs.release-please.outputs.release_created }} # only publish on release + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/gyp-next + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Publish Python distribution to GitHub Release + needs: + - release-please + - build + if: ${{ needs.release-please.outputs.release_created }} # only publish on release + runs-on: ubuntu-latest + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v2.1.1 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + ${{ needs.release-please.outputs.tag_name }} dist/** + --repo '${{ github.repository }}' diff --git a/gyp/.release-please-manifest.json b/gyp/.release-please-manifest.json new file mode 100644 index 0000000000..cbd0ca0683 --- /dev/null +++ b/gyp/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.18.1" +} diff --git a/gyp/CHANGELOG.md b/gyp/CHANGELOG.md index 483943e013..8a8282a5f8 100644 --- a/gyp/CHANGELOG.md +++ b/gyp/CHANGELOG.md @@ -1,5 +1,46 @@ # Changelog +## [0.18.1](https://github.com/nodejs/gyp-next/compare/v0.18.0...v0.18.1) (2024-05-26) + + +### Bug Fixes + +* **ci:** add Python 3.13 pre-release to test matrix ([#257](https://github.com/nodejs/gyp-next/issues/257)) ([8597203](https://github.com/nodejs/gyp-next/commit/8597203b687325c7516367135e026586279d0583)) + + +### Documentation + +* vendor docs from gyp.gsrc.io ([#254](https://github.com/nodejs/gyp-next/issues/254)) ([8d7ba6e](https://github.com/nodejs/gyp-next/commit/8d7ba6e784dedf1122a0456150c739d2a09ecf57)) + +## [0.18.0](https://github.com/nodejs/gyp-next/compare/v0.17.0...v0.18.0) (2024-05-08) + + +### Features + +* support language standard keys in msvs_settings ([#252](https://github.com/nodejs/gyp-next/issues/252)) ([322f6d5](https://github.com/nodejs/gyp-next/commit/322f6d5d5233967522f3e55c623a8e7d7281e024)) + +## [0.17.0](https://github.com/nodejs/gyp-next/compare/v0.16.2...v0.17.0) (2024-04-29) + + +### Features + +* generate compile_commands.json with ninja ([#228](https://github.com/nodejs/gyp-next/issues/228)) ([7b20b46](https://github.com/nodejs/gyp-next/commit/7b20b4673d8cf46ff61898eb19569007d55c854a)) + + +### Bug Fixes + +* failed to detect flavor if compiler path include white spaces ([#240](https://github.com/nodejs/gyp-next/issues/240)) ([f3b9753](https://github.com/nodejs/gyp-next/commit/f3b9753e7526377020e7d40e66b624db771cf84a)) +* support cross compiling for wasm with make generator ([#222](https://github.com/nodejs/gyp-next/issues/222)) ([de0e1c9](https://github.com/nodejs/gyp-next/commit/de0e1c9a5791d1bf4bc3103f878ab74814864ab4)) +* support empty dictionary keys in input ([#245](https://github.com/nodejs/gyp-next/issues/245)) ([178459f](https://github.com/nodejs/gyp-next/commit/178459ff343a2771d5f30f04467d2f032d6b3565)) +* update Ruff to 0.3.1 ([876ccaf](https://github.com/nodejs/gyp-next/commit/876ccaf5629e1b95e13aaa2b0eb6cbd08fa80593)) + +## [0.16.2](https://github.com/nodejs/gyp-next/compare/v0.16.1...v0.16.2) (2024-03-07) + + +### Bug Fixes + +* avoid quoting cflag name and parameter with space separator ([#223](https://github.com/nodejs/gyp-next/issues/223)) ([2b9703d](https://github.com/nodejs/gyp-next/commit/2b9703dbd5b3b8a935faf257c6103033b47bf8bf)) + ## [0.16.1](https://github.com/nodejs/gyp-next/compare/v0.16.0...v0.16.1) (2023-10-25) diff --git a/gyp/CONTRIBUTING.md b/gyp/CONTRIBUTING.md index 1a0bcde2b4..5237b216fb 100644 --- a/gyp/CONTRIBUTING.md +++ b/gyp/CONTRIBUTING.md @@ -1,5 +1,9 @@ # Contributing to gyp-next +## Start contributing + +Read the docs at [`./docs/Hacking.md`](./docs/Hacking.md) to get started. + ## Code of Conduct This project is bound to the [Node.js Code of Conduct](https://github.com/nodejs/admin/blob/HEAD/CODE_OF_CONDUCT.md). diff --git a/gyp/README.md b/gyp/README.md index be1d7b9ebf..38792e1de4 100644 --- a/gyp/README.md +++ b/gyp/README.md @@ -1,7 +1,7 @@ GYP can Generate Your Projects. =================================== -Documents are available at [gyp.gsrc.io](https://gyp.gsrc.io), or you can check out ```md-pages``` branch to read those documents offline. +Documents are available at [`./docs`](./docs). __gyp-next__ is [released](https://github.com/nodejs/gyp-next/releases) to the [__Python Packaging Index__](https://pypi.org/project/gyp-next) (PyPI) and can be installed with the command: * `python3 -m pip install gyp-next` diff --git a/gyp/data/ninja/build.ninja b/gyp/data/ninja/build.ninja new file mode 100644 index 0000000000..2400dbb1f0 --- /dev/null +++ b/gyp/data/ninja/build.ninja @@ -0,0 +1,4 @@ +rule cc + command = cc $in $out + +build my.out: cc my.in diff --git a/gyp/docs/GypVsCMake.md b/gyp/docs/GypVsCMake.md new file mode 100644 index 0000000000..6d659a6123 --- /dev/null +++ b/gyp/docs/GypVsCMake.md @@ -0,0 +1,116 @@ +# vs. CMake + +GYP was originally created to generate native IDE project files (Visual Studio, Xcode) for building [Chromium](http://www.chromim.org). + +The functionality of GYP is very similar to the [CMake](http://www.cmake.org) +build tool. Bradley Nelson wrote up the following description of why the team +created GYP instead of using CMake. The text below is copied from +http://www.mail-archive.com/webkit-dev@lists.webkit.org/msg11029.html + +``` + +Re: [webkit-dev] CMake as a build system? +Bradley Nelson +Mon, 19 Apr 2010 22:38:30 -0700 + +Here's the innards of an email with a laundry list of stuff I came up with a +while back on the gyp-developers list in response to Mike Craddick regarding +what motivated gyp's development, since we were aware of cmake at the time +(we'd even started a speculative port): + + +I did an exploratory port of portions of Chromium to cmake (I think I got as +far as net, base, sandbox, and part of webkit). +There were a number of motivations, not all of which would apply to other +projects. Also, some of the design of gyp was informed by experience at +Google with large projects built wholly from source, leading to features +absent from cmake, but not strictly required for Chromium. + +1. Ability to incrementally transition on Windows. It took us about 6 months +to switch fully to gyp. Previous attempts to move to scons had taken a long +time and failed, due to the requirement to transition while in flight. For a +substantial period of time, we had a hybrid of checked in vcproj and gyp generated +vcproj. To this day we still have a good number of GUIDs pinned in the gyp files, +because different parts of our release pipeline have leftover assumptions +regarding manipulating the raw sln/vcprojs. This transition occurred from +the bottom up, largely because modules like base were easier to convert, and +had a lower churn rate. During early stages of the transition, the majority +of the team wasn't even aware they were using gyp, as it integrated into +their existing workflow, and only affected modules that had been converted. + +2. Generation of a more 'normal' vcproj file. Gyp attempts, particularly on +Windows, to generate vcprojs which resemble hand generated projects. It +doesn't generate any Makefile type projects, but instead produces msvs +Custom Build Steps and Custom Build Rules. This makes the resulting projects +easier to understand from the IDE and avoids parts of the IDE that simply +don't function correctly if you use Makefile projects. Our early hope with +gyp was to support the least common denominator of features present in each +of the platform specific project file formats, rather than falling back on +generated Makefiles/shell scripts to emulate some common abstraction. CMake by +comparison makes a good faith attempt to use native project features, but +falls back on generated scripts in order to preserve the same semantics on +each platforms. + +3. Abstraction on the level of project settings, rather than command line +flags. In gyp's syntax you can add nearly any option present in a hand +generated xcode/vcproj file. This allows you to use abstractions built into +the IDEs rather than reverse engineering them possibly incorrectly for +things like: manifest generation, precompiled headers, bundle generation. +When somebody wants to use a particular menu option from msvs, I'm able to +do a web search on the name of the setting from the IDE and provide them +with a gyp stanza that does the equivalent. In many cases, not all project +file constructs correspond to command line flags. + +4. Strong notion of module public/private interface. Gyp allows targets to +publish a set of direct_dependent_settings, specifying things like +include_dirs, defines, platforms specific settings, etc. This means that +when module A depends on module B, it automatically acquires the right build +settings without module A being filled with assumptions/knowledge of exactly +how module B is built. Additionally, all of the transitive dependencies of +module B are pulled in. This avoids their being a single top level view of +the project, rather each gyp file expresses knowledge about its immediate +neighbors. This keep local knowledge local. CMake effectively has a large +shared global namespace. + +5. Cross platform generation. CMake is not able to generate all project +files on all platforms. For example xcode projects cannot be generated from +windows (cmake uses mac specific libraries to do project generation). This +means that for instance generating a tarball containing pregenerated +projects for all platforms is hard with Cmake (requires distribution to +several machine types). + +6. Gyp has rudimentary cross compile support. Currently we've added enough +functionality to gyp to support x86 -> arm cross compiles. Last I checked +this functionality wasn't present in cmake. (This occurred later). + + +That being said there are a number of drawbacks currently to gyp: + +1. Because platform specific settings are expressed at the project file +level (rather than the command line level). Settings which might otherwise +be shared in common between platforms (flags to gcc on mac/linux), end up +being repeated twice. Though in fairness there is actually less sharing here +than you'd think. include_dirs and defines actually represent 90% of what +can be typically shared. + +2. CMake may be more mature, having been applied to a broader range of +projects. There a number of 'tool modules' for cmake, which are shared in a +common community. + +3. gyp currently makes some nasty assumptions about the availability of +chromium's hermetic copy of cygwin on windows. This causes you to either +have to special case a number of rules, or swallow this copy of cygwin as a +build time dependency. + +4. CMake includes a fairly readable imperative language. Currently Gyp has a +somewhat poorly specified declarative language (variable expansion happens +in sometimes weird and counter-intuitive ways). In fairness though, gyp assumes +that external python scripts can be used as an escape hatch. Also gyp avoids +a lot of the things you'd need imperative code for, by having a nice target +settings publication mechanism. + +5. (Feature/drawback depending on personal preference). Gyp's syntax is +DEEPLY nested. It suffers from all of Lisp's advantages and drawbacks. + +-BradN +``` diff --git a/gyp/docs/Hacking.md b/gyp/docs/Hacking.md new file mode 100644 index 0000000000..89b3b8bea9 --- /dev/null +++ b/gyp/docs/Hacking.md @@ -0,0 +1,46 @@ +# Hacking + +## Getting the sources + +Git is required to hack on anything, you can set up a git clone of GYP +as follows: + +``` +mkdir foo +cd foo +git clone git@github.com:nodejs/gyp-next.git +cd gyp +``` + +(this will clone gyp underneath it into `foo/gyp`. +`foo` can be any directory name you want. Once you've done that, +you can use the repo like anything other Git repo. + +## Testing your change + +GYP has a suite of tests which you can run with the provided test driver +to make sure your changes aren't breaking anything important. + +You run the test driver with e.g. + +``` sh +$ python -m pip install --upgrade pip setuptools +$ pip install --editable ".[dev]" +$ python -m pytest +``` + +See [Testing](Testing.md) for more details on the test framework. + +Note that it can be handy to look at the project files output by the tests +to diagnose problems. The easiest way to do that is by kindly asking the +test driver to leave the temporary directories it creates in-place. +This is done by setting the enviroment variable "PRESERVE", e.g. + +``` +set PRESERVE=all # On Windows +export PRESERVE=all # On saner platforms. +``` + +## Reviewing your change + +All changes to GYP must be code reviewed before submission. diff --git a/gyp/docs/InputFormatReference.md b/gyp/docs/InputFormatReference.md new file mode 100644 index 0000000000..2b2c180f44 --- /dev/null +++ b/gyp/docs/InputFormatReference.md @@ -0,0 +1,1080 @@ +# Input Format Reference + +## Primitive Types + +The following primitive types are found within input files: + + * String values, which may be represented by enclosing them in + `'single quotes'` or `"double quotes"`. By convention, single + quotes are used. + * Integer values, which are represented in decimal without any special + decoration. Integers are fairly rare in input files, but have a few + applications in boolean contexts, where the convention is to + represent true values with `1` and false with `0`. + * Lists, which are represented as a sequence of items separated by + commas (`,`) within square brackets (`[` and `]`). A list may + contain any other primitive types, including other lists. + Generally, each item of a list must be of the same type as all other + items in the list, but in some cases (such as within `conditions` + sections), the list structure is more tightly specified. A trailing + comma is permitted. + + This example list contains three string values. + + ``` + [ 'Generate', 'Your', 'Projects', ] + ``` + + * Dictionaries, which map keys to values. All keys are strings. + Values may be of any other primitive type, including other + dictionaries. A dictionary is enclosed within curly braces (`{` and + `}`). Keys precede values, separated by a colon (`:`). Successive + dictionary entries are separated by commas (`,`). A trailing comma + is permitted. It is an error for keys to be duplicated within a + single dictionary as written in an input file, although keys may + replace other keys during [merging](#Merging). + + This example dictionary maps each of three keys to different values. + + ``` + { + 'inputs': ['version.c.in'], + 'outputs': ['version.c'], + 'process_outputs_as_sources': 1, + } + ``` + +## Overall Structure + +A GYP input file is organized as structured data. At the root scope of +each `.gyp` or `.gypi` (include) file is a dictionary. The keys and +values of this dictionary, along with any descendants contained within +the values, provide the data contained within the file. This data is +given meaning by interpreting specific key names and their associated +values in specific ways (see [Settings Keys](#Settings_Keys)). + +### Comments (#) + +Within an input file, a comment is introduced by a pound sign (`#`) not +within a string. Any text following the pound sign, up until the end of +the line, is treated as a comment. + +#### Example + +``` +{ + 'school_supplies': [ + 'Marble composition book', + 'Sharp #2 pencil', + 'Safety scissors', # You still shouldn't run with these + ], +} +``` + +In this example, the # in `'Sharp #2 pencil'` is not taken as +introducing a comment because it occurs within a string, but the text +after `'Safety scissors'` is treated as a comment having no impact on +the data within the file. + +## Merging + +### Merge Basics (=, ?, +) + +Many operations on GYP input files occurs by merging dictionary and list +items together. During merge operations, it is important to recognize +the distinction between source and destination values. Items from the +source value are merged into the destination, which leaves the source +unchanged and the destination modified by the source. A dictionary may +only be merged into another dictionary, and a list may only be merged +into another list. + + * When merging a dictionary, for each key in the source: + * If the key does not exist in the destination dictionary, insert it + and copy the associated value directly. + * If the key does exist: + * If the associated value is a dictionary, perform the dictionary + merging procedure using the source's and destination's value + dictionaries. + * If the associated value is a list, perform the list merging + procedure using the source's and destination's value lists. + * If the associated value is a string or integer, the destination + value is replaced by the source value. + * When merging a list, merge according to the suffix appended to the + key name, if the list is a value within a dictionary. + * If the key ends with an equals sign (`=`), the policy is for the + source list to completely replace the destination list if it + exists. _Mnemonic: `=` for assignment._ + * If the key ends with a question mark (`?`), the policy is for the + source list to be set as the destination list only if the key is + not already present in the destination. _Mnemonic: `?` for + conditional assignment_. + * If the key ends with a plus sign (`+`), the policy is for the + source list contents to be prepended to the destination list. + _Mnemonic: `+` for addition or concatenation._ + * If the list key is undecorated, the policy is for the source list + contents to be appended to the destination list. This is the + default list merge policy. + +#### Example + +Source dictionary: + +``` +{ + 'include_dirs+': [ + 'shared_stuff/public', + ], + 'link_settings': { + 'libraries': [ + '-lshared_stuff', + ], + }, + 'test': 1, +} +``` + +Destination dictionary: + +``` +{ + 'target_name': 'hello', + 'sources': [ + 'kitty.cc', + ], + 'include_dirs': [ + 'headers', + ], + 'link_settings': { + 'libraries': [ + '-lm', + ], + 'library_dirs': [ + '/usr/lib', + ], + }, + 'test': 0, +} +``` + +Merged dictionary: + +``` +{ + 'target_name': 'hello', + 'sources': [ + 'kitty.cc', + ], + 'include_dirs': [ + 'shared_stuff/public', # Merged, list item prepended due to include_dirs+ + 'headers', + ], + 'link_settings': { + 'libraries': [ + '-lm', + '-lshared_stuff', # Merged, list item appended + ], + 'library_dirs': [ + '/usr/lib', + ], + }, + 'test': 1, # Merged, int value replaced +} +``` + +## Pathname Relativization + +In a `.gyp` or `.gypi` file, many string values are treated as pathnames +relative to the file in which they are defined. + +String values associated with the following keys, or contained within +lists associated with the following keys, are treated as pathnames: + + * destination + * files + * include\_dirs + * inputs + * libraries + * outputs + * sources + * mac\_bundle\_resources + * mac\_framework\_dirs + * msvs\_cygwin\_dirs + * msvs\_props + +Additionally, string values associated with keys ending in the following +suffixes, or contained within lists associated with keys ending in the +following suffixes, are treated as pathnames: + + * `_dir` + * `_dirs` + * `_file` + * `_files` + * `_path` + * `_paths` + +However, any string value beginning with any of these characters is +excluded from pathname relativization: + + * `/` for identifying absolute paths. + * `$` for introducing build system variable expansions. + * `-` to support specifying such items as `-llib`, meaning “library + `lib` in the library search path.” + * `<`, `>`, and `!` for GYP expansions. + +When merging such relative pathnames, they are adjusted so that they can +remain valid relative pathnames, despite being relative to a new home. + +#### Example + +Source dictionary from `../build/common.gypi`: + +``` +{ + 'include_dirs': ['include'], # Treated as relative to ../build + 'libraries': ['-lz'], # Not treated as a pathname, begins with a dash + 'defines': ['NDEBUG'], # defines does not contain pathnames +} +``` + +Target dictionary, from `base.gyp`: + +``` +{ + 'sources': ['string_util.cc'], +} +``` + +Merged dictionary: + +``` +{ + 'sources': ['string_util.cc'], + 'include_dirs': ['../build/include'], + 'libraries': ['-lz'], + 'defines': ['NDEBUG'], +} +``` + +Because of pathname relativization, after the merge is complete, all of +the pathnames in the merged dictionary are valid relative to the +directory containing `base.gyp`. + +## List Singletons + +Some list items are treated as singletons, and the list merge process +will enforce special rules when merging them. At present, any string +item in a list that does not begin with a dash (`-`) is treated as a +singleton, although **this is subject to change.** When appending or +prepending a singleton to a list, if the item is already in the list, +only the earlier instance is retained in the merged list. + +#### Example + +Source dictionary: + +``` +{ + 'defines': [ + 'EXPERIMENT=1', + 'NDEBUG', + ], +} +``` + +Destination dictionary: + +``` +{ + 'defines': [ + 'NDEBUG', + 'USE_THREADS', + ], +} +``` + +Merged dictionary: + +``` +{ + 'defines': [ + 'NDEBUG', + 'USE_THREADS', + 'EXPERIMENT=1', # Note that NDEBUG is not appended after this. + ], +} +``` + +## Including Other Files + +If the `-I` (`--include`) argument was used to invoke GYP, any files +specified will be implicitly merged into the root dictionary of all +`.gyp` files. + +An [includes](#includes) section may be placed anywhere within a +`.gyp` or `.gypi` (include) file. `includes` sections contain lists of +other files to include. They are processed sequentially and merged into +the enclosing dictionary at the point that the `includes` section was +found. `includes` sections at the root of a `.gyp` file dictionary are +merged after any `-I` includes from the command line. + +[includes](#includes) sections are processed immediately after a file is +loaded, even before [variable and conditional +processing](#Variables_and_Conditionals), so it is not possible to +include a file based on a [variable reference](#Variable_Expansions). +While it would be useful to be able to include files based on variable +expansions, it is most likely more useful to allow included files access +to variables set by the files that included them. + +An [includes](#includes) section may, however, be placed within a +[conditional](#Conditionals) section. The included file itself will +be loaded unconditionally, but its dictionary will be discarded if the +associated condition is not true. + +## Variables and Conditionals + +### Variables + +There are three main types of variables within GYP. + + * Predefined variables. By convention, these are named with + `CAPITAL_LETTERS`. Predefined variables are set automatically by + GYP. They may be overridden, but it is not advisable to do so. See + [Predefined Variables](#Predefined_Variables) for a list of + variables that GYP provides. + * User-defined variables. Within any dictionary, a key named + `variables` can be provided, containing a mapping between variable + names (keys) and their contents (values), which may be strings, + integers, or lists of strings. By convention, user-defined + variables are named with `lowercase_letters`. + * Automatic variables. Within any dictionary, any key with a string + value has a corresponding automatic variable whose name is the same + as the key name with an underscore (`_`) prefixed. For example, if + your dictionary contains `type: 'static_library'`, an automatic + variable named `_type` will be provided, and its value will be a + string, `'static_library'`. + +Variables are inherited from enclosing scopes. + +### Providing Default Values for Variables (%) + +Within a `variables` section, keys named with percent sign (`%`) +suffixes mean that the variable should be set only if it is undefined at +the time it is processed. This can be used to provide defaults for +variables that would otherwise be undefined, so that they may reliably +be used in [variable expansion or conditional +processing](#Variables_and_Conditionals). + +### Predefined Variables + +Each GYP generator module provides defaults for the following variables: + + * `OS`: The name of the operating system that the generator produces + output for. Common values for values for `OS` are: + + * `'linux'` + * `'mac'` + * `'win'` + + But other values may be encountered and this list should not be + considered exhaustive. The `gypd` (debug) generator module does not + provide a predefined value for `OS`. When invoking GYP with the + `gypd` module, if a value for `OS` is needed, it must be provided on + the command line, such as `gyp -f gypd -DOS=mac`. + + GYP generators also provide defaults for these variables. They may + be expressed in terms of variables used by the build system that + they generate for, often in `$(VARIABLE)` format. For example, the + GYP `PRODUCT_DIR` variable maps to the Xcode `BUILT_PRODUCTS_DIR` + variable, so `PRODUCT_DIR` is defined by the Xcode generator as + `$(BUILT_PRODUCTS_DIR)`. + * `EXECUTABLE_PREFIX`: A prefix, if any, applied to executable names. + Usually this will be an empty string. + * `EXECUTABLE_SUFFIX`: A suffix, if any, applied to executable names. + On Windows, this will be `.exe`, elsewhere, it will usually be an + empty string. + * `INTERMEDIATE_DIR`: A directory that can be used to place + intermediate build results in. `INTERMEDIATE_DIR` is only + guaranteed to be accessible within a single target (See targets). + This variable is most useful within the context of rules and actions + (See rules, See actions). Compare with `SHARED_INTERMEDIATE_DIR`. + * `PRODUCT_DIR`: The directory in which the primary output of each + target, such as executables and libraries, is placed. + * `RULE_INPUT_ROOT`: The base name for the input file (e.g. "`foo`"). + See Rules. + * `RULE_INPUT_EXT`: The file extension for the input file (e.g. + "`.cc`"). See Rules. + * `RULE_INPUT_NAME`: Full name of the input file (e.g. "`foo.cc`"). + See Rules. + * `RULE_INPUT_PATH`: Full path to the input file (e.g. + "`/bar/foo.cc`"). See Rules. + * `SHARED_INTERMEDIATE_DIR`: A directory that can be used to place + intermediate build results in, and have them be accessible to other + targets. Unlike `INTERMEDIATE_DIR`, each target in a project, + possibly spanning multiple `.gyp` files, shares the same + `SHARED_INTERMEDIATE_DIR`. + +The following additional predefined variables may be available under +certain circumstances: + + * `DEPTH`. When GYP is invoked with a `--depth` argument, when + processing any `.gyp` file, `DEPTH` will be a relative path from the + `.gyp` file to the directory specified by the `--depth` argument. + +### User-Defined Variables + +A user-defined variable may be defined in terms of other variables, but +not other variables that have definitions provided in the same scope. + +### Variable Expansions (<, >, <@, >@) + +GYP provides two forms of variable expansions, “early” or “pre” +expansions, and “late,” “post,” or “target” expansions. They have +similar syntax, differing only in the character used to introduce them. + + * Early expansions are introduced by a less-than (`<`) character. + _Mnemonic: the arrow points to the left, earlier on a timeline._ + * Late expansions are introduced by a less-than (`>`) character. + _Mnemonic: the arrow points to the right, later on a timeline._ + +The difference the two phases of expansion is described in [Early and +Late Phases](#Early_and_Late_Phases). + +These characters were chosen based upon the requirement that they not +conflict with the variable format used natively by build systems. While +the dollar sign (`$`) is the most natural fit for variable expansions, +its use was ruled out because most build systems already use that +character for their own variable expansions. Using different characters +means that no escaping mechanism was needed to differentiate between GYP +variables and build system variables, and writing build system variables +into GYP files is not cumbersome. + +Variables may contain lists or strings, and variable expansions may +occur in list or string context. There are variant forms of variable +expansions that may be used to determine how each type of variable is to +be expanded in each context. + + * When a variable is referenced by `<(VAR)` or `>(VAR)`: + * If `VAR` is a string, the variable reference within the string is + replaced by variable's string value. + * If `VAR` is a list, the variable reference within the string is + replaced by a string containing the concatenation of all of the + variable’s list items. Generally, the items are joined with + spaces between each, but the specific behavior is + generator-specific. The precise encoding used by any generator + should be one that would allow each list item to be treated as a + separate argument when used as program arguments on the system + that the generator produces output for. + * When a variable is referenced by `<@(VAR)` or `>@(VAR)`: + * The expansion must occur in list context. + * The list item must be `'<@(VAR)'` or `'>@(VAR)'` exactly. + * If `VAR` is a list, each of its elements are inserted into the + list in which expansion is taking place, replacing the list item + containing the variable reference. + * If `VAR` is a string, the string is converted to a list which is + inserted into the list in which expansion is taking place as + above. The conversion into a list is generator-specific, but + generally, spaces in the string are taken as separators between + list items. The specific method of converting the string to a + list should be the inverse of the encoding method used to expand + list variables in string context, above. + +GYP treats references to undefined variables as errors. + +### Command Expansions (` form + of [variable expansions](#Variable_Expansions), + and on the `!` form of [command + expansions](#Command_Expansions_(!,_!@)). + +These two phases are provided because there are some circumstances in +which each is desirable. + +The “early” phase is appropriate for most expansions and evaluations. +“Early” expansions and evaluations may be performed anywhere within any +`.gyp` or `.gypi` file. + +The “late” phase is appropriate when expansion or evaluation must be +deferred until a specific section has been merged into target context. +“Late” expansions and evaluations only occur within `targets` sections +and their descendants. The typical use case for a late-phase expansion +is to provide, in some globally-included `.gypi` file, distinct +behaviors depending on the specifics of a target. + +#### Example + +Given this input: + +``` +{ + 'target_defaults': { + 'target_conditions': [ + ['_type=="shared_library"', {'cflags': ['-fPIC']}], + ], + }, + 'targets': [ + { + 'target_name': 'sharing_is_caring', + 'type': 'shared_library', + }, + { + 'target_name': 'static_in_the_attic', + 'type': 'static_library', + }, + ] +} +``` + +The conditional needs to be evaluated only in target context; it is +nonsense outside of target context because no `_type` variable is +defined. [target\_conditions](#target_conditions) allows evaluation +to be deferred until after the [targets](#targets) sections are +merged into their copies of [target\_defaults](#target_defaults). +The resulting targets, after “late” phase processing: + +``` +{ + 'targets': [ + { + 'target_name': 'sharing_is_caring', + 'type': 'shared_library', + 'cflags': ['-fPIC'], + }, + { + 'target_name': 'static_in_the_attic', + 'type': 'static_library', + }, + ] +} +``` + +### Expansion and Evaluation Performed Simultaneously + +During any expansion and evaluation phase, both expansion and evaluation +are performed simultaneously. The process for handling variable +expansions and conditional evaluation within a dictionary is: + + * Load [automatic variables](#Variables) (those with leading + underscores). + * If a [variables](#variables) section is present, recurse into its + dictionary. This allows [conditionals](#Conditionals) to be + present within the `variables` dictionary. + * Load [Variables user-defined variables](#User-Defined) from the + [variables](#variables) section. + * For each string value in the dictionary, perform [variable + expansion](#Variable_Expansions) and, if operating + during the “late” phase, [command + expansions](#Command_Expansions). + * Reload [automatic variables](#Variables) and [Variables + user-defined variables](#User-Defined) because the variable + expansion step may have resulted in changes to the automatic + variables. + * If a [conditions](#conditions) or + [target\_conditions](#target_conditions) section (depending on + phase) is present, recurse into its dictionary. This is done after + variable expansion so that conditionals may take advantage of + expanded automatic variables. + * Evaluate [conditionals](#Conditionals). + * Reload [automatic variables](#Variables) and [Variables + user-defined variables](#User-Defined) because the conditional + evaluation step may have resulted in changes to the automatic + variables. + * Recurse into child dictionaries or lists that have not yet been + processed. + +One quirk of this ordering is that you cannot expect a +[variables](#variables) section within a dictionary’s +[conditional](#Conditionals) to be effective in the dictionary +itself, but the added variables will be effective in any child +dictionaries or lists. It is thought to be far more worthwhile to +provide resolved [automatic variables](#Variables) to +[conditional](#Conditionals) sections, though. As a workaround, to +conditionalize variable values, place a [conditions](#conditions) or +[target\_conditions](#target_conditions) section within the +[variables](#variables) section. + +## Dependencies and Dependents + +In GYP, “dependents” are targets that rely on other targets, called +“dependencies.” Dependents declare their reliance with a special +section within their target dictionary, +[dependencies](#dependencies). + +### Dependent Settings + +It is useful for targets to “advertise” settings to their dependents. +For example, a target might require that all of its dependents add +certain directories to their include paths, link against special +libraries, or define certain preprocessor macros. GYP allows these +cases to be handled gracefully with “dependent settings” sections. +There are three types of such sections: + + * [direct\_dependent\_settings](#direct_dependent_settings), which + advertises settings to a target's direct dependents only. + * [all\_dependent\_settings](#all_dependnet_settings), which + advertises settings to all of a target's dependents, both direct and + indirect. + * [link\_settings](#link_settings), which contains settings that + should be applied when a target’s object files are used as linker + input. + +Furthermore, in some cases, a target needs to pass its dependencies’ +settings on to its own dependents. This might happen when a target’s +own public header files include header files provided by its dependency. +[export\_dependent\_settings](#export_dependent_settings) allows a +target to declare dependencies for which +[direct\_dependent\_settings](#direct_dependent_settings) should be +passed through to its own dependents. + +Dependent settings processing merges a copy of the relevant dependent +settings dictionary from a dependency into its relevant dependent +targets. + +In most instances, +[direct\_dependent\_settings](#direct_dependent_settings) will be +used. There are very few cases where +[all\_dependent\_settings](#all_dependent_settings) is actually +correct; in most of the cases where it is tempting to use, it would be +preferable to declare +[export\_dependent\_settings](#export_dependent_settings). Most +[libraries](#libraries) and [library\_dirs](#library_dirs) +sections should be placed within [link\_settings](#link_settings) +sections. + +#### Example + +Given: + +``` +{ + 'targets': [ + { + 'target_name': 'cruncher', + 'type': 'static_library', + 'sources': ['cruncher.cc'], + 'direct_dependent_settings': { + 'include_dirs': ['.'], # dependents need to find cruncher.h. + }, + 'link_settings': { + 'libraries': ['-lm'], # cruncher.cc does math. + }, + }, + { + 'target_name': 'cruncher_test', + 'type': 'executable', + 'dependencies': ['cruncher'], + 'sources': ['cruncher_test.cc'], + }, + ], +} +``` + +After dependent settings processing, the dictionary for `cruncher_test` +will be: + +``` +{ + 'target_name': 'cruncher_test', + 'type': 'executable', + 'dependencies': ['cruncher'], # implies linking against cruncher + 'sources': ['cruncher_test.cc'], + 'include_dirs': ['.'] + 'libraries': ['-lm'], +}, +``` + +If `cruncher` was declared as a `shared_library` instead of a +`static_library`, the `cruncher_test` target would not contain `-lm`, +but instead, `cruncher` itself would link against `-lm`. + +## Linking Dependencies + +The precise meaning of a dependency relationship varies with the +[types](#type) of the [targets](#targets) at either end of the +relationship. In GYP, a dependency relationship can indicate two things +about how targets relate to each other: + + * Whether the dependent target needs to link against the dependency. + * Whether the dependency target needs to be built prior to the + dependent. If the former case is true, this case must be true as + well. + +The analysis of the first item is complicated by the differences between +static and shared libraries. + + * Static libraries are simply collections of object files (`.o` or + `.obj`) that are used as inputs to a linker (`ld` or `link.exe`). + Static libraries don't link against other libraries, they’re + collected together and used when eventually linking a shared library + or executable. + * Shared libraries are linker output and must undergo symbol + resolution. They must link against other libraries (static or + shared) in order to facilitate symbol resolution. They may be used + as libraries in subsequent link steps. + * Executables are also linker output, and also undergo symbol + resolution. Like shared libraries, they must link against static + and shared libraries to facilitate symbol resolution. They may not + be reused as linker inputs in subsequent link steps. + +Accordingly, GYP performs an operation referred to as “static library +dependency adjustment,” in which it makes each linker output target +(shared libraries and executables) link against the static libraries it +depends on, either directly or indirectly. Because the linkable targets +link against these static libraries, they are also made direct +dependents of the static libraries. + +As part of this process, GYP is also able to remove the direct +dependency relationships between two static library targets, as a +dependent static library does not actually need to link against a +dependency static library. This removal facilitates speedier builds +under some build systems, as they are now free to build the two targets +in parallel. The removal of this dependency is incorrect in some cases, +such as when the dependency target contains [rules](#rules) or +[actions](#actions) that generate header files required by the +dependent target. In such cases, the dependency target, the one +providing the side-effect files, must declare itself as a +[hard\_dependency](#hard_dependency). This setting instructs GYP to +not remove the dependency link between two static library targets in its +generated output. + +## Loading Files to Resolve Dependencies + +When GYP runs, it loads all `.gyp` files needed to resolve dependencies +found in [dependencies](#dependencies) sections. These files are not +merged into the files that reference them, but they may contain special +sections that are merged into dependent target dictionaries. + +## Build Configurations + +Explain this. + +## List Filters + +GYP allows list items to be filtered by “exclusions” and “patterns.” +Any list containing string values in a dictionary may have this +filtering applied. For the purposes of this section, a list modified by +exclusions or patterns is referred to as a “base list”, in contrast to +the “exclusion list” and “pattern list” that operates on it. + + * For a base list identified by key name `key`, the `key!` list + provides exclusions. + * For a base list identified by key name `key`, the `key/` list + provides regular expression pattern-based filtering. + +Both `key!` and `key/` may be present. The `key!` exclusion list will +be processed first, followed by the `key/` pattern list. + +Exclusion lists are most powerful when used in conjunction with +[conditionals](#Conditionals). + +## Exclusion Lists (!) + +An exclusion list provides a way to remove items from the related list +based on exact matching. Any item found in an exclusion list will be +removed from the corresponding base list. + +#### Example + +This example excludes files from the `sources` based on the setting of +the `OS` variable. + +``` +{ + 'sources:' [ + 'mac_util.mm', + 'win_util.cc', + ], + 'conditions': [ + ['OS=="mac"', {'sources!': ['win_util.cc']}], + ['OS=="win"', {'sources!': ['mac_util.cc']}], + ], +} +``` + +## Pattern Lists (/) + +Pattern lists are similar to, but more powerful than, [exclusion +lists](#Exclusion_Lists_(!)). Each item in a pattern list is itself +a two-element list. The first item is a string, either `'include'` or +`'exclude'`, specifying the action to take. The second item is a string +specifying a regular expression. Any item in the base list matching the +regular expression pattern will either be included or excluded, based on +the action specified. + +Items in a pattern list are processed in sequence, and an excluded item +that is later included will not be removed from the list (unless it is +subsequently excluded again.) + +Pattern lists are processed after [exclusion +lists](#Exclusion_Lists_(!)), so it is possible for a pattern list to +re-include items previously excluded by an exclusion list. + +Nothing is actually removed from a base list until all items in an +[exclusion list](#Exclusion_Lists_(!)) and pattern list have been +evaluated. This allows items to retain their correct position relative +to one another even after being excluded and subsequently included. + +#### Example + +In this example, a uniform naming scheme is adopted for +platform-specific files. + +``` +{ + 'sources': [ + 'io_posix.cc', + 'io_win.cc', + 'launcher_mac.cc', + 'main.cc', + 'platform_util_linux.cc', + 'platform_util_mac.mm', + ], + 'sources/': [ + ['exclude', '_win\\.cc$'], + ], + 'conditions': [ + ['OS!="linux"', {'sources/': [['exclude', '_linux\\.cc$']]}], + ['OS!="mac"', {'sources/': [['exclude', '_mac\\.cc|mm?$']]}], + ['OS=="win"', {'sources/': [ + ['include', '_win\\.cc$'], + ['exclude', '_posix\\.cc$'], + ]}], + ], +} +``` + +After the pattern list is applied, `sources` will have the following +values, depending on the setting of `OS`: + + * When `OS` is `linux`: `['io_posix.cc', 'main.cc', + 'platform_util_linux.cc']` + * When `OS` is `mac`: `['io_posix.cc', 'launcher_mac.cc', 'main.cc', + 'platform_util_mac.mm']` + * When `OS` is `win`: `['io_win.cc', 'main.cc', + 'platform_util_win.cc']` + +Note that when `OS` is `win`, the `include` for `_win.cc` files is +processed after the `exclude` matching the same pattern, because the +`sources/` list participates in [merging](#Merging) during +[conditional evaluation](#Conditonals) just like any other list +would. This guarantees that the `_win.cc` files, previously +unconditionally excluded, will be re-included when `OS` is `win`. + +## Locating Excluded Items + +In some cases, a GYP generator needs to access to items that were +excluded by an [exclusion list](#Exclusion_Lists_(!)) or [pattern +list](#Pattern_Lists_(/)). When GYP excludes items during processing +of either of these list types, it places the results in an `_excluded` +list. In the example above, when `OS` is `mac`, `sources_excluded` +would be set to `['io_win.cc', 'platform_util_linux.cc']`. Some GYP +generators use this feature to display excluded files in the project +files they generate for the convenience of users, who may wish to refer +to other implementations. + +## Processing Order + +GYP uses a defined and predictable order to execute the various steps +performed between loading files and generating output. + + * Load files. + * Load `.gyp` files. Merge any [command-line + includes](#Including_Other_Files) into each `.gyp` file’s root + dictionary. As [includes](#Including_Other_Files) are found, + load them as well and [merge](#Merging) them into the scope in + which the [includes](#includes) section was found. + * Perform [“early” or “pre”](#Early_and_Late_Phases) [variable + expansion and conditional + evaluation](#Variables_and_Conditionals). + * [Merge](#Merging) each [target’s](#targets) dictionary into + the `.gyp` file’s root [target\_defaults](#target_defaults) + dictionary. + * Scan each [target](#targets) for + [dependencies](#dependencies), and repeat the above steps for + any newly-referenced `.gyp` files not yet loaded. + * Scan each [target](#targets) for wildcard + [dependencies](#dependencies), expanding the wildcards. + * Process [dependent settings](#Dependent_Settings). These + sections are processed, in order: + * [all\_dependent\_settings](#all_dependent_settings) + * [direct\_dependent\_settings](#direct_dependent_settings) + * [link\_dependent\_settings](#link_dependent_settings) + * Perform [static library dependency + adjustment](#Linking_Dependencies). + * Perform [“late,” “post,” or “target”](#Early_and_Late_Phases) + [variable expansion and conditional + evaluation](#Variables_and_Conditionals) on [target](#targets) + dictionaries. + * Merge [target](#targets) settings into + [configurations](#configurations) as appropriate. + * Process [exclusion and pattern + lists](#List_Exclusions_and_Patterns). + +## Settings Keys + +### Settings that may appear anywhere + +#### conditions + +_List of `condition` items_ + +A `conditions` section introduces a subdictionary that is only merged +into the enclosing scope based on the evaluation of a conditional +expression. Each `condition` within a `conditions` list is itself a +list of at least two items: + + 1. A string containing the conditional expression itself. Conditional + expressions may take the following forms: + * For string values, `var=="value"` and `var!="value"` to test + equality and inequality. For example, `'OS=="linux"'` is true + when the `OS` variable is set to `"linux"`. + * For integer values, `var==value`, `var!=value`, `var=value`, and `var>value`, to test equality and + several common forms of inequality. For example, + `'chromium_code==0'` is true when the `chromium_code` variable is + set to `0`. + * It is an error for a conditional expression to reference any + undefined variable. + 1. A dictionary containing the subdictionary to be merged into the + enclosing scope if the conditional expression evaluates to true. + +These two items can be followed by any number of similar two items that +will be evaluated if the previous conditional expression does not +evaluate to true. + +An additional optional dictionary can be appended to this sequence of +two items. This optional dictionary will be merged into the enclosing +scope if none of the conditional expressions evaluate to true. + +Within a `conditions` section, each item is processed sequentially, so +it is possible to predict the order in which operations will occur. + +There is no restriction on nesting `conditions` sections. + +`conditions` sections are very similar to `target_conditions` sections. +See target\_conditions. + +#### Example + +``` +{ + 'sources': [ + 'common.cc', + ], + 'conditions': [ + ['OS=="mac"', {'sources': ['mac_util.mm']}], + ['OS=="win"', {'sources': ['win_main.cc']}, {'sources': ['posix_main.cc']}], + ['OS=="mac"', {'sources': ['mac_impl.mm']}, + 'OS=="win"', {'sources': ['win_impl.cc']}, + {'sources': ['default_impl.cc']} + ], + ], +} +``` + +Given this input, the `sources` list will take on different values based +on the `OS` variable. + + * If `OS` is `"mac"`, `sources` will contain `['common.cc', + 'mac_util.mm', 'posix_main.cc', 'mac_impl.mm']`. + * If `OS` is `"win"`, `sources` will contain `['common.cc', + 'win_main.cc', 'win_impl.cc']`. + * If `OS` is any other value such as `"linux"`, `sources` will contain + `['common.cc', 'posix_main.cc', 'default_impl.cc']`. diff --git a/gyp/docs/LanguageSpecification.md b/gyp/docs/LanguageSpecification.md new file mode 100644 index 0000000000..178b8c8316 --- /dev/null +++ b/gyp/docs/LanguageSpecification.md @@ -0,0 +1,430 @@ +# Language Specification + +## Objective + +Create a tool for the Chromium project that generates native Visual Studio, +Xcode and SCons and/or make build files from a platform-independent input +format. Make the input format as reasonably general as possible without +spending extra time trying to "get everything right," except where not doing so +would likely lead Chromium to an eventual dead end. When in doubt, do what +Chromium needs and don't worry about generalizing the solution. + +## Background + +Numerous other projects, both inside and outside Google, have tried to +create a simple, universal cross-platform build representation that +still allows sufficient per-platform flexibility to accommodate +irreconcilable differences. The fact that no obvious working candidate +exists that meets Chromium's requirements indicates this is probably a +tougher problem than it appears at first glance. We aim to succeed by +creating a tool that is highly specific to Chromium's specific use case, +not to the general case of design a completely platform-independent tool +for expressing any possible build. + +The Mac has the most sophisticated model for application development +through an IDE. Consequently, we will use the Xcode model as the +starting point (the input file format must handle Chromium's use of +Xcode seamlessly) and adapt the design as necessary for the other +platforms. + +## Overview + +The overall design has the following characteristics: + + * Input configurations are specified in files with the suffix `.gyp`. + * Each `.gyp` file specifies how to build the targets for the + "component" defined by that file. + * Each `.gyp` file generates one or more output files appropriate to + the platform: + * On Mac, a `.gyp` file generates one Xcode .xcodeproj bundle with + information about how its targets are built. + * On Windows, a `.gyp` file generates one Visual Studio .sln file, + and one Visual Studio .vcproj file per target. + * On Linux, a `.gyp` file generates one SCons file and/or one + Makefile per target + * The `.gyp` file syntax is a Python data structure. + * Use of arbitrary Python in `.gyp` files is forbidden. + * Use of eval() with restricted globals and locals on `.gyp` file + contents restricts the input to an evaluated expression, not + arbitrary Python statements. + * All input is expected to comply with JSON, with two exceptions: + the # character (not inside strings) begins a comment that lasts + until the end of the line, and trailing commas are permitted at + the end of list and dict contents. + * Input data is a dictionary of keywords and values. + * "Invalid" keywords on any given data structure are not illegal, + they're just ignored. + * TODO: providing warnings on use of illegal keywords would help + users catch typos. Figure out something nice to do with this. + +## Detailed Design + +Some up-front design principles/thoughts/TODOs: + + * Re-use keywords consistently. + * Keywords that allow configuration of a platform-specific concept get + prefixed appropriately: + * Examples: `msvs_disabled_warnings`, `xcode_framework_dirs` + * The input syntax is declarative and data-driven. + * This gets enforced by using Python `eval()` (which only evaluates + an expression) instead of `exec` (which executes arbitrary python) + * Semantic meanings of specific keyword values get deferred until all + are read and the configuration is being evaluated to spit out the + appropriate file(s) + * Source file lists: + * Are flat lists. Any imposed ordering within the `.gyp` file (e.g. + alphabetically) is purely by convention and for developer + convenience. When source files are linked or archived together, + it is expected that this will occur in the order that files are + listed in the `.gyp` file. + * Source file lists contain no mechanism for by-hand folder + configuration (`Filter` tags in Visual Studio, `Groups` in Xcode) + * A folder hierarchy is created automatically that mirrors the file + system + +### Example + +``` +{ + 'target_defaults': { + 'defines': [ + 'U_STATIC_IMPLEMENTATION', + ['LOGFILE', 'foo.log',], + ], + 'include_dirs': [ + '..', + ], + }, + 'targets': [ + { + 'target_name': 'foo', + 'type': 'static_library', + 'sources': [ + 'foo/src/foo.cc', + 'foo/src/foo_main.cc', + ], + 'include_dirs': [ + 'foo', + 'foo/include', + ], + 'conditions': [ + [ 'OS==mac', { 'sources': [ 'platform_test_mac.mm' ] } ] + ], + 'direct_dependent_settings': { + 'defines': [ + 'UNIT_TEST', + ], + 'include_dirs': [ + 'foo', + 'foo/include', + ], + }, + }, + ], +} +``` + +### Structural Elements + +### Top-level Dictionary + +This is the single dictionary in the `.gyp` file that defines the +targets and how they're to be built. + +The following keywords are meaningful within the top-level dictionary +definition: + +| *Keyword* | *Description* | +|:------------------|:------------------| +| `conditions` | A conditional section that may contain other items that can be present in a top-level dictionary, on a conditional basis. See the "Conditionals" section below. | +| `includes` | A list of `.gypi` files to be included in the top-level dictionary. | +| `target_defaults` | A dictionary of default settings to be inherited by all targets in the top-level dictionary. See the "Settings keywords" section below. | +| `targets` | A list of target specifications. See the "targets" below. | +| `variables` | A dictionary containing variable definitions. Each key in this dictionary is the name of a variable, and each value must be a string value that the variable is to be set to. | + +### targets + +A list of dictionaries defining targets to be built by the files +generated from this `.gyp` file. + +Targets may contain `includes`, `conditions`, and `variables` sections +as permitted in the root dictionary. The following additional keywords +have structural meaning for target definitions: + +| *Keyword* | *Description* | +|:---------------------------- |:------------------------------------------| +| `actions` | A list of special custom actions to perform on a specific input file, or files, to produce output files. See the "Actions" section below. | +| `all_dependent_settings` | A dictionary of settings to be applied to all dependents of the target, transitively. This includes direct dependents and the entire set of their dependents, and so on. This section may contain anything found within a `target` dictionary, except `configurations`, `target_name`, and `type` sections. Compare `direct_dependent_settings` and `link_settings`. | +| `configurations` | A list of dictionaries defining build configurations for the target. See the "Configurations" section below. | +| `copies` | A list of copy actions to perform. See the "Copies" section below. | +| `defines` | A list of preprocesor definitions to be passed on the command line to the C/C++ compiler (via `-D` or `/D` options). | +| `dependencies` | A list of targets on which this target depends. Targets in other `.gyp` files are specified as `../path/to/other.gyp:target_we_want`. | +| `direct_dependent_settings` | A dictionary of settings to be applied to other targets that depend on this target. These settings will only be applied to direct dependents. This section may contain anything found within a `target` dictionary, except `configurations`, `target_name`, and `type` sections. Compare with `all_dependent_settings` and `link_settings`. | +| `include_dirs` | A list of include directories to be passed on the command line to the C/C++ compiler (via `-I` or `/I` options). | +| `libraries` | A list of list of libraries (and/or frameworks) on which this target depends. | +| `link_settings` | A dictionary of settings to be applied to targets in which this target's contents are linked. `executable` and `shared_library` targets are linkable, so if they depend on a non-linkable target such as a `static_library`, they will adopt its `link_settings`. This section can contain anything found within a `target` dictionary, except `configurations`, `target_name`, and `type` sections. Compare `all_dependent_settings` and `direct_dependent_settings`. | +| `rules` | A special custom action to perform on a list of input files, to produce output files. See the "Rules" section below. | +| `sources` | A list of source files that are used to build this target or which should otherwise show up in the IDE for this target. In practice, we expect this list to be a union of all files necessary to build the target on all platforms, as well as other related files that aren't actually used for building, like README files. | +| `target_conditions` | Like `conditions`, but evaluation is delayed until the settings have been merged into an actual target. `target_conditions` may be used to place conditionals into a `target_defaults` section but have them still depend on specific target settings. | +| `target_name` | The name of a target being defined. | +| `type` | The type of target being defined. This field currently supports `executable`, `static_library`, `shared_library`, and `none`. The `none` target type is useful when producing output which is not linked. For example, converting raw translation files into resources or documentation into platform specific help files. | +| `msvs_props` | A list of Visual Studio property sheets (`.vsprops` files) to be used to build the target. | +| `xcode_config_file` | An Xcode configuration (`.xcconfig` file) to be used to build the target. | +| `xcode_framework_dirs` | A list of framework directories be used to build the target. | + +You can affect the way that lists/dictionaries are merged together (for +example the way a list in target\_defaults interacts with the same named +list in the target itself) with a couple of special characters, which +are covered in [Merge +Basics](InputFormatReference#Merge_Basics_(=,_?,_+).md) and [List +Filters](InputFormatReference#List_Filters.md) on the +InputFormatReference page. + +### configurations + +`configurations` sections may be found within `targets` or +`target_defaults` sections. The `configurations` section is a list of +dictionaries specifying different build configurations. Because +configurations are implemented as lists, it is not currently possible to +override aspects of configurations that are imported into a target from +a `target_defaults` section. + +NOTE: It is extremely important that each target within a project define +the same set of configurations. This continues to apply even when a +project spans across multiple `.gyp` files. + +A configuration dictionary may contain anything that can be found within +a target dictionary, except for `actions`, `all_dependent_settings`, +`configurations`, `dependencies`, `direct_dependent_settings`, +`libraries`, `link_settings`, `sources`, `target_name`, and `type`. + +Configuration dictionaries may also contain these elements: + +| *Keyword* | *Description* | +|:---------------------|:----------------------------------------------------| +| `configuration_name` | Required attribute. The name of the configuration. | + +### Conditionals + +Conditionals may appear within any dictionary in a `.gyp` file. There +are two tpes of conditionals, which differ only in the timing of their +processing. `conditons` sections are processed shortly after loading +`.gyp` files, and `target_conditons` sections are processed after all +dependencies have been computed. + +A conditional section is introduced with a `conditions` or +`target_conditions` dictionary keyword, and is composed of a list. Each +list contains two or three elements. The first two elements, which are +always required, are the conditional expression to evaluate and a +dictionary containing settings to merge into the dictionary containing +the `conditions` or `target_conditions` section if the expression +evaluates to true. The third, optional, list element is a dictionary to +merge if the expression evaluates to false. + +The `eval()` of the expression string takes place in the context of +global and/or local dictionaries that constructed from the `.gyp` input +data, and overrides the `__builtin__` dictionary, to prevent the +execution of arbitrary Python code. + +### Actions + +An `actions` section provides a list of custom build actions to perform +on inputs, producing outputs. The `actions` section is organized as a +list. Each item in the list is a dictionary having the following form: + +| *Keyword* | *Type* | *Description* | +|:--------------|:-------|:-----------------------------| +| `action_name` | string | The name of the action. Depending on how actions are implemented in the various generators, some may desire or require this property to be set to a unique name; others may ignore this property entirely. | +| `inputs` | list | A list of pathnames treated as inputs to the custom action. | +| `outputs` | list | A list of pathnames that the custom action produces. | +| `action` | list | A command line invocation used to produce `outputs` from `inputs`. For maximum cross-platform compatibility, invocations that require a Python interpreter should be specified with a first element `"python"`. This will enable generators for environments with specialized Python installations to be able to perform the action in an appropriate Python environment. | +| `message` | string | A message to be displayed to the user by the build system when the action is run. | + +Build environments will compare `inputs` and `outputs`. If any `output` +is missing or is outdated relative to any `input`, the custom action +will be invoked. If all `outputs` are present and newer than all +`inputs`, the `outputs` are considered up-to-date and the action need +not be invoked. + +Actions are implemented in Xcode as shell script build phases performed +prior to the compilation phase. In the Visual Studio generator, actions +appear files with a `FileConfiguration` containing a custom +`VCCustomBuildTool` specifying the remainder of the inputs, the outputs, +and the action. + +Combined with variable expansions, actions can be quite powerful. Here +is an example action that leverages variable expansions to minimize +duplication of pathnames: + +``` + 'sources': [ + # libraries.cc is generated by the js2c action below. + '<(INTERMEDIATE_DIR)/libraries.cc', + ], + 'actions': [ + { + 'variables': { + 'core_library_files': [ + 'src/runtime.js', + 'src/v8natives.js', + 'src/macros.py', + ], + }, + 'action_name': 'js2c', + 'inputs': [ + 'tools/js2c.py', + '<@(core_library_files)', + ], + 'outputs': [ + '<(INTERMEDIATE_DIR)/libraries.cc', + '<(INTERMEDIATE_DIR)/libraries-empty.cc', + ], + 'action': ['python', 'tools/js2c.py', '<@(_outputs)', 'CORE', '<@(core_library_files)'], + }, + ], +``` + +### Rules + +A `rules` section provides custom build action to perform on inputs, producing +outputs. The `rules` section is organized as a list. Each item in the list is +a dictionary having the following form: + +| *Keyword* | *Type* | *Description* | +|:------------|:-------|:-----------------------------------------| +| `rule_name` | string | The name of the rule. Depending on how Rules are implemented in the various generators, some may desire or require this property to be set to a unique name; others may ignore this property entirely. | +| `extension` | string | All source files of the current target with the given extension will be treated successively as inputs to the rule. | +| `inputs` | list | Additional dependencies of the rule. | +| `outputs` | list | A list of pathnames that the rule produces. Has access to `RULE_INPUT_` variables (see below). | +| `action` | list | A command line invocation used to produce `outputs` from `inputs`. For maximum cross-platform compatibility, invocations that require a Python interpreter should be specified with a first element `"python"`. This will enable generators for environments with specialized Python installations to be able to perform the action in an appropriate Python environment. Has access to `RULE_INPUT_` variables (see below). | +| `message` | string | A message to be displayed to the user by the build system when the action is run. Has access to `RULE_INPUT_` variables (see below). | + +There are several variables available to `outputs`, `action`, and `message`. + +| *Variable* | *Description* | +|:---------------------|:------------------------------------| +| `RULE_INPUT_PATH` | The full path to the current input. | +| `RULE_INPUT_DIRNAME` | The directory of the current input. | +| `RULE_INPUT_NAME` | The file name of the current input. | +| `RULE_INPUT_ROOT` | The file name of the current input without extension. | +| `RULE_INPUT_EXT` | The file name extension of the current input. | + +Rules can be thought of as Action generators. For each source selected +by `extension` an special action is created. This action starts out with +the same `inputs`, `outputs`, `action`, and `message` as the rule. The +source is added to the action's `inputs`. The `outputs`, `action`, and +`message` are then handled the same but with the additional variables. +If the `_output` variable is used in the `action` or `message` the +`RULE_INPUT_` variables in `output` will be expanded for the current +source. + +### Copies + +A `copies` section provides a simple means of copying files. The +`copies` section is organized as a list. Each item in the list is a +dictionary having the following form: + +| *Keyword* | *Type* | *Description* | +|:--------------|:-------|:------------------------------| +| `destination` | string | The directory into which the `files` will be copied. | +| `files` | list | A list of files to be copied. | + +The copies will be created in `destination` and have the same file name +as the file they are copied from. Even if the `files` are from multiple +directories they will all be copied into the `destination` directory. +Each `destination` file has an implicit build dependency on the file it +is copied from. + +### Generated Xcode .pbxproj Files + +We derive the following things in a `project.pbxproj` plist file within +an `.xcodeproj` bundle from the above input file formats as follows: + + * `Group hierarchy`: This is generated in a fixed format with contents + derived from the input files. There is no provision for the user to + specify additional groups or create a custom hierarchy. + * `Configuration group`: This will be used with the + `xcode_config_file` property above, if needed. + * `Source group`: The union of the `sources` lists of all `targets` + after applying appropriate `conditions`. The resulting list is + sorted and put into a group hierarchy that matches the layout of + the directory tree on disk, with a root of // (the top of the + hierarchy). + * `Frameworks group`: Taken directly from `libraries` value for the + target, after applying appropriate conditions. + * `Projects group`: References to other `.xcodeproj` bundles that + are needed by the `.xcodeproj` in which the group is contained. + * `Products group`: Output from the various targets. + * `Project References`: + * `Project Configurations`: + * Per-`.xcodeproj` file settings are not supported, all settings are + applied at the target level. + * `Targets`: + * `Phases`: Copy sources, link with libraries/frameworks, ... + * `Target Configurations`: Specified by input. + * `Dependencies`: (local and remote) + +### Generated Visual Studio .vcproj Files + +We derive the following sections in a `.vcproj` file from the above +input file formats as follows: + + * `VisualStudioProject`: + * `Platforms`: + * `ToolFiles`: + * `Configurations`: + * `Configuration`: + * `References`: + * `Files`: + * `Filter`: + * `File`: + * `FileConfiguration`: + * `Tool`: + * `Globals`: + +### Generated Visual Studio .sln Files + +We derive the following sections in a `.sln` file from the above input +file formats as follows: + + * `Projects`: + * `WebsiteProperties`: + * `ProjectDependencies`: + * `Global`: + * `SolutionConfigurationPlatforms`: + * `ProjectConfigurationPlatforms`: + * `SolutionProperties`: + * `NestedProjects`: + +## Caveats + +Notes/Question from very first prototype draft of the language. +Make sure these issues are addressed somewhere before deleting. + + * Libraries are easy, application abstraction is harder + * Applications involves resource compilation + * Applications involve many inputs + * Applications include transitive closure of dependencies + * Specific use cases like cc\_library + * Mac compiles more than just .c/.cpp files (specifically, .m and .mm + files) + * Compiler options vary by: + * File type + * Target type + * Individual file + * Files may have custom settings per file per platform, but we probably + don't care or need to support this in gyp. + * Will all linked non-Chromium projects always use the same versions of every + subsystem? + * Variants are difficult. We've identified the following variants (some + specific to Chromium, some typical of other projects in the same ballpark): + * Target platform + * V8 vs. JSC + * Debug vs. Release + * Toolchain (VS version, gcc, version) + * Host platform + * L10N + * Vendor + * Purify / Valgrind + * Will everyone upgrade VS at once? + * What does a dylib dependency mean? diff --git a/gyp/docs/README.md b/gyp/docs/README.md new file mode 100644 index 0000000000..5f9b6a59ce --- /dev/null +++ b/gyp/docs/README.md @@ -0,0 +1,27 @@ +# Generate Your Projects (gyp-next) + +GYP is a Meta-Build system: a build system that generates other build systems. + +* [User documentation](./UserDocumentation.md) +* [Input Format Reference](./InputFormatReference.md) +* [Language specification](./LanguageSpecification.md) +* [Hacking](./Hacking.md) +* [Testing](./Testing.md) +* [GYP vs. CMake](./GypVsCMake.md) + +GYP is intended to support large projects that need to be built on multiple +platforms (e.g., Mac, Windows, Linux), and where it is important that +the project can be built using the IDEs that are popular on each platform +as if the project is a "native" one. + +It can be used to generate XCode projects, Visual Studio projects, Ninja +build files, and Makefiles. In each case GYP's goal is to replicate as +closely as possible the way one would set up a native build of the project +using the IDE. + +GYP can also be used to generate "hybrid" projects that provide the IDE +scaffolding for a nice user experience but call out to Ninja to do the actual +building (which is usually much faster than the native build systems of the +IDEs). + +For more information on GYP, click on the links above. diff --git a/gyp/docs/Testing.md b/gyp/docs/Testing.md new file mode 100644 index 0000000000..baeb65f944 --- /dev/null +++ b/gyp/docs/Testing.md @@ -0,0 +1,450 @@ +# Testing + +NOTE: this document is outdated and needs to be updated. Read with your own discretion. + +## Introduction + +This document describes the GYP testing infrastructure, +as provided by the `TestGyp.py` module. + +These tests emphasize testing the _behavior_ of the +various GYP-generated build configurations: +Visual Studio, Xcode, SCons, Make, etc. +The goal is _not_ to test the output of the GYP generators by, +for example, comparing a GYP-generated Makefile +against a set of known "golden" Makefiles +(although the testing infrastructure could +be used to write those kinds of tests). +The idea is that the generated build configuration files +could be completely written to add a feature or fix a bug +so long as they continue to support the functional behaviors +defined by the tests: building programs, shared libraries, etc. + +## "Hello, world!" GYP test configuration + +Here is an actual test configuration, +a simple build of a C program to print `"Hello, world!"`. + +``` + $ ls -l test/hello + total 20 + -rw-r--r-- 1 knight knight 312 Jul 30 20:22 gyptest-all.py + -rw-r--r-- 1 knight knight 307 Jul 30 20:22 gyptest-default.py + -rwxr-xr-x 1 knight knight 326 Jul 30 20:22 gyptest-target.py + -rw-r--r-- 1 knight knight 98 Jul 30 20:22 hello.c + -rw-r--r-- 1 knight knight 142 Jul 30 20:22 hello.gyp + $ +``` + +The `gyptest-*.py` files are three separate tests (test scripts) +that use this configuration. The first one, `gyptest-all.py`, +looks like this: + +``` + #!/usr/bin/env python + + """ + Verifies simplest-possible build of a "Hello, world!" program + using an explicit build target of 'all'. + """ + + import TestGyp + + test = TestGyp.TestGyp() + + test.run_gyp('hello.gyp') + + test.build_all('hello.gyp') + + test.run_built_executable('hello', stdout="Hello, world!\n") + + test.pass_test() +``` + +The test script above runs GYP against the specified input file +(`hello.gyp`) to generate a build configuration. +It then tries to build the `'all'` target +(or its equivalent) using the generated build configuration. +Last, it verifies that the build worked as expected +by running the executable program (`hello`) +that was just presumably built by the generated configuration, +and verifies that the output from the program +matches the expected `stdout` string (`"Hello, world!\n"`). + +Which configuration is generated +(i.e., which build tool to test) +is specified when the test is run; +see the next section. + +Surrounding the functional parts of the test +described above are the header, +which should be basically the same for each test +(modulo a different description in the docstring): + +``` + #!/usr/bin/env python + + """ + Verifies simplest-possible build of a "Hello, world!" program + using an explicit build target of 'all'. + """ + + import TestGyp + + test = TestGyp.TestGyp() +``` + +Similarly, the footer should be the same in every test: + +``` + test.pass_test() +``` + +## Running tests + +Test scripts are run by the `gyptest.py` script. +You can specify (an) explicit test script(s) to run: + +``` + $ python gyptest.py test/hello/gyptest-all.py + PYTHONPATH=/home/knight/src/gyp/trunk/test/lib + TESTGYP_FORMAT=scons + /usr/bin/python test/hello/gyptest-all.py + PASSED + $ +``` + +If you specify a directory, all test scripts +(scripts prefixed with `gyptest-`) underneath +the directory will be run: + +``` + $ python gyptest.py test/hello + PYTHONPATH=/home/knight/src/gyp/trunk/test/lib + TESTGYP_FORMAT=scons + /usr/bin/python test/hello/gyptest-all.py + PASSED + /usr/bin/python test/hello/gyptest-default.py + PASSED + /usr/bin/python test/hello/gyptest-target.py + PASSED + $ +``` + +Or you can specify the `-a` option to run all scripts +in the tree: + +``` + $ python gyptest.py -a + PYTHONPATH=/home/knight/src/gyp/trunk/test/lib + TESTGYP_FORMAT=scons + /usr/bin/python test/configurations/gyptest-configurations.py + PASSED + /usr/bin/python test/defines/gyptest-defines.py + PASSED + . + . + . + . + /usr/bin/python test/variables/gyptest-commands.py + PASSED + $ +``` + +If any tests fail during the run, +the `gyptest.py` script will report them in a +summary at the end. + +## Debugging tests + +Tests that create intermediate output do so under the gyp/out/testworkarea +directory. On test completion, intermediate output is cleaned up. To preserve +this output, set the environment variable PRESERVE=1. This can be handy to +inspect intermediate data when debugging a test. + +You can also set PRESERVE\_PASS=1, PRESERVE\_FAIL=1 or PRESERVE\_NO\_RESULT=1 +to preserve output for tests that fall into one of those categories. + +# Specifying the format (build tool) to use + +By default, the `gyptest.py` script will generate configurations for +the "primary" supported build tool for the platform you're on: +Visual Studio on Windows, +Xcode on Mac, +and (currently) SCons on Linux. +An alternate format (build tool) may be specified +using the `-f` option: + +``` + $ python gyptest.py -f make test/hello/gyptest-all.py + PYTHONPATH=/home/knight/src/gyp/trunk/test/lib + TESTGYP_FORMAT=make + /usr/bin/python test/hello/gyptest-all.py + PASSED + $ +``` + +Multiple tools may be specified in a single pass as +a comma-separated list: + +``` + $ python gyptest.py -f make,scons test/hello/gyptest-all.py + PYTHONPATH=/home/knight/src/gyp/trunk/test/lib + TESTGYP_FORMAT=make + /usr/bin/python test/hello/gyptest-all.py + PASSED + TESTGYP_FORMAT=scons + /usr/bin/python test/hello/gyptest-all.py + PASSED + $ +``` + +## Test script functions and methods + +The `TestGyp` class contains a lot of functionality +intended to make it easy to write tests. +This section describes the most useful pieces for GYP testing. + +(The `TestGyp` class is actually a subclass of more generic +`TestCommon` and `TestCmd` base classes +that contain even more functionality than is +described here.) + +### Initialization + +The standard initialization formula is: + +``` + import TestGyp + test = TestGyp.TestGyp() +``` + +This copies the contents of the directory tree in which +the test script lives to a temporary directory for execution, +and arranges for the temporary directory's removal on exit. + +By default, any comparisons of output or file contents +must be exact matches for the test to pass. +If you need to use regular expressions for matches, +a useful alternative initialization is: + +``` + import TestGyp + test = TestGyp.TestGyp(match = TestGyp.match_re, + diff = TestGyp.diff_re)` +``` + +### Running GYP + +The canonical invocation is to simply specify the `.gyp` file to be executed: + +``` + test.run_gyp('file.gyp') +``` + +Additional GYP arguments may be specified: + +``` + test.run_gyp('file.gyp', arguments=['arg1', 'arg2', ...]) +``` + +To execute GYP from a subdirectory (where, presumably, the specified file +lives): + +``` + test.run_gyp('file.gyp', chdir='subdir') +``` + +### Running the build tool + +Running the build tool requires passing in a `.gyp` file, which may be used to +calculate the name of a specific build configuration file (such as a MSVS +solution file corresponding to the `.gyp` file). + +There are several different `.build_*()` methods for invoking different types +of builds. + +To invoke a build tool with an explicit `all` target (or equivalent): + +``` + test.build_all('file.gyp') +``` + +To invoke a build tool with its default behavior (for example, executing `make` +with no targets specified): + +``` + test.build_default('file.gyp') +``` + +To invoke a build tool with an explicit specified target: + +``` + test.build_target('file.gyp', 'target') +``` + +### Running executables + +The most useful method executes a program built by the GYP-generated +configuration: + +``` + test.run_built_executable('program') +``` + +The `.run_built_executable()` method will account for the actual built target +output location for the build tool being tested, as well as tack on any +necessary executable file suffix for the platform (for example `.exe` on +Windows). + +`stdout=` and `stderr=` keyword arguments specify expected standard output and +error output, respectively. Failure to match these (if specified) will cause +the test to fail. An explicit `None` value will suppress that verification: + +``` + test.run_built_executable('program', + stdout="expect this output\n", + stderr=None) +``` + +Note that the default values are `stdout=None` and `stderr=''` (that is, no +check for standard output, and error output must be empty). + +Arbitrary executables (not necessarily those built by GYP) can be executed with +the lower-level `.run()` method: + +``` + test.run('program') +``` + +The program must be in the local directory (that is, the temporary directory +for test execution) or be an absolute path name. + +### Fetching command output + +``` + test.stdout() +``` + +Returns the standard output from the most recent executed command (including +`.run_gyp()`, `.build_*()`, or `.run*()` methods). + +``` + test.stderr() +``` + +Returns the error output from the most recent executed command (including +`.run_gyp()`, `.build_*()`, or `.run*()` methods). + +### Verifying existence or non-existence of files or directories + +``` + test.must_exist('file_or_dir') +``` + +Verifies that the specified file or directory exists, and fails the test if it +doesn't. + +``` + test.must_not_exist('file_or_dir') +``` + +Verifies that the specified file or directory does not exist, and fails the +test if it does. + +### Verifying file contents + +``` + test.must_match('file', 'expected content\n') +``` + +Verifies that the content of the specified file match the expected string, and +fails the test if it does not. By default, the match must be exact, but +line-by-line regular expressions may be used if the `TestGyp` object was +initialized with `TestGyp.match_re`. + +``` + test.must_not_match('file', 'expected content\n') +``` + +Verifies that the content of the specified file does _not_ match the expected +string, and fails the test if it does. By default, the match must be exact, +but line-by-line regular expressions may be used if the `TestGyp` object was +initialized with `TestGyp.match_re`. + +``` + test.must_contain('file', 'substring') +``` + +Verifies that the specified file contains the specified substring, and fails +the test if it does not. + +``` + test.must_not_contain('file', 'substring') +``` + +Verifies that the specified file does not contain the specified substring, and +fails the test if it does. + +``` + test.must_contain_all_lines(output, lines) +``` + +Verifies that the output string contains all of the "lines" in the specified +list of lines. In practice, the lines can be any substring and need not be +`\n`-terminaed lines per se. If any line is missing, the test fails. + +``` + test.must_not_contain_any_lines(output, lines) +``` + +Verifies that the output string does _not_ contain any of the "lines" in the +specified list of lines. In practice, the lines can be any substring and need +not be `\n`-terminaed lines per se. If any line exists in the output string, +the test fails. + +``` + test.must_contain_any_line(output, lines) +``` + +Verifies that the output string contains at least one of the "lines" in the +specified list of lines. In practice, the lines can be any substring and need +not be `\n`-terminaed lines per se. If none of the specified lines is present, +the test fails. + +### Reading file contents + +``` + test.read('file') +``` + +Returns the contents of the specified file. Directory elements contained in a +list will be joined: + +``` + test.read(['subdir', 'file']) +``` + +### Test success or failure + +``` + test.fail_test() +``` + +Fails the test, reporting `FAILED` on standard output and exiting with an exit +status of `1`. + +``` + test.pass_test() +``` + +Passes the test, reporting `PASSED` on standard output and exiting with an exit +status of `0`. + +``` + test.no_result() +``` + +Indicates the test had no valid result (i.e., the conditions could not be +tested because of an external factor like a full file system). Reports `NO +RESULT` on standard output and exits with a status of `2`. diff --git a/gyp/docs/UserDocumentation.md b/gyp/docs/UserDocumentation.md new file mode 100644 index 0000000000..808f37a1a9 --- /dev/null +++ b/gyp/docs/UserDocumentation.md @@ -0,0 +1,965 @@ +# User Documentation + +## Introduction + +This document is intended to provide a user-level guide to GYP. The +emphasis here is on how to use GYP to accomplish specific tasks, not on +the complete technical language specification. (For that, see the +[LanguageSpecification](LanguageSpecification.md).) + +The document below starts with some overviews to provide context: an +overview of the structure of a `.gyp` file itself, an overview of a +typical executable-program target in a `.gyp` file, an an overview of a +typical library target in a `.gyp` file. + +After the overviews, there are examples of `gyp` patterns for different +common use cases. + +## Skeleton of a typical Chromium .gyp file + +Here is the skeleton of a typical `.gyp` file in the Chromium tree: + +``` + { + 'variables': { + . + . + . + }, + 'includes': [ + '../build/common.gypi', + ], + 'target_defaults': { + . + . + . + }, + 'targets': [ + { + 'target_name': 'target_1', + . + . + . + }, + { + 'target_name': 'target_2', + . + . + . + }, + ], + 'conditions': [ + ['OS=="linux"', { + 'targets': [ + { + 'target_name': 'linux_target_3', + . + . + . + }, + ], + }], + ['OS=="win"', { + 'targets': [ + { + 'target_name': 'windows_target_4', + . + . + . + }, + ], + }, { # OS != "win" + 'targets': [ + { + 'target_name': 'non_windows_target_5', + . + . + . + }, + }], + ], + } +``` + +The entire file just contains a Python dictionary. (It's actually JSON, +with two small Pythonic deviations: comments are introduced with `#`, +and a `,` (comma)) is legal after the last element in a list or +dictionary.) + +The top-level pieces in the `.gyp` file are as follows: + +`'variables'`: Definitions of variables that can be interpolated and +used in various other parts of the file. + +`'includes'`: A list of of other files that will be included in this +file. By convention, included files have the suffix `.gypi` (gyp +include). + +`'target_defaults'`: Settings that will apply to _all_ of the targets +defined in this `.gyp` file. + +`'targets'`: The list of targets for which this `.gyp` file can +generate builds. Each target is a dictionary that contains settings +describing all the information necessary to build the target. + +`'conditions'`: A list of condition specifications that can modify the +contents of the items in the global dictionary defined by this `.gyp` +file based on the values of different variablwes. As implied by the +above example, the most common use of a `conditions` section in the +top-level dictionary is to add platform-specific targets to the +`targets` list. + +## Skeleton of a typical executable target in a .gyp file + +The most straightforward target is probably a simple executable program. +Here is an example `executable` target that demonstrates the features +that should cover most simple uses of gyp: + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'type': 'executable', + 'msvs_guid': '5ECEC9E5-8F23-47B6-93E0-C3B328B3BE65', + 'dependencies': [ + 'xyzzy', + '../bar/bar.gyp:bar', + ], + 'defines': [ + 'DEFINE_FOO', + 'DEFINE_A_VALUE=value', + ], + 'include_dirs': [ + '..', + ], + 'sources': [ + 'file1.cc', + 'file2.cc', + ], + 'conditions': [ + ['OS=="linux"', { + 'defines': [ + 'LINUX_DEFINE', + ], + 'include_dirs': [ + 'include/linux', + ], + }], + ['OS=="win"', { + 'defines': [ + 'WINDOWS_SPECIFIC_DEFINE', + ], + }, { # OS != "win", + 'defines': [ + 'NON_WINDOWS_DEFINE', + ], + }] + ], + }, + ], + } +``` + +The top-level settings in the target include: + +`'target_name'`: The name by which the target should be known, which +should be unique across all `.gyp` files. This name will be used as the +project name in the generated Visual Studio solution, as the target name +in the generated XCode configuration, and as the alias for building this +target from the command line of the generated SCons configuration. + +`'type'`: Set to `executable`, logically enough. + +`'msvs_guid'`: THIS IS ONLY TRANSITIONAL. This is a hard-coded GUID +values that will be used in the generated Visual Studio solution +file(s). This allows us to check in a `chrome.sln` file that +interoperates with gyp-generated project files. Once everything in +Chromium is being generated by gyp, it will no longer be important that +the GUIDs stay constant across invocations, and we'll likely get rid of +these settings, + +`'dependencies'`: This lists other targets that this target depends on. +The gyp-generated files will guarantee that the other targets are built +before this target. Any library targets in the `dependencies` list will +be linked with this target. The various settings (`defines`, +`include_dirs`, etc.) listed in the `direct_dependent_settings` sections +of the targets in this list will be applied to how _this_ target is +built and linked. See the more complete discussion of +`direct_dependent_settings`, below. + +`'defines'`: The C preprocessor definitions that will be passed in on +compilation command lines (using `-D` or `/D` options). + +`'include_dirs'`: The directories in which included header files live. +These will be passed in on compilation command lines (using `-I` or `/I` +options). + +`'sources'`: The source files for this target. + +`'conditions'`: A block of conditions that will be evaluated to update +the different settings in the target dictionary. + +## Skeleton of a typical library target in a .gyp file + +The vast majority of targets are libraries. Here is an example of a +library target including the additional features that should cover most +needs of libraries: + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'type': '<(library)' + 'msvs_guid': '5ECEC9E5-8F23-47B6-93E0-C3B328B3BE65', + 'dependencies': [ + 'xyzzy', + '../bar/bar.gyp:bar', + ], + 'defines': [ + 'DEFINE_FOO', + 'DEFINE_A_VALUE=value', + ], + 'include_dirs': [ + '..', + ], + 'direct_dependent_settings': { + 'defines': [ + 'DEFINE_FOO', + 'DEFINE_ADDITIONAL', + ], + 'linkflags': [ + ], + }, + 'export_dependent_settings': [ + '../bar/bar.gyp:bar', + ], + 'sources': [ + 'file1.cc', + 'file2.cc', + ], + 'conditions': [ + ['OS=="linux"', { + 'defines': [ + 'LINUX_DEFINE', + ], + 'include_dirs': [ + 'include/linux', + ], + ], + ['OS=="win"', { + 'defines': [ + 'WINDOWS_SPECIFIC_DEFINE', + ], + }, { # OS != "win", + 'defines': [ + 'NON_WINDOWS_DEFINE', + ], + }] + ], + ], + } +``` + +The possible entries in a library target are largely the same as those +that can be specified for an executable target (`defines`, +`include_dirs`, etc.). The differences include: + +`'type'`: This should almost always be set to '<(library)', which allows +the user to define at gyp time whether libraries are to be built static +or shared. (On Linux, at least, linking with shared libraries saves +significant link time.) If it's necessary to pin down the type of +library to be built, the `type` can be set explicitly to +`static_library` or `shared_library`. + +`'direct_dependent_settings'`: This defines the settings that will be +applied to other targets that _directly depend_ on this target--that is, +that list _this_ target in their `'dependencies'` setting. This is +where you list the `defines`, `include_dirs`, `cflags` and `linkflags` +that other targets that compile or link against this target need to +build consistently. + +`'export_dependent_settings'`: This lists the targets whose +`direct_dependent_settings` should be "passed on" to other targets that +use (depend on) this target. `TODO: expand on this description.` + +## Use Cases + +These use cases are intended to cover the most common actions performed +by developers using GYP. + +Note that these examples are _not_ fully-functioning, self-contained +examples (or else they'd be way too long). Each example mostly contains +just the keywords and settings relevant to the example, with perhaps a +few extra keywords for context. The intent is to try to show the +specific pieces you need to pay attention to when doing something. +[NOTE: if practical use shows that these examples are confusing without +additional context, please add what's necessary to clarify things.] + +### Add new source files + +There are similar but slightly different patterns for adding a +platform-independent source file vs. adding a source file that only +builds on some of the supported platforms. + +#### Add a source file that builds on all platforms + +**Simplest possible case**: You are adding a file(s) that builds on all +platforms. + +Just add the file(s) to the `sources` list of the appropriate dictionary +in the `targets` list: + +``` + { + 'targets': [ + { + 'target_name': 'my_target', + 'type': 'executable', + 'sources': [ + '../other/file_1.cc', + 'new_file.cc', + 'subdir/file3.cc', + ], + }, + ], + }, +``` + +File path names are relative to the directory in which the `.gyp` file lives. + +Keep the list sorted alphabetically (unless there's a really, really, +_really_ good reason not to). + +#### Add a platform-specific source file + +##### Your platform-specific file is named `*_linux.{ext}`, `*_mac.{ext}`, `*_posix.{ext}` or `*_win.{ext}` + +The simplest way to add a platform-specific source file, assuming you're +adding a completely new file and get to name it, is to use one of the +following standard suffixes: + + * `_linux` (e.g. `foo_linux.cc`) + * `_mac` (e.g. `foo_mac.cc`) + * `_posix` (e.g. `foo_posix.cc`) + * `_win` (e.g. `foo_win.cc`) + +Simply add the file to the `sources` list of the appropriate dict within +the `targets` list, like you would any other source file. + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'type': 'executable', + 'sources': [ + 'independent.cc', + 'specific_win.cc', + ], + }, + ], + }, +``` + +The Chromium `.gyp` files all have appropriate `conditions` entries to +filter out the files that aren't appropriate for the current platform. +In the above example, the `specific_win.cc` file will be removed +automatically from the source-list on non-Windows builds. + +##### Your platform-specific file does not use an already-defined pattern + +If your platform-specific file does not contain a +`*_{linux,mac,posix,win}` substring (or some other pattern that's +already in the `conditions` for the target), and you can't change the +file name, there are two patterns that can be used. + +**Prefererred**: Add the file to the `sources` list of the appropriate +dictionary within the `targets` list. Add an appropriate `conditions` +section to exclude the specific files name: + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'type': 'executable', + 'sources': [ + 'linux_specific.cc', + ], + 'conditions': [ + ['OS != "linux"', { + 'sources!': [ + # Linux-only; exclude on other platforms. + 'linux_specific.cc', + ] + }[, + ], + }, + ], + }, +``` + +Despite the duplicate listing, the above is generally preferred because +the `sources` list contains a useful global list of all sources on all +platforms with consistent sorting on all platforms. + +**Non-preferred**: In some situations, however, it might make sense to +list a platform-specific file only in a `conditions` section that +specifically _includes_ it in the `sources` list: + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'type': 'executable', + 'sources': [], + ['OS == "linux"', { + 'sources': [ + # Only add to sources list on Linux. + 'linux_specific.cc', + ] + }], + }, + ], + }, +``` + +The above two examples end up generating equivalent builds, with the +small exception that the `sources` lists will list the files in +different orders. (The first example defines explicitly where +`linux_specific.cc` appears in the list--perhaps in in the +middle--whereas the second example will always tack it on to the end of +the list.) + +**Including or excluding files using patterns**: There are more +complicated ways to construct a `sources` list based on patterns. See +`TODO` below. + +### Add a new executable + +An executable program is probably the most straightforward type of +target, since all it typically needs is a list of source files, some +compiler/linker settings (probably varied by platform), and some library +targets on which it depends and which must be used in the final link. + +#### Add an executable that builds on all platforms + +Add a dictionary defining the new executable target to the `targets` +list in the appropriate `.gyp` file. Example: + +``` + { + 'targets': [ + { + 'target_name': 'new_unit_tests', + 'type': 'executable', + 'defines': [ + 'FOO', + ], + 'include_dirs': [ + '..', + ], + 'dependencies': [ + 'other_target_in_this_file', + 'other_gyp2:target_in_other_gyp2', + ], + 'sources': [ + 'new_additional_source.cc', + 'new_unit_tests.cc', + ], + }, + ], + } +``` + +#### Add a platform-specific executable + +Add a dictionary defining the new executable target to the `targets` +list within an appropriate `conditions` block for the platform. The +`conditions` block should be a sibling to the top-level `targets` list: + +``` + { + 'targets': [ + ], + 'conditions': [ + ['OS=="win"', { + 'targets': [ + { + 'target_name': 'new_unit_tests', + 'type': 'executable', + 'defines': [ + 'FOO', + ], + 'include_dirs': [ + '..', + ], + 'dependencies': [ + 'other_target_in_this_file', + 'other_gyp2:target_in_other_gyp2', + ], + 'sources': [ + 'new_additional_source.cc', + 'new_unit_tests.cc', + ], + }, + ], + }], + ], + } +``` + +### Add settings to a target + +There are several different types of settings that can be defined for +any given target. + +#### Add new preprocessor definitions (`-D` or `/D` flags) + +New preprocessor definitions are added by the `defines` setting: + +``` + { + 'targets': [ + { + 'target_name': 'existing_target', + 'defines': [ + 'FOO', + 'BAR=some_value', + ], + }, + ], + }, +``` + +These may be specified directly in a target's settings, as in the above +example, or in a `conditions` section. + +#### Add a new include directory (`-I` or `/I` flags) + +New include directories are added by the `include_dirs` setting: + +``` + { + 'targets': [ + { + 'target_name': 'existing_target', + 'include_dirs': [ + '..', + 'include', + ], + }, + ], + }, +``` + +These may be specified directly in a target's settings, as in the above +example, or in a `conditions` section. + +#### Add new compiler flags + +Specific compiler flags can be added with the `cflags` setting: + +``` + { + 'targets': [ + { + 'target_name': 'existing_target', + 'conditions': [ + ['OS=="win"', { + 'cflags': [ + '/WX', + ], + }, { # OS != "win" + 'cflags': [ + '-Werror', + ], + }], + ], + }, + ], + }, +``` + +Because these flags will be specific to the actual compiler involved, +they will almost always be only set within a `conditions` section. + +#### Add new linker flags + +Setting linker flags is OS-specific. On linux and most non-mac posix +systems, they can be added with the `ldflags` setting: + +``` + { + 'targets': [ + { + 'target_name': 'existing_target', + 'conditions': [ + ['OS=="linux"', { + 'ldflags': [ + '-pthread', + ], + }], + ], + }, + ], + }, +``` + +Because these flags will be specific to the actual linker involved, +they will almost always be only set within a `conditions` section. + +On OS X, linker settings are set via `xcode_settings`, on Windows via +`msvs_settings`. + +#### Exclude settings on a platform + +Any given settings keyword (`defines`, `include_dirs`, etc.) has a +corresponding form with a trailing `!` (exclamation point) to remove +values from a setting. One useful example of this is to remove the +Linux `-Werror` flag from the global settings defined in +`build/common.gypi`: + +``` + { + 'targets': [ + { + 'target_name': 'third_party_target', + 'conditions': [ + ['OS=="linux"', { + 'cflags!': [ + '-Werror', + ], + }], + ], + }, + ], + }, +``` + +### Cross-compiling + +GYP has some (relatively limited) support for cross-compiling. + +If the variable `GYP_CROSSCOMPILE` or one of the toolchain-related +variables (like `CC_host` or `CC_target`) is set, GYP will think that +you wish to do a cross-compile. + +When cross-compiling, each target can be part of a "host" build, a +"target" build, or both. By default, the target is assumed to be (only) +part of the "target" build. The 'toolsets' property can be set on a +target to change the default. + +A target's dependencies are assumed to match the build type (so, if A +depends on B, by default that means that a target build of A depends on +a target build of B). You can explicitly depend on targets across +toolchains by specifying "#host" or "#target" in the dependencies list. +If GYP is not doing a cross-compile, the "#host" and "#target" will be +stripped as needed, so nothing breaks. + +### Add a new library + +TODO: write intro + +#### Add a library that builds on all platforms + +Add the a dictionary defining the new library target to the `targets` +list in the appropriate `.gyp` file. Example: + +``` + { + 'targets': [ + { + 'target_name': 'new_library', + 'type': '<(library)', + 'defines': [ + 'FOO', + 'BAR=some_value', + ], + 'include_dirs': [ + '..', + ], + 'dependencies': [ + 'other_target_in_this_file', + 'other_gyp2:target_in_other_gyp2', + ], + 'direct_dependent_settings': { + 'include_dirs': '.', + }, + 'export_dependent_settings': [ + 'other_target_in_this_file', + ], + 'sources': [ + 'new_additional_source.cc', + 'new_library.cc', + ], + }, + ], + } +``` + +The use of the `<(library)` variable above should be the default `type` +setting for most library targets, as it allows the developer to choose, +at `gyp` time, whether to build with static or shared libraries. +(Building with shared libraries saves a _lot_ of link time on Linux.) + +It may be necessary to build a specific library as a fixed type. Is so, +the `type` field can be hard-wired appropriately. For a static library: + +``` + 'type': 'static_library', +``` + +For a shared library: + +``` + 'type': 'shared_library', +``` + +#### Add a platform-specific library + +Add a dictionary defining the new library target to the `targets` list +within a `conditions` block that's a sibling to the top-level `targets` +list: + +``` + { + 'targets': [ + ], + 'conditions': [ + ['OS=="win"', { + 'targets': [ + { + 'target_name': 'new_library', + 'type': '<(library)', + 'defines': [ + 'FOO', + 'BAR=some_value', + ], + 'include_dirs': [ + '..', + ], + 'dependencies': [ + 'other_target_in_this_file', + 'other_gyp2:target_in_other_gyp2', + ], + 'direct_dependent_settings': { + 'include_dirs': '.', + }, + 'export_dependent_settings': [ + 'other_target_in_this_file', + ], + 'sources': [ + 'new_additional_source.cc', + 'new_library.cc', + ], + }, + ], + }], + ], + } +``` + +### Dependencies between targets + +GYP provides useful primitives for establishing dependencies between +targets, which need to be configured in the following situations. + +#### Linking with another library target + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'dependencies': [ + 'libbar', + ], + }, + { + 'target_name': 'libbar', + 'type': '<(library)', + 'sources': [ + ], + }, + ], + } +``` + +Note that if the library target is in a different `.gyp` file, you have +to specify the path to other `.gyp` file, relative to this `.gyp` file's +directory: + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'dependencies': [ + '../bar/bar.gyp:libbar', + ], + }, + ], + } +``` + +Adding a library often involves updating multiple `.gyp` files, adding +the target to the approprate `.gyp` file (possibly a newly-added `.gyp` +file), and updating targets in the other `.gyp` files that depend on +(link with) the new library. + +#### Compiling with necessary flags for a library target dependency + +We need to build a library (often a third-party library) with specific +preprocessor definitions or command-line flags, and need to ensure that +targets that depend on the library build with the same settings. This +situation is handled by a `direct_dependent_settings` block: + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'type': 'executable', + 'dependencies': [ + 'libbar', + ], + }, + { + 'target_name': 'libbar', + 'type': '<(library)', + 'defines': [ + 'LOCAL_DEFINE_FOR_LIBBAR', + 'DEFINE_TO_USE_LIBBAR', + ], + 'include_dirs': [ + '..', + 'include/libbar', + ], + 'direct_dependent_settings': { + 'defines': [ + 'DEFINE_TO_USE_LIBBAR', + ], + 'include_dirs': [ + 'include/libbar', + ], + }, + }, + ], + } +``` + +In the above example, the sources of the `foo` executable will be +compiled with the options `-DDEFINE_TO_USE_LIBBAR -Iinclude/libbar`, +because of those settings' being listed in the +`direct_dependent_settings` block. + +Note that these settings will likely need to be replicated in the +settings for the library target itsef, so that the library will build +with the same options. This does not prevent the target from defining +additional options for its "internal" use when compiling its own source +files. (In the above example, these are the `LOCAL_DEFINE_FOR_LIBBAR` +define, and the `..` entry in the `include_dirs` list.) + +#### When a library depends on an additional library at final link time + +``` + { + 'targets': [ + { + 'target_name': 'foo', + 'type': 'executable', + 'dependencies': [ + 'libbar', + ], + }, + { + 'target_name': 'libbar', + 'type': '<(library)', + 'dependencies': [ + 'libother' + ], + 'export_dependent_settings': [ + 'libother' + ], + }, + { + 'target_name': 'libother', + 'type': '<(library)', + 'direct_dependent_settings': { + 'defines': [ + 'DEFINE_FOR_LIBOTHER', + ], + 'include_dirs': [ + 'include/libother', + ], + }, + }, + ], + } +``` + +### Support for Mac OS X bundles + +gyp supports building bundles on OS X (.app, .framework, .bundle, etc). +Here is an example of this: + +``` + { + 'target_name': 'test_app', + 'product_name': 'Test App Gyp', + 'type': 'executable', + 'mac_bundle': 1, + 'sources': [ + 'main.m', + 'TestAppAppDelegate.h', + 'TestAppAppDelegate.m', + ], + 'mac_bundle_resources': [ + 'TestApp/English.lproj/InfoPlist.strings', + 'TestApp/English.lproj/MainMenu.xib', + ], + 'link_settings': { + 'libraries': [ + '$(SDKROOT)/System/Library/Frameworks/Cocoa.framework', + ], + }, + 'xcode_settings': { + 'INFOPLIST_FILE': 'TestApp/TestApp-Info.plist', + }, + }, +``` + +The `mac_bundle` key tells gyp that this target should be a bundle. +`executable` targets get extension `.app` by default, `shared_library` +targets get `.framework` – but you can change the bundle extensions by +setting `product_extension` if you want. Files listed in +`mac_bundle_resources` will be copied to the bundle's `Resource` folder +of the bundle. You can also set +`process_outputs_as_mac_bundle_resources` to 1 in actions and rules to +let the output of actions and rules be added to that folder (similar to +`process_outputs_as_sources`). If `product_name` is not set, the bundle +will be named after `target_name`as usual. + +### Move files (refactoring) + +TODO + +### Custom build steps + +TODO + +#### Adding an explicit build step to generate specific files + +TODO + +#### Adding a rule to handle files with a new suffix + +TODO + +### Build flavors + +TODO diff --git a/gyp/pylib/gyp/MSVSSettings.py b/gyp/pylib/gyp/MSVSSettings.py index 93633dbca1..ac87f572b2 100644 --- a/gyp/pylib/gyp/MSVSSettings.py +++ b/gyp/pylib/gyp/MSVSSettings.py @@ -793,6 +793,8 @@ def _ValidateSettings(validators, settings, stderr): _compile, "CompileAsManaged", _Enumeration([], new=["false", "true"]) ) # /clr _MSBuildOnly(_compile, "CreateHotpatchableImage", _boolean) # /hotpatch +_MSBuildOnly(_compile, "LanguageStandard", _string) +_MSBuildOnly(_compile, "LanguageStandard_C", _string) _MSBuildOnly(_compile, "MultiProcessorCompilation", _boolean) # /MP _MSBuildOnly(_compile, "PreprocessOutputPath", _string) # /Fi _MSBuildOnly(_compile, "ProcessorNumber", _integer) # the number of processors diff --git a/gyp/pylib/gyp/common.py b/gyp/pylib/gyp/common.py index b73a0c55b1..762ae02109 100644 --- a/gyp/pylib/gyp/common.py +++ b/gyp/pylib/gyp/common.py @@ -9,6 +9,7 @@ import tempfile import sys import subprocess +import shlex from collections.abc import MutableSet @@ -422,8 +423,54 @@ def EnsureDirExists(path): except OSError: pass +def GetCrossCompilerPredefines(): # -> dict + cmd = [] + + # shlex.split() will eat '\' in posix mode, but + # setting posix=False will preserve extra '"' cause CreateProcess fail on Windows + # this makes '\' in %CC_target% and %CFLAGS% work + def replace_sep(s): + return s.replace(os.sep, "/") if os.sep != "/" else s + + if CC := os.environ.get("CC_target") or os.environ.get("CC"): + cmd += shlex.split(replace_sep(CC)) + if CFLAGS := os.environ.get("CFLAGS"): + cmd += shlex.split(replace_sep(CFLAGS)) + elif CXX := os.environ.get("CXX_target") or os.environ.get("CXX"): + cmd += shlex.split(replace_sep(CXX)) + if CXXFLAGS := os.environ.get("CXXFLAGS"): + cmd += shlex.split(replace_sep(CXXFLAGS)) + else: + return {} -def GetFlavor(params): + if sys.platform == "win32": + fd, input = tempfile.mkstemp(suffix=".c") + real_cmd = [*cmd, "-dM", "-E", "-x", "c", input] + try: + os.close(fd) + stdout = subprocess.run( + real_cmd, shell=True, + capture_output=True, check=True + ).stdout + finally: + os.unlink(input) + else: + input = "/dev/null" + real_cmd = [*cmd, "-dM", "-E", "-x", "c", input] + stdout = subprocess.run( + real_cmd, shell=False, + capture_output=True, check=True + ).stdout + + defines = {} + lines = stdout.decode("utf-8").replace("\r\n", "\n").split("\n") + for line in lines: + if (line or "").startswith("#define "): + _, key, *value = line.split(" ") + defines[key] = " ".join(value) + return defines + +def GetFlavorByPlatform(): """Returns |params.flavor| if it's set, the system's default flavor else.""" flavors = { "cygwin": "win", @@ -431,8 +478,6 @@ def GetFlavor(params): "darwin": "mac", } - if "flavor" in params: - return params["flavor"] if sys.platform in flavors: return flavors[sys.platform] if sys.platform.startswith("sunos"): @@ -452,6 +497,18 @@ def GetFlavor(params): return "linux" +def GetFlavor(params): + if "flavor" in params: + return params["flavor"] + + defines = GetCrossCompilerPredefines() + if "__EMSCRIPTEN__" in defines: + return "emscripten" + if "__wasm__" in defines: + return "wasi" if "__wasi__" in defines else "wasm" + + return GetFlavorByPlatform() + def CopyTool(flavor, out_path, generator_flags={}): """Finds (flock|mac|win)_tool.gyp in the gyp directory and copies it diff --git a/gyp/pylib/gyp/common_test.py b/gyp/pylib/gyp/common_test.py index 05344085ad..b6c4cccc1a 100755 --- a/gyp/pylib/gyp/common_test.py +++ b/gyp/pylib/gyp/common_test.py @@ -9,7 +9,8 @@ import gyp.common import unittest import sys - +import os +from unittest.mock import patch, MagicMock class TestTopologicallySorted(unittest.TestCase): def test_Valid(self): @@ -24,9 +25,8 @@ def test_Valid(self): def GetEdge(node): return tuple(graph[node]) - self.assertEqual( - gyp.common.TopologicallySorted(graph.keys(), GetEdge), ["a", "c", "d", "b"] - ) + assert gyp.common.TopologicallySorted( + graph.keys(), GetEdge) == ["a", "c", "d", "b"] def test_Cycle(self): """Test that an exception is thrown on a cyclic graph.""" @@ -58,7 +58,7 @@ def tearDown(self): def assertFlavor(self, expected, argument, param): sys.platform = argument - self.assertEqual(expected, gyp.common.GetFlavor(param)) + assert expected == gyp.common.GetFlavor(param) def test_platform_default(self): self.assertFlavor("freebsd", "freebsd9", {}) @@ -73,6 +73,99 @@ def test_platform_default(self): def test_param(self): self.assertFlavor("foobar", "linux2", {"flavor": "foobar"}) + class MockCommunicate: + def __init__(self, stdout): + self.stdout = stdout + + def decode(self, encoding): + return self.stdout + + @patch("os.close") + @patch("os.unlink") + @patch("tempfile.mkstemp") + def test_GetCrossCompilerPredefines(self, mock_mkstemp, mock_unlink, mock_close): + mock_close.return_value = None + mock_unlink.return_value = None + mock_mkstemp.return_value = (0, "temp.c") + + def mock_run(env, defines_stdout, expected_cmd): + with patch("subprocess.run") as mock_run: + mock_process = MagicMock() + mock_process.returncode = 0 + mock_process.stdout = TestGetFlavor.MockCommunicate(defines_stdout) + mock_run.return_value = mock_process + expected_input = "temp.c" if sys.platform == "win32" else "/dev/null" + with patch.dict(os.environ, env): + defines = gyp.common.GetCrossCompilerPredefines() + flavor = gyp.common.GetFlavor({}) + if env.get("CC_target"): + mock_run.assert_called_with( + [ + *expected_cmd, + "-dM", "-E", "-x", "c", expected_input + ], + shell=sys.platform == "win32", + capture_output=True, check=True) + return [defines, flavor] + + [defines1, _] = mock_run({}, "", []) + assert {} == defines1 + + [defines2, flavor2] = mock_run( + { "CC_target": "/opt/wasi-sdk/bin/clang" }, + "#define __wasm__ 1\n#define __wasi__ 1\n", + ["/opt/wasi-sdk/bin/clang"] + ) + assert { "__wasm__": "1", "__wasi__": "1" } == defines2 + assert flavor2 == "wasi" + + [defines3, flavor3] = mock_run( + { "CC_target": "/opt/wasi-sdk/bin/clang --target=wasm32" }, + "#define __wasm__ 1\n", + ["/opt/wasi-sdk/bin/clang", "--target=wasm32"] + ) + assert { "__wasm__": "1" } == defines3 + assert flavor3 == "wasm" + + [defines4, flavor4] = mock_run( + { "CC_target": "/emsdk/upstream/emscripten/emcc" }, + "#define __EMSCRIPTEN__ 1\n", + ["/emsdk/upstream/emscripten/emcc"] + ) + assert { "__EMSCRIPTEN__": "1" } == defines4 + assert flavor4 == "emscripten" + + # Test path which include white space + [defines5, flavor5] = mock_run( + { + "CC_target": "\"/Users/Toyo Li/wasi-sdk/bin/clang\" -O3", + "CFLAGS": "--target=wasm32-wasi-threads -pthread" + }, + "#define __wasm__ 1\n#define __wasi__ 1\n#define _REENTRANT 1\n", + [ + "/Users/Toyo Li/wasi-sdk/bin/clang", + "-O3", + "--target=wasm32-wasi-threads", + "-pthread" + ] + ) + assert { + "__wasm__": "1", + "__wasi__": "1", + "_REENTRANT": "1" + } == defines5 + assert flavor5 == "wasi" + + original_sep = os.sep + os.sep = "\\" + [defines6, flavor6] = mock_run( + { "CC_target": "\"C:\\Program Files\\wasi-sdk\\clang.exe\"" }, + "#define __wasm__ 1\n#define __wasi__ 1\n", + ["C:/Program Files/wasi-sdk/clang.exe"] + ) + os.sep = original_sep + assert { "__wasm__": "1", "__wasi__": "1" } == defines6 + assert flavor6 == "wasi" if __name__ == "__main__": unittest.main() diff --git a/gyp/pylib/gyp/generator/android.py b/gyp/pylib/gyp/generator/android.py index 9a79670214..2a63f412db 100644 --- a/gyp/pylib/gyp/generator/android.py +++ b/gyp/pylib/gyp/generator/android.py @@ -740,8 +740,8 @@ def ComputeOutput(self, spec): ) else: path = ( - "$(call intermediates-dir-for," - f"{self.android_class},{self.android_module},,,$(GYP_VAR_PREFIX))" + f"$(call intermediates-dir-for,{self.android_class}," + f"{self.android_module},,,$(GYP_VAR_PREFIX))" ) assert spec.get("product_dir") is None # TODO: not supported? diff --git a/gyp/pylib/gyp/generator/compile_commands_json.py b/gyp/pylib/gyp/generator/compile_commands_json.py index 0ffa3bb598..5d7f14da96 100644 --- a/gyp/pylib/gyp/generator/compile_commands_json.py +++ b/gyp/pylib/gyp/generator/compile_commands_json.py @@ -108,10 +108,14 @@ def GenerateOutput(target_list, target_dicts, data, params): cwd = os.path.dirname(build_file) AddCommandsForTarget(cwd, target, params, per_config_commands) + output_dir = None try: - output_dir = params["options"].generator_output - except (AttributeError, KeyError): - output_dir = params["generator_flags"].get("output_dir", "out") + # generator_output can be `None` on Windows machines, or even not + # defined in other cases + output_dir = params.get("options").generator_output + except AttributeError: + pass + output_dir = output_dir or params["generator_flags"].get("output_dir", "out") for configuration_name, commands in per_config_commands.items(): filename = os.path.join(output_dir, configuration_name, "compile_commands.json") gyp.common.EnsureDirExists(filename) diff --git a/gyp/pylib/gyp/generator/gypsh.py b/gyp/pylib/gyp/generator/gypsh.py index 625b6d65ca..8dfb1f1645 100644 --- a/gyp/pylib/gyp/generator/gypsh.py +++ b/gyp/pylib/gyp/generator/gypsh.py @@ -51,7 +51,7 @@ def GenerateOutput(target_list, target_dicts, data, params): # locals are available, and identify gypsh. banner = ( f"Python {sys.version} on {sys.platform}\nlocals.keys() = " - f"{repr(sorted(locals.keys()))}\ngypsh" + f"{sorted(locals.keys())!r}\ngypsh" ) code.interact(banner, local=locals) diff --git a/gyp/pylib/gyp/generator/make.py b/gyp/pylib/gyp/generator/make.py index 1b9974948e..392d900914 100644 --- a/gyp/pylib/gyp/generator/make.py +++ b/gyp/pylib/gyp/generator/make.py @@ -25,6 +25,7 @@ import os import re import subprocess +import sys import gyp import gyp.common import gyp.xcode_emulation @@ -378,7 +379,7 @@ def CalculateGeneratorInputInfo(params): CXXFLAGS.target ?= $(CPPFLAGS) $(CXXFLAGS) LINK.target ?= %(LINK.target)s LDFLAGS.target ?= $(LDFLAGS) -AR.target ?= $(AR) +AR.target ?= %(AR.target)s PLI.target ?= %(PLI.target)s # C++ apps need to be linked with g++. @@ -442,13 +443,21 @@ def CalculateGeneratorInputInfo(params): define fixup_dep # The depfile may not exist if the input file didn't have any #includes. touch $(depfile).raw -# Fixup path as in (1). -sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile) +# Fixup path as in (1).""" + + (r""" +sed -e "s|^$(notdir $@)|$@|" -re 's/\\\\([^$$])/\/\1/g' $(depfile).raw >> $(depfile)""" + if sys.platform == 'win32' else r""" +sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile)""") + + r""" # Add extra rules as in (2). # We remove slashes and replace spaces with new lines; # remove blank lines; -# delete the first line and append a colon to the remaining lines. -sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\ +# delete the first line and append a colon to the remaining lines.""" + + (""" +sed -e 's/\\\\\\\\$$//' -e 's/\\\\\\\\/\\//g' -e 'y| |\\n|' $(depfile).raw |\\""" + if sys.platform == 'win32' else """ +sed -e 's|\\\\||' -e 'y| |\\n|' $(depfile).raw |\\""") + + r""" grep -v '^$$' |\ sed -e 1d -e 's|$$|:|' \ >> $(depfile) @@ -724,6 +733,10 @@ def QuoteIfNecessary(string): string = '"' + string.replace('"', '\\"') + '"' return string +def replace_sep(string): + if sys.platform == 'win32': + string = string.replace('\\\\', '/').replace('\\', '/') + return string def StringToMakefileVariable(string): """Convert a string to a value that is acceptable as a make variable name.""" @@ -859,7 +872,7 @@ def Write( self.output = self.ComputeMacBundleOutput(spec) self.output_binary = self.ComputeMacBundleBinaryOutput(spec) else: - self.output = self.output_binary = self.ComputeOutput(spec) + self.output = self.output_binary = replace_sep(self.ComputeOutput(spec)) self.is_standalone_static_library = bool( spec.get("standalone_static_library", 0) @@ -985,7 +998,7 @@ def WriteSubMake(self, output_filename, makefile_path, targets, build_dir): # sub-project dir (see test/subdirectory/gyptest-subdir-all.py). self.WriteLn( "export builddir_name ?= %s" - % os.path.join(os.path.dirname(output_filename), build_dir) + % replace_sep(os.path.join(os.path.dirname(output_filename), build_dir)) ) self.WriteLn(".PHONY: all") self.WriteLn("all:") @@ -2063,7 +2076,7 @@ def WriteList(self, value_list, variable=None, prefix="", quoter=QuoteIfNecessar """ values = "" if value_list: - value_list = [quoter(prefix + value) for value in value_list] + value_list = [replace_sep(quoter(prefix + value)) for value in value_list] values = " \\\n\t" + " \\\n\t".join(value_list) self.fp.write(f"{variable} :={values}\n\n") @@ -2369,10 +2382,12 @@ def WriteAutoRegenerationRule(params, root_makefile, makefile_name, build_files) "\t$(call do_cmd,regen_makefile)\n\n" % { "makefile_name": makefile_name, - "deps": " ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files), - "cmd": gyp.common.EncodePOSIXShellList( - [gyp_binary, "-fmake"] + gyp.RegenerateFlags(options) + build_files_args + "deps": replace_sep( + " ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files) ), + "cmd": replace_sep(gyp.common.EncodePOSIXShellList( + [gyp_binary, "-fmake"] + gyp.RegenerateFlags(options) + build_files_args + )), } ) @@ -2435,33 +2450,52 @@ def CalculateMakefilePath(build_file, base_name): makefile_path = os.path.join( options.toplevel_dir, options.generator_output, makefile_name ) - srcdir = gyp.common.RelativePath(srcdir, options.generator_output) + srcdir = replace_sep(gyp.common.RelativePath(srcdir, options.generator_output)) srcdir_prefix = "$(srcdir)/" flock_command = "flock" copy_archive_arguments = "-af" makedep_arguments = "-MMD" + + # wasm-ld doesn't support --start-group/--end-group + link_commands = LINK_COMMANDS_LINUX + if flavor in ["wasi", "wasm"]: + link_commands = link_commands.replace(' -Wl,--start-group', '').replace( + ' -Wl,--end-group', '' + ) + + CC_target = replace_sep(GetEnvironFallback(("CC_target", "CC"), "$(CC)")) + AR_target = replace_sep(GetEnvironFallback(("AR_target", "AR"), "$(AR)")) + CXX_target = replace_sep(GetEnvironFallback(("CXX_target", "CXX"), "$(CXX)")) + LINK_target = replace_sep(GetEnvironFallback(("LINK_target", "LINK"), "$(LINK)")) + PLI_target = replace_sep(GetEnvironFallback(("PLI_target", "PLI"), "pli")) + CC_host = replace_sep(GetEnvironFallback(("CC_host", "CC"), "gcc")) + AR_host = replace_sep(GetEnvironFallback(("AR_host", "AR"), "ar")) + CXX_host = replace_sep(GetEnvironFallback(("CXX_host", "CXX"), "g++")) + LINK_host = replace_sep(GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)")) + PLI_host = replace_sep(GetEnvironFallback(("PLI_host", "PLI"), "pli")) + header_params = { "default_target": default_target, "builddir": builddir_name, "default_configuration": default_configuration, "flock": flock_command, "flock_index": 1, - "link_commands": LINK_COMMANDS_LINUX, + "link_commands": link_commands, "extra_commands": "", "srcdir": srcdir, "copy_archive_args": copy_archive_arguments, "makedep_args": makedep_arguments, - "CC.target": GetEnvironFallback(("CC_target", "CC"), "$(CC)"), - "AR.target": GetEnvironFallback(("AR_target", "AR"), "$(AR)"), - "CXX.target": GetEnvironFallback(("CXX_target", "CXX"), "$(CXX)"), - "LINK.target": GetEnvironFallback(("LINK_target", "LINK"), "$(LINK)"), - "PLI.target": GetEnvironFallback(("PLI_target", "PLI"), "pli"), - "CC.host": GetEnvironFallback(("CC_host", "CC"), "gcc"), - "AR.host": GetEnvironFallback(("AR_host", "AR"), "ar"), - "CXX.host": GetEnvironFallback(("CXX_host", "CXX"), "g++"), - "LINK.host": GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)"), - "PLI.host": GetEnvironFallback(("PLI_host", "PLI"), "pli"), + "CC.target": CC_target, + "AR.target": AR_target, + "CXX.target": CXX_target, + "LINK.target": LINK_target, + "PLI.target": PLI_target, + "CC.host": CC_host, + "AR.host": AR_host, + "CXX.host": CXX_host, + "LINK.host": LINK_host, + "PLI.host": PLI_host, } if flavor == "mac": flock_command = "./gyp-mac-tool flock" diff --git a/gyp/pylib/gyp/generator/msvs.py b/gyp/pylib/gyp/generator/msvs.py index 6f0f8c1ab6..6b5b24acc0 100644 --- a/gyp/pylib/gyp/generator/msvs.py +++ b/gyp/pylib/gyp/generator/msvs.py @@ -1779,8 +1779,8 @@ def _GetCopies(spec): fixed_dst = _FixPath(dst) full_dst = f'"{fixed_dst}\\{outer_dir}\\"' cmd = ( - f'mkdir {full_dst} 2>nul & cd "{_FixPath(base_dir)}" && ' - f'xcopy /e /f /y "{outer_dir}" {full_dst}' + f'mkdir {full_dst} 2>nul & cd "{_FixPath(base_dir)}" ' + f'&& xcopy /e /f /y "{outer_dir}" {full_dst}' ) copies.append( ( @@ -1896,9 +1896,8 @@ def _GetPlatformOverridesOfProject(spec): for config_name, c in spec["configurations"].items(): config_fullname = _ConfigFullName(config_name, c) platform = c.get("msvs_target_platform", _ConfigPlatform(c)) - fixed_config_fullname = ( - f"{_ConfigBaseName(config_name, _ConfigPlatform(c))}|{platform}" - ) + base_name = _ConfigBaseName(config_name, _ConfigPlatform(c)) + fixed_config_fullname = f"{base_name}|{platform}" if spec["toolset"] == "host" and generator_supports_multiple_toolsets: fixed_config_fullname = f"{config_name}|x64" config_platform_overrides[config_fullname] = fixed_config_fullname diff --git a/gyp/pylib/gyp/generator/ninja.py b/gyp/pylib/gyp/generator/ninja.py index 8ba341e96d..0146c49962 100644 --- a/gyp/pylib/gyp/generator/ninja.py +++ b/gyp/pylib/gyp/generator/ninja.py @@ -11,6 +11,7 @@ import os.path import re import signal +import shutil import subprocess import sys import gyp @@ -2210,6 +2211,7 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, config_name options = params["options"] flavor = gyp.common.GetFlavor(params) generator_flags = params.get("generator_flags", {}) + generate_compile_commands = generator_flags.get("compile_commands", False) # build_dir: relative path from source root to our output files. # e.g. "out/Debug" @@ -2878,6 +2880,35 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, config_name master_ninja_file.close() + if generate_compile_commands: + compile_db = GenerateCompileDBWithNinja(toplevel_build) + compile_db_file = OpenOutput( + os.path.join(toplevel_build, "compile_commands.json") + ) + compile_db_file.write(json.dumps(compile_db, indent=2)) + compile_db_file.close() + + +def GenerateCompileDBWithNinja(path, targets=["all"]): + """Generates a compile database using ninja. + + Args: + path: The build directory to generate a compile database for. + targets: Additional targets to pass to ninja. + + Returns: + List of the contents of the compile database. + """ + ninja_path = shutil.which("ninja") + if ninja_path is None: + raise Exception("ninja not found in PATH") + json_compile_db = subprocess.check_output( + [ninja_path, "-C", path] + + targets + + ["-t", "compdb", "cc", "cxx", "objc", "objcxx"] + ) + return json.loads(json_compile_db) + def PerformBuild(data, configurations, params): options = params["options"] diff --git a/gyp/pylib/gyp/generator/ninja_test.py b/gyp/pylib/gyp/generator/ninja_test.py index 7d180685b2..15cddfdf24 100644 --- a/gyp/pylib/gyp/generator/ninja_test.py +++ b/gyp/pylib/gyp/generator/ninja_test.py @@ -6,6 +6,7 @@ """ Unit tests for the ninja.py file. """ +from pathlib import Path import sys import unittest @@ -50,6 +51,17 @@ def test_BinaryNamesLinux(self): writer.ComputeOutputFileName(spec, "static_library").endswith(".a") ) + def test_GenerateCompileDBWithNinja(self): + build_dir = ( + Path(__file__).resolve().parent.parent.parent.parent / "data" / "ninja" + ) + compile_db = ninja.GenerateCompileDBWithNinja(build_dir) + assert len(compile_db) == 1 + assert compile_db[0]["directory"] == str(build_dir) + assert compile_db[0]["command"] == "cc my.in my.out" + assert compile_db[0]["file"] == "my.in" + assert compile_db[0]["output"] == "my.out" + if __name__ == "__main__": unittest.main() diff --git a/gyp/pylib/gyp/input.py b/gyp/pylib/gyp/input.py index 0b56c72750..7150269cda 100644 --- a/gyp/pylib/gyp/input.py +++ b/gyp/pylib/gyp/input.py @@ -1135,16 +1135,16 @@ def EvalCondition(condition, conditions_key, phase, variables, build_file): true_dict = condition[i + 1] if type(true_dict) is not dict: raise GypError( - f"{conditions_key} {cond_expr} must be followed by a dictionary, not " - f"{type(true_dict)}" + f"{conditions_key} {cond_expr} must be followed by a dictionary, " + f"not {type(true_dict)}" ) if len(condition) > i + 2 and type(condition[i + 2]) is dict: false_dict = condition[i + 2] i = i + 3 if i != len(condition): raise GypError( - f"{conditions_key} {cond_expr} has {len(condition) - i} " - "unexpected trailing items" + f"{conditions_key} {cond_expr} has " + f"{len(condition) - i} unexpected trailing items" ) else: false_dict = None @@ -2536,6 +2536,8 @@ def ProcessListFiltersInDict(name, the_dict): lists = [] del_lists = [] for key, value in the_dict.items(): + if not key: + continue operation = key[-1] if operation not in {"!", "/"}: continue diff --git a/gyp/pylib/gyp/msvs_emulation.py b/gyp/pylib/gyp/msvs_emulation.py index 847d1b8dc1..adda5a0273 100644 --- a/gyp/pylib/gyp/msvs_emulation.py +++ b/gyp/pylib/gyp/msvs_emulation.py @@ -830,14 +830,15 @@ def _GetLdManifestFlags( ("VCLinkerTool", "UACUIAccess"), config, default="false" ) + level = execution_level_map[execution_level] inner = f""" - + -""" # noqa: E501 +""" else: inner = "" diff --git a/gyp/pylib/gyp/xcode_emulation.py b/gyp/pylib/gyp/xcode_emulation.py index 29caf1ce7f..5f2c097f63 100644 --- a/gyp/pylib/gyp/xcode_emulation.py +++ b/gyp/pylib/gyp/xcode_emulation.py @@ -579,7 +579,8 @@ def GetCflags(self, configname, arch=None): sdk_root = self._SdkPath() if "SDKROOT" in self._Settings() and sdk_root: - cflags.append("-isysroot %s" % sdk_root) + cflags.append("-isysroot") + cflags.append(sdk_root) if self.header_map_path: cflags.append("-I%s" % self.header_map_path) @@ -664,7 +665,8 @@ def GetCflags(self, configname, arch=None): # TODO: Supporting fat binaries will be annoying. self._WarnUnimplemented("ARCHS") archs = ["i386"] - cflags.append("-arch " + archs[0]) + cflags.append("-arch") + cflags.append(archs[0]) if archs[0] in ("i386", "x86_64"): if self._Test("GCC_ENABLE_SSE3_EXTENSIONS", "YES", default="NO"): @@ -924,17 +926,15 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None): self._AppendPlatformVersionMinFlags(ldflags) if "SDKROOT" in self._Settings() and self._SdkPath(): - ldflags.append("-isysroot " + self._SdkPath()) + ldflags.append("-isysroot") + ldflags.append(self._SdkPath()) for library_path in self._Settings().get("LIBRARY_SEARCH_PATHS", []): ldflags.append("-L" + gyp_to_build_path(library_path)) if "ORDER_FILE" in self._Settings(): - ldflags.append( - "-Wl,-order_file " - + "-Wl," - + gyp_to_build_path(self._Settings()["ORDER_FILE"]) - ) + ldflags.append("-Wl,-order_file") + ldflags.append("-Wl," + gyp_to_build_path(self._Settings()["ORDER_FILE"])) if not gyp.common.CrossCompileRequested(): if arch is not None: @@ -946,7 +946,9 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None): # TODO: Supporting fat binaries will be annoying. self._WarnUnimplemented("ARCHS") archs = ["i386"] - ldflags.append("-arch " + archs[0]) + # Avoid quoting the space between -arch and the arch name + ldflags.append("-arch") + ldflags.append(archs[0]) # Xcode adds the product directory by default. # Rewrite -L. to -L./ to work around http://www.openradar.me/25313838 @@ -954,7 +956,8 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None): install_name = self.GetInstallName() if install_name and self.spec["type"] != "loadable_module": - ldflags.append("-install_name " + install_name.replace(" ", r"\ ")) + ldflags.append("-install_name") + ldflags.append(install_name.replace(" ", r"\ ")) for rpath in self._Settings().get("LD_RUNPATH_SEARCH_PATHS", []): ldflags.append("-Wl,-rpath," + rpath) @@ -971,7 +974,8 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None): platform_root = self._XcodePlatformPath(configname) if sdk_root and platform_root: ldflags.append("-F" + platform_root + "/Developer/Library/Frameworks/") - ldflags.append("-framework XCTest") + ldflags.append("-framework") + ldflags.append("XCTest") is_extension = self._IsIosAppExtension() or self._IsIosWatchKitExtension() if sdk_root and is_extension: @@ -987,7 +991,8 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None): + "/System/Library/PrivateFrameworks/PlugInKit.framework/PlugInKit" ) else: - ldflags.append("-e _NSExtensionMain") + ldflags.append("-e") + ldflags.append("_NSExtensionMain") ldflags.append("-fapplication-extension") self._Appendf(ldflags, "CLANG_CXX_LIBRARY", "-stdlib=%s") diff --git a/gyp/pylib/gyp/xcode_emulation_test.py b/gyp/pylib/gyp/xcode_emulation_test.py new file mode 100644 index 0000000000..98b02320d5 --- /dev/null +++ b/gyp/pylib/gyp/xcode_emulation_test.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +"""Unit tests for the xcode_emulation.py file.""" + +from gyp.xcode_emulation import XcodeSettings +import sys +import unittest + + +class TestXcodeSettings(unittest.TestCase): + def setUp(self): + if sys.platform != "darwin": + self.skipTest("This test only runs on macOS") + + def test_GetCflags(self): + target = { + "type": "static_library", + "configurations": { + "Release": {}, + }, + } + configuration_name = "Release" + xcode_settings = XcodeSettings(target) + cflags = xcode_settings.GetCflags(configuration_name, "arm64") + + # Do not quote `-arch arm64` with spaces in one string. + self.assertEqual( + cflags, + ["-fasm-blocks", "-mpascal-strings", "-Os", "-gdwarf-2", "-arch", "arm64"], + ) + + def GypToBuildPath(self, path): + return path + + def test_GetLdflags(self): + target = { + "type": "static_library", + "configurations": { + "Release": {}, + }, + } + configuration_name = "Release" + xcode_settings = XcodeSettings(target) + ldflags = xcode_settings.GetLdflags( + configuration_name, "PRODUCT_DIR", self.GypToBuildPath, "arm64" + ) + + # Do not quote `-arch arm64` with spaces in one string. + self.assertEqual(ldflags, ["-arch", "arm64", "-LPRODUCT_DIR"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/gyp/pyproject.toml b/gyp/pyproject.toml index 7183e07d3c..def9858e44 100644 --- a/gyp/pyproject.toml +++ b/gyp/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "gyp-next" -version = "0.16.1" +version = "0.18.1" authors = [ { name="Node.js contributors", email="ryzokuken@disroot.org" }, ] @@ -12,8 +12,7 @@ description = "A fork of the GYP build system for use in the Node.js projects" readme = "README.md" license = { file="LICENSE" } requires-python = ">=3.8" -# The Python module "packaging" is vendored in the "pylib/packaging" directory to support Python >= 3.12. -# dependencies = ["packaging>=23.1"] # Uncomment this line if the vendored version is removed. +dependencies = ["packaging>=24.0", "setuptools>=69.5.1"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", @@ -29,7 +28,7 @@ classifiers = [ ] [project.optional-dependencies] -dev = ["flake8", "ruff", "pytest"] +dev = ["pytest", "ruff"] [project.scripts] gyp = "gyp:script_main" @@ -38,7 +37,12 @@ gyp = "gyp:script_main" "Homepage" = "https://github.com/nodejs/gyp-next" [tool.ruff] -lint.select = [ +extend-exclude = ["pylib/packaging"] +line-length = 88 +target-version = "py37" + +[tool.ruff.lint] +select = [ "C4", # flake8-comprehensions "C90", # McCabe cyclomatic complexity "DTZ", # flake8-datetimez @@ -87,7 +91,7 @@ lint.select = [ # "T20", # flake8-print # "TRY", # tryceratops ] -lint.ignore = [ +ignore = [ "E721", "PLC1901", "PLR0402", @@ -101,9 +105,6 @@ lint.ignore = [ "RUF012", "UP031", ] -extend-exclude = ["pylib/packaging"] -line-length = 88 -target-version = "py37" [tool.ruff.lint.mccabe] max-complexity = 101 diff --git a/gyp/release-please-config.json b/gyp/release-please-config.json new file mode 100644 index 0000000000..b6cad32a2d --- /dev/null +++ b/gyp/release-please-config.json @@ -0,0 +1,11 @@ +{ + "last-release-sha": "78756421b0d7bb335992a9c7d26ba3cc8b619708", + "packages": { + ".": { + "release-type": "python", + "package-name": "gyp-next", + "bump-minor-pre-major": true, + "include-component-in-tag": false + } + } +} From e89dc9abd8ffc859e2c996d452509c59d4f00a61 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Thu, 6 Jun 2024 23:34:50 +0800 Subject: [PATCH 2/3] ci: setup ninja --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6c4541f53a..1e8e8bc8cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -115,6 +115,7 @@ jobs: python-version: ${{ matrix.python }} env: PYTHON_VERSION: ${{ matrix.python }} # Why do this? + - uses: seanmiddleditch/gha-setup-ninja@v4 - name: Install Dependencies run: | npm install From e84d8fe1c5f92a25f144475747affedeb9379352 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Fri, 7 Jun 2024 22:03:23 +0800 Subject: [PATCH 3/3] ci: visual-studio job use python 3.12 --- .github/workflows/visual-studio.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/visual-studio.yml b/.github/workflows/visual-studio.yml index 19993a57f9..0f51f9c8d0 100644 --- a/.github/workflows/visual-studio.yml +++ b/.github/workflows/visual-studio.yml @@ -25,10 +25,12 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v4 + - name: Use Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" - name: Install Dependencies run: npm install - name: Run Node tests shell: pwsh - run: | - $pythonLocation = (Get-Command python).Source - npm run test --python="${pythonLocation}" --msvs-version="${{ matrix.msvs-version }}" + run: npm run test --python="${env:pythonLocation}\\python.exe" --msvs-version="${{ matrix.msvs-version }}"