diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f08b668dd..ff7fe4ac8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,23 +3,23 @@ name: Build on: [push, pull_request] jobs: - build-and-deploy: - permissions: - contents: 'read' - id-token: 'write' - deployments: 'write' + client: strategy: - matrix: - node-version: - - '16.x' - runs-on: ubuntu-latest + matrix: + node-version: + - '16.x' + os: + - 'ubuntu-latest' + - 'windows-latest' + - 'macos-latest' + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v2 with: path: "**/node_modules" key: ${{ runner.os }}-${{ matrix.node-version }}-node_modules-${{ hashFiles('**/package-lock.json') }} @@ -27,43 +27,226 @@ jobs: - name: Typecheck with TypeScript run: npm run typecheck - name: Build client bundles - run: node ./config/esbuild-cli.js --config=min --no-typecheck --define STATE_SERVERS=$(cat config/state_servers.json | tr -d " \t\n\r") - - run: cp -r ./dist/min appengine/frontend/static/ - - name: Get branch name (merge) - if: github.event_name != 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV - - name: Get branch name (pull request) - if: github.event_name == 'pull_request' + run: npm run build -- --no-typecheck + - name: Build JavaScript module + run: npm run build-module -- --no-typecheck + - name: Build Python client bundles + run: npm run build-python -- --no-typecheck + - uses: ./.github/actions/setup-firefox + - name: Run JavaScript tests (including WebGL) + # Swiftshader, used by Chrome headless, crashes when running Neuroglancer + # tests. + # + # The only reliable headless configuration is Firefox on Linux under + # xvfb-run, which uses Mesa software rendering. + if: startsWith(runner.os, 'Linux') + run: xvfb-run --auto-servernum --server-args='-screen 0 1024x768x24' npm run test -- --browsers Firefox + - name: Run JavaScript tests (excluding WebGL) + if: ${{ !startsWith(runner.os, 'Linux') }} + run: npm run test -- --browsers ChromeHeadless --define=NEUROGLANCER_SKIP_WEBGL_TESTS + - name: Run JavaScript benchmarks + run: npm run benchmark + + # Builds Python package and runs Python tests + # + # On ubuntu-latest, this also runs browser-based tests. On Mac OS and + # Windows, this only runs tests that do not require a browser, since a working + # headless WebGL2 implementation is not available on Github actions. + python-tox: + strategy: + matrix: + python-version: + - '3.8' + - '3.9' + - '3.10' + - '3.11' + node-version: + - '16.x' + os: + - 'ubuntu-latest' + - 'windows-latest' + - 'macos-latest' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + # Need full history to determine version number. + fetch-depth: 0 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + with: + path: "**/node_modules" + key: ${{ runner.os }}-${{ matrix.node-version }}-node_modules-${{ hashFiles('**/package-lock.json') }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('setup.py') }} + # Uncomment the action below for an interactive shell + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + - name: Install Python packaging/test tools + run: python -m pip install --upgrade pip tox wheel numpy pytest + - uses: ./.github/actions/setup-firefox + - name: Test with tox + run: tox -e ${{ fromJSON('["skip-browser-tests","firefox-xvfb"]')[runner.os == 'Linux'] }} + env: + TOX_TESTENV_PASSENV: GH_TOKEN + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Verify that editable install works + - name: Install in editable form + run: pip install -e . --config-settings editable_mode=strict + - name: Run Python tests against editable install (excluding WebGL) + working-directory: python/tests + run: pytest -vv --skip-browser-tests + + python-build-package: + strategy: + matrix: + include: + - os: 'ubuntu-latest' + cibw_build: '*' + wheel_identifier: 'linux' + node-version: '16.x' + - os: 'windows-latest' + cibw_build: '*' + wheel_identifier: 'windows' + node-version: '16.x' + - os: 'macos-latest' + cibw_build: '*_x86_64' + wheel_identifier: 'macos_x86_64' + node-version: '16.x' + - os: 'macos-latest' + cibw_build: '*_arm64' + wheel_identifier: 'macos_arm64' + node-version: '16.x' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + # Need full history to determine version number. + fetch-depth: 0 + - name: Use Node.js + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + - uses: actions/cache@v2 + with: + path: "**/node_modules" + key: ${{ runner.os }}-${{ matrix.node-version }}-node_modules-${{ hashFiles('**/package-lock.json') }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-buildwheel-${{ hashFiles('setup.py') }} + - run: npm install + - run: | + build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" + npm run build-python -- --no-typecheck --define NEUROGLANCER_BUILD_INFO="${build_info}" shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV - - run: echo "BRANCH_NAME_URL=$(echo ${{ env.BRANCH_NAME }} | tr / - | tr _ -)" >> $GITHUB_ENV - - name: start deployment - uses: bobheadxi/deployments@v1 - id: deployment - with: - step: start - token: ${{ secrets.GITHUB_TOKEN }} - env: ${{ env.BRANCH_NAME }} - desc: Setting up staging deployment for ${{ env.BRANCH_NAME }} - - id: 'auth' - uses: 'google-github-actions/auth@v1' - with: - workload_identity_provider: 'projects/483670036293/locations/global/workloadIdentityPools/neuroglancer-github/providers/github' - service_account: 'chris-apps-deploy@seung-lab.iam.gserviceaccount.com' - - id: deploy - uses: google-github-actions/deploy-appengine@main - with: - version: ${{ env.BRANCH_NAME_URL }} - deliverables: appengine/frontend/app.yaml - promote: false - - name: update deployment status - uses: bobheadxi/deployments@v1 - if: always() - with: - step: finish - token: ${{ secrets.GITHUB_TOKEN }} - env: ${{ steps.deployment.outputs.env }} - env_url: ${{ steps.deploy.outputs.url }} - status: ${{ job.status }} - deployment_id: ${{ steps.deployment.outputs.deployment_id }} + - name: Check for dirty working directory + run: git diff --exit-code + - name: Build Python source distribution (sdist) + run: python setup.py sdist --format gztar + if: ${{ runner.os == 'Linux' }} + - name: Install cibuildwheel + run: pip install cibuildwheel + - name: Build Python wheels + run: bash -xve ./python/build_tools/cibuildwheel.sh + env: + # On Linux, share pip cache with manylinux docker containers + CIBW_ENVIRONMENT_LINUX: PIP_CACHE_DIR=/host${{ steps.pip-cache.outputs.dir }} + CIBW_BEFORE_ALL_LINUX: /project/python/build_tools/cibuildwheel_linux_cache_setup.sh /host${{ steps.pip-cache.outputs.dir }} + CIBW_BUILD: ${{ matrix.cibw_build }} + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v2 + with: + name: python-wheels-${{ matrix.wheel_identifier }} + path: | + dist/*.whl + dist/*.tar.gz + + python-publish-package: + # Only publish package on push to tag or default branch. + if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master') }} + runs-on: ubuntu-latest + needs: + - 'python-build-package' + steps: + - uses: actions/download-artifact@v2 + with: + name: python-wheels-linux + path: dist + - uses: actions/download-artifact@v2 + with: + name: python-wheels-macos_x86_64 + path: dist + - uses: actions/download-artifact@v2 + with: + name: python-wheels-macos_arm64 + path: dist + - uses: actions/download-artifact@v2 + with: + name: python-wheels-windows + path: dist + # - name: Publish to PyPI (test server) + # uses: pypa/gh-action-pypi-publish@54b39fb9371c0b3a6f9f14bb8a67394defc7a806 # 2020-09-25 + # with: + # user: __token__ + # password: ${{ secrets.pypi_test_token }} + - name: Publish to PyPI (main server) + uses: pypa/gh-action-pypi-publish@54b39fb9371c0b3a6f9f14bb8a67394defc7a806 # 2020-09-25 + with: + user: __token__ + password: ${{ secrets.pypi_token }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + ngauth: + strategy: + matrix: + go-version: ['1.19'] + os: + - ubuntu-latest + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Setup go ${{ matrix.go-version }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + - run: go build . + working-directory: ngauth_server + wasm: + # Ensures that .wasm files are reproducible. + strategy: + matrix: + os: + - ubuntu-latest + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - run: ./src/neuroglancer/mesh/draco/build.sh + - run: ./src/neuroglancer/sliceview/compresso/build.sh + - run: ./src/neuroglancer/sliceview/png/build.sh + # Check that there are no differences. + - run: git diff --exit-code diff --git a/appengine/frontend/app.yaml b/appengine/frontend/app.yaml deleted file mode 100644 index 7c9baf75a..000000000 --- a/appengine/frontend/app.yaml +++ /dev/null @@ -1,32 +0,0 @@ -runtime: python27 -api_version: 1 -threadsafe: true - -# The skip_files element specifies which files -# in the application directory are not to be -# uploaded to App Engine. -skip_files: -- ^(.*/)?#.*#$ -- ^(.*/)?.*~$ -- ^(.*/)?.*\.py[co]$ -- ^(.*/)?.*/RCS/.*$ -- ^(.*/)?\..*$ -- ^node_modules$ - -service: neuroglancer - -handlers: -# Handle the main page by serving the index page. -# Note the $ to specify the end of the path, since app.yaml does prefix matching. -- url: /$ - static_files: static/index.html - upload: static/index.html - login: optional - secure: always - redirect_http_response_code: 301 - -- url: / - static_dir: static - login: optional - secure: always - redirect_http_response_code: 301 diff --git a/config/state_servers.json b/config/state_servers.json deleted file mode 100644 index c61efbd86..000000000 --- a/config/state_servers.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cave": { - "url": "middleauth+https://global.daf-apis.com/nglstate/api/v1/post", - "default": true - } -}