diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f211774c..39e39f263 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,21 +1,17 @@ name: Build -on: - push: - branches: - - master - tags: - - v** - pull_request: +on: [push] jobs: - client: + build-and-deploy: + permissions: + contents: "read" + id-token: "write" + deployments: "write" strategy: matrix: os: - "ubuntu-latest" - - "windows-latest" - - "macos-latest" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -30,228 +26,52 @@ jobs: package-lock.json examples/**/package-lock.json - run: npm install - - run: npm run format:fix - - name: Check for dirty working directory - run: git diff --exit-code - - run: npm run lint:check - name: Typecheck with TypeScript run: npm run typecheck - - name: Build client bundles - run: | - build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" - npm run build -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" - shell: bash - - name: Build Python client bundles - run: npm run build-python -- --no-typecheck --no-lint - - run: npm run build-package - - run: npm publish --dry-run - working-directory: dist/package - - uses: ./.github/actions/setup-firefox - - name: Run JavaScript tests (including WebGL) - run: npm test - - name: Run JavaScript benchmarks - run: npm run benchmark - - name: Upload NPM package as artifact - uses: actions/upload-artifact@v4 - with: - name: npm-package - path: dist/package - if: ${{ runner.os == 'Linux' }} - - name: Upload client as artifact - uses: actions/upload-artifact@v4 - with: - name: client - path: dist/client - if: ${{ runner.os == 'Linux' }} - - # Builds Python package and runs Python tests - # - # On ubuntu-latest, this also runs browser-based tests. On Mac OS and - # Windows, this only runs tests that do not require a browser, since a working - # headless WebGL2 implementation is not available on Github actions. - python-tests: - strategy: - matrix: - python-version: - - "3.9" - - "3.12" - os: - - "ubuntu-latest" - - "windows-latest" - - "macos-latest" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - # Need full history to determine version number. - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 20.x - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - # Uncomment the action below for an interactive shell - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - - name: Install Python packaging/test tools - run: python -m pip install --upgrade pip tox nox wheel numpy -r python/requirements-test.txt - - uses: ./.github/actions/setup-firefox - - run: nox -s lint format mypy - - name: Check for dirty working directory - run: git diff --exit-code - - name: Test with tox - run: tox -e ${{ fromJSON('["skip-browser-tests","firefox-xvfb"]')[runner.os == 'Linux'] }} - # Verify that editable install works - - name: Install in editable form - run: pip install -e . --config-settings editable_mode=strict - - name: Run Python tests against editable install (excluding WebGL) - working-directory: python/tests - run: pytest -vv --skip-browser-tests - - python-build-package: - strategy: - matrix: - include: - - os: "ubuntu-latest" - cibw_build: "*" - wheel_identifier: "linux" - - os: "windows-latest" - cibw_build: "*" - wheel_identifier: "windows" - - os: "macos-14" - cibw_build: "*_x86_64" - wheel_identifier: "macos_x86_64" - - os: "macos-14" - cibw_build: "*_arm64" - wheel_identifier: "macos_arm64" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - # Need full history to determine version number. - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 20.x - cache: "npm" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Get pip cache dir - id: pip-cache - run: | - echo "dir=$(pip cache dir)" >> "$GITHUB_OUTPUT" - - run: npm install - run: | build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" npm run build-python -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" shell: bash - name: Check for dirty working directory run: git diff --exit-code - - name: Install setuptools - run: pip install setuptools - - name: Build Python source distribution (sdist) - run: python setup.py sdist --format gztar - if: ${{ runner.os == 'Linux' }} - - name: Install cibuildwheel - run: pip install cibuildwheel - - name: Build Python wheels - run: bash -xve ./python/build_tools/cibuildwheel.sh - env: - # On Linux, share pip cache with manylinux docker containers - CIBW_ENVIRONMENT_LINUX: PIP_CACHE_DIR=/host${{ steps.pip-cache.outputs.dir }} - CIBW_BEFORE_ALL_LINUX: /project/python/build_tools/cibuildwheel_linux_cache_setup.sh /host${{ steps.pip-cache.outputs.dir }} - CIBW_BUILD: ${{ matrix.cibw_build }} - - name: Upload wheels as artifacts - uses: actions/upload-artifact@v4 - with: - name: python-wheels-${{ matrix.wheel_identifier }} - path: | - dist/*.whl - dist/*.tar.gz - - publish-package: - # Only publish package on push to tag or default branch. - if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master') }} - runs-on: ubuntu-latest - needs: - - "client" - - "python-build-package" - steps: - - uses: actions/checkout@v4 - - name: Use Node.js - uses: actions/setup-node@v4 - with: - node-version: 20.x - registry-url: "https://registry.npmjs.org" - - uses: actions/download-artifact@v4 - with: - pattern: python-wheels-* - path: dist - merge-multiple: true - - uses: actions/download-artifact@v4 - with: - name: npm-package - path: npm-package - # - name: Publish to PyPI (test server) - # uses: pypa/gh-action-pypi-publish@54b39fb9371c0b3a6f9f14bb8a67394defc7a806 # 2020-09-25 - # with: - # user: __token__ - # password: ${{ secrets.pypi_test_token }} - - name: Publish to PyPI (main server) - uses: pypa/gh-action-pypi-publish@54b39fb9371c0b3a6f9f14bb8a67394defc7a806 # 2020-09-25 - with: - user: __token__ - password: ${{ secrets.pypi_token }} - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - - name: Publish to NPM registry - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - run: npm publish - working-directory: npm-package - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - # Download dist/client after publishing to PyPI, because PyPI publish - # action fails if dist/client directory is present. - - uses: actions/download-artifact@v4 - with: - name: client - path: dist/client - - name: Publish to Firebase hosting - uses: FirebaseExtended/action-hosting-deploy@v0 - with: - firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}" - projectId: neuroglancer-demo - channelId: live - - ngauth: - strategy: - matrix: - os: - - ubuntu-latest - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: ngauth_server/go.mod - cache-dependency-path: ngauth_server/go.sum - - run: go build . - working-directory: ngauth_server - wasm: - # Ensures that .wasm files are reproducible. - strategy: - matrix: - os: - - ubuntu-latest - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - run: ./src/mesh/draco/build.sh - - run: ./src/sliceview/compresso/build.sh - - run: ./src/sliceview/png/build.sh - # Check that there are no differences. - - run: git diff --exit-code + - name: Build client bundles + run: npm run build -- --no-typecheck --no-lint --define STATE_SERVERS=$(cat config/state_servers.json | tr -d " \t\n\r") + - run: cp -r ./dist/client appengine/frontend/static/ + - name: Get branch name (merge) + if: github.event_name != 'pull_request' + shell: bash + run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV + - name: Get branch name (pull request) + if: github.event_name == 'pull_request' + shell: bash + run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV + - run: echo "BRANCH_NAME_URL=$(echo ${{ env.BRANCH_NAME }} | tr / - | tr _ -)" >> $GITHUB_ENV + - name: start deployment + uses: bobheadxi/deployments@v1 + id: deployment + with: + step: start + token: ${{ secrets.GITHUB_TOKEN }} + env: ${{ env.BRANCH_NAME }} + desc: Setting up staging deployment for ${{ env.BRANCH_NAME }} + - id: "auth" + uses: "google-github-actions/auth@v1" + with: + workload_identity_provider: "projects/483670036293/locations/global/workloadIdentityPools/neuroglancer-github/providers/github" + service_account: "chris-apps-deploy@seung-lab.iam.gserviceaccount.com" + - id: deploy + uses: google-github-actions/deploy-appengine@main + with: + version: ${{ env.BRANCH_NAME_URL }} + deliverables: appengine/frontend/app.yaml + promote: false + - name: update deployment status + uses: bobheadxi/deployments@v1 + if: always() + with: + step: finish + token: ${{ secrets.GITHUB_TOKEN }} + env: ${{ steps.deployment.outputs.env }} + env_url: ${{ steps.deploy.outputs.url }} + status: ${{ job.status }} + deployment_id: ${{ steps.deployment.outputs.deployment_id }} diff --git a/.github/workflows/build_preview.yml b/.github/workflows/build_preview.yml deleted file mode 100644 index 9dcac7d1e..000000000 --- a/.github/workflows/build_preview.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Build preview - -on: - pull_request: - -jobs: - upload: - strategy: - matrix: - node-version: - - "20.x" - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: | - build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" - npm run build -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" - - name: Upload client as artifact - uses: actions/upload-artifact@v4 - with: - name: client - path: | - dist/client/* diff --git a/.github/workflows/deploy_preview.yml b/.github/workflows/deploy_preview.yml deleted file mode 100644 index 545bda17c..000000000 --- a/.github/workflows/deploy_preview.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Deploy preview - -on: - workflow_run: - workflows: ["Build preview"] - types: [completed] - -jobs: - deploy: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: "Create commit status" - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const commitId = "${{ github.event.workflow_run.head_commit.id }}"; - await github.rest.repos.createCommitStatus({ - context: "client-preview", - owner: context.repo.owner, - repo: context.repo.repo, - sha: commitId, - state: "pending", - description: `Creating preview`, - target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, - }); - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: client - path: dist/client - github-token: "${{ secrets.GITHUB_TOKEN }}" - run-id: "${{ github.event.workflow_run.id }}" - - name: Get PR ID - # https://github.com/orgs/community/discussions/25220#discussioncomment-7532132 - id: pr-id - run: | - PR_ID=$(gh run view -R ${{ github.repository }} ${{ github.event.workflow_run.id }} | grep -oP '#[0-9]+ . ${{ github.event.workflow_run.id }}' | grep -oP '#[0-9]+' | cut -c 2-) - echo "pr-id=${PR_ID}" >> $GITHUB_OUTPUT - env: - GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - uses: FirebaseExtended/action-hosting-deploy@v0 - id: deploy - with: - repoToken: "${{ secrets.GITHUB_TOKEN }}" - firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}" - expires: 30d - channelId: "pr${{ steps.pr-id.outputs.pr-id }}" - projectId: neuroglancer-demo - - name: "Update commit status" - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const expires = new Date("${{ steps.deploy.outputs.expire_time }}"); - const commitId = "${{ github.event.workflow_run.head_commit.id }}"; - await github.rest.repos.createCommitStatus({ - context: "client-preview", - owner: context.repo.owner, - repo: context.repo.repo, - sha: commitId, - state: "success", - target_url: "${{ steps.deploy.outputs.details_url }}", - description: `Preview created, expires at: ${expires.toISOString()}`, - }); diff --git a/appengine/frontend/.gcloudignore b/appengine/frontend/.gcloudignore new file mode 100644 index 000000000..c2edb00e9 --- /dev/null +++ b/appengine/frontend/.gcloudignore @@ -0,0 +1,5 @@ +.gcloudignore +.envrc +.git +.gitignore +node_modules/ \ No newline at end of file diff --git a/appengine/frontend/app.yaml b/appengine/frontend/app.yaml new file mode 100644 index 000000000..7d375d9d1 --- /dev/null +++ b/appengine/frontend/app.yaml @@ -0,0 +1,19 @@ +runtime: python312 + +service: neuroglancer + +handlers: + # Handle the main page by serving the index page. + # Note the $ to specify the end of the path, since app.yaml does prefix matching. + - url: /$ + static_files: static/index.html + upload: static/index.html + login: optional + secure: always + redirect_http_response_code: 301 + + - url: / + static_dir: static + login: optional + secure: always + redirect_http_response_code: 301 diff --git a/config/state_servers.json b/config/state_servers.json new file mode 100644 index 000000000..c61efbd86 --- /dev/null +++ b/config/state_servers.json @@ -0,0 +1,6 @@ +{ + "cave": { + "url": "middleauth+https://global.daf-apis.com/nglstate/api/v1/post", + "default": true + } +}