refactor: split ak._do into meta and content
#4034
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docs | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| release: | |
| types: [released] | |
| workflow_dispatch: | |
| concurrency: | |
| group: 'docs-${{ github.head_ref || github.run_id }}' | |
| cancel-in-progress: true | |
| env: | |
| X86_64_PYTHON_VERSION: "3.11.0" | |
| SOURCE_DATE_EPOCH: "1668811211" | |
| jobs: | |
| pyodide-python-version: | |
| name: Determine Pyodide Python version | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| python-version: ${{ steps.retrieve-python-version.outputs.python-version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Cache python version | |
| id: cache-pyodide-python-version | |
| uses: actions/cache@v3 | |
| with: | |
| path: pyodide-python-version | |
| key: ${{ runner.os }}-docs-pyodide-python-version-${{ hashFiles('docs/requirements-wasm.txt') }} | |
| - name: Setup Python | |
| if: steps.cache-pyodide-python-version.outputs.cache-hit != 'true' | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{ env.X86_64_PYTHON_VERSION }}" | |
| - name: Install dependencies | |
| if: steps.cache-pyodide-python-version.outputs.cache-hit != 'true' | |
| run: python3 -m pip install -r docs/requirements-wasm.txt | |
| - name: Determine Python version | |
| if: steps.cache-pyodide-python-version.outputs.cache-hit != 'true' | |
| id: compute-python-version | |
| run: | | |
| # Save Python version | |
| PYTHON_VERSION=$(pyodide config get python_version) | |
| echo $PYTHON_VERSION > pyodide-python-version | |
| - name: Retrieve Python version | |
| id: retrieve-python-version | |
| run: | | |
| PYTHON_VERSION=$(cat pyodide-python-version) | |
| echo "python-version=$PYTHON_VERSION" >> "$GITHUB_OUTPUT" | |
| awkward-cpp-wasm: | |
| name: Build C++ WASM | |
| runs-on: ubuntu-22.04 | |
| needs: [pyodide-python-version] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{ needs.pyodide-python-version.outputs.python-version }}" | |
| - name: Generate build files | |
| run: pipx run nox -s prepare -- --headers --signatures | |
| - name: Cache wheel | |
| id: cache-awkward-cpp-wasm-wheel | |
| uses: actions/cache@v3 | |
| with: | |
| path: ./awkward-cpp/dist | |
| key: ${{ runner.os }}-"${{ needs.pyodide-python-version.outputs.python-version }}-awkward-cpp-wasm-${{ hashFiles('awkward-cpp/**') }} | |
| - name: Install dependencies | |
| if: steps.cache-awkward-cpp-wasm-wheel.outputs.cache-hit != 'true' | |
| run: python3 -m pip install -r docs/requirements-wasm.txt | |
| - name: Determine EMSDK version | |
| if: steps.cache-awkward-cpp-wasm-wheel.outputs.cache-hit != 'true' | |
| id: compute-emsdk-version | |
| run: | | |
| # Prepare xbuild environment (side-effect) | |
| pyodide config list | |
| # Save EMSDK version | |
| EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) | |
| echo "emsdk-version=$EMSCRIPTEN_VERSION" >> $GITHUB_OUTPUT | |
| working-directory: awkward-cpp | |
| - name: Install EMSDK | |
| uses: mymindstorm/setup-emsdk@v13 | |
| if: steps.cache-awkward-cpp-wasm-wheel.outputs.cache-hit != 'true' | |
| with: | |
| version: ${{ steps.compute-emsdk-version.outputs.emsdk-version }} | |
| - name: Build wheel | |
| if: steps.cache-awkward-cpp-wasm-wheel.outputs.cache-hit != 'true' | |
| id: build-awkward-cpp-wasm-wheel | |
| run: | | |
| # pyodide-build doesn't work out of the box with pipx | |
| CFLAGS=-fexceptions LDFLAGS=-fexceptions pyodide build --exports whole_archive | |
| working-directory: awkward-cpp | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: awkward-cpp-wasm | |
| path: awkward-cpp/dist/awkward*wasm32.whl | |
| awkward-cpp-x86-64: | |
| runs-on: ubuntu-22.04 | |
| name: Build C++ x86 | |
| defaults: | |
| run: | |
| # Ensure conda env is activated | |
| shell: "bash -l {0}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| # TODO: remove this once mamba-org/mamba#1726 is fixed | |
| # and replace with `-r requirements.txt` in a | |
| # non-template `environment.yml` | |
| - name: Template environment.yml | |
| run: pipx run --spec cogapp cog -o environment.yml environment.yml.cog | |
| working-directory: docs | |
| # Technically this give us an environment that is incompatible with | |
| # the wheel built in the awkward-x86-64 job if the environments | |
| # solve with different external library versions. By default, | |
| # ROOT uses cxx-compiler too, so hopefully this won't be an issue | |
| - name: Setup Python via Conda | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| # Cache invalidates daily by default | |
| cache-environment: true | |
| cache-downloads: true | |
| environment-file: docs/environment.yml | |
| create-args: >- | |
| python=${{ env.X86_64_PYTHON_VERSION }} | |
| doxygen | |
| python-build | |
| - name: Generate build files | |
| run: pipx run nox -s prepare -- --headers --signatures | |
| - name: Cache wheel | |
| id: cache-awkward-cpp-x86-64-wheel | |
| uses: actions/cache@v3 | |
| with: | |
| path: ./awkward-cpp/dist | |
| key: ${{ runner.os }}-${{ env.X86_64_PYTHON_VERSION }}-awkward-x86-64-wasm-${{ hashFiles('awkward-cpp/**') }} | |
| - name: Build wheel | |
| if: steps.cache-awkward-cpp-x86-64-wheel.outputs.cache-hit != 'true' | |
| run: python -m build -w ./awkward-cpp | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: awkward-cpp-x86-64 | |
| path: awkward-cpp/dist/awkward*.whl | |
| awkward: | |
| runs-on: ubuntu-22.04 | |
| name: Build Python | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Generate build files | |
| run: pipx run nox -s prepare -- --headers --signatures | |
| - name: Build wheel | |
| run: pipx run build -w | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: awkward | |
| path: dist/awkward*.whl | |
| execute-cppyy: | |
| needs: [awkward-cpp-x86-64, awkward] | |
| runs-on: ubuntu-22.04 | |
| name: Execute cppyy notebook | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{ env.X86_64_PYTHON_VERSION }}" | |
| - name: Extract pre-built manylinux cppyy wheels | |
| uses: shrink/actions-docker-extract@v3 | |
| with: | |
| image: "docker.io/agoose77/cppyy-wheels:cp311" | |
| path: "/wheels/." | |
| destination: "/tmp/wheels/" | |
| - name: Download awkward wheel | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: awkward | |
| path: /tmp/wheels/ | |
| - name: Download awkward-cpp x86_64 wheel | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: awkward-cpp-x86-64 | |
| path: /tmp/wheels/ | |
| - name: Install dependencies | |
| working-directory: docs | |
| run: python -m pip install /tmp/wheels/*.whl -r requirements.txt | |
| - name: Execute cppyy notebook | |
| working-directory: docs | |
| run: jupyter nbconvert --to notebook --execute --inplace user-guide/how-to-use-in-cpp-cppyy.ipynb | |
| - name: Save executed notebook | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: how-to-use-in-cpp-cppyy | |
| path: docs/user-guide/how-to-use-in-cpp-cppyy.ipynb | |
| build-docs: | |
| runs-on: ubuntu-22.04 | |
| needs: [awkward-cpp-wasm, awkward-cpp-x86-64, awkward, execute-cppyy] | |
| name: Build Docs | |
| defaults: | |
| run: | |
| # Ensure conda env is activated | |
| shell: "bash -l {0}" | |
| env: | |
| DOCS_CANONICAL_VERSION: main | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # TODO: remove this once mamba-org/mamba#1726 is fixed | |
| # and replace with `-r requirements.txt` in a | |
| # non-template `environment.yml` | |
| - name: Template environment.yml | |
| run: pipx run --spec cogapp cog -o environment.yml environment.yml.cog | |
| working-directory: docs | |
| # Technically this give us an environment that is incompatible with | |
| # the wheel built in the awkward-x86-64 job if the environments | |
| # solve with different external library versions. By default, | |
| # ROOT uses cxx-compiler too, so hopefully this won't be an issue | |
| - name: Setup Python via Conda | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| # Cache invalidates daily by default | |
| cache-environment: true | |
| cache-downloads: true | |
| environment-file: docs/environment.yml | |
| create-args: >- | |
| python=${{ env.X86_64_PYTHON_VERSION }} | |
| doxygen | |
| - name: Download awkward wheel | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: awkward | |
| path: dist | |
| - name: Copy awkward wheel to JupyterLite | |
| run: | | |
| mkdir -p docs/lite/pypi/ | |
| cp dist/awkward*.whl docs/lite/pypi/ | |
| - name: Download & copy awkward-cpp WASM wheel to JupyterLite | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: awkward-cpp-wasm | |
| path: docs/lite/pypi | |
| - name: Download awkward-cpp x86_64 wheel | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: awkward-cpp-x86-64 | |
| path: dist | |
| - name: Install awkward and awkward-cpp wheels | |
| run: python -m pip install dist/awkward*.whl --force-reinstall --no-deps | |
| - name: Generate build files | |
| run: pipx run nox -s prepare -- --docs --headers | |
| - name: Generate C++ documentation | |
| run: doxygen | |
| working-directory: awkward-cpp/docs | |
| - name: Copy C++ documentation | |
| run: cp -r awkward-cpp/docs/html/ docs/_static/doxygen | |
| - name: Enable analytics & version selector | |
| if: github.event_name == 'push' || github.event_name == 'release' | |
| run: | | |
| echo "DOCS_REPORT_ANALYTICS=1" >> $GITHUB_ENV | |
| echo "DOCS_SHOW_VERSION=1" >> $GITHUB_ENV | |
| - name: Set version to main | |
| if: github.event_name == 'push' | |
| run: | | |
| echo "DOCS_VERSION=main" >> $GITHUB_ENV | |
| - name: Download cppyy notebook | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: how-to-use-in-cpp-cppyy | |
| path: docs/user-guide | |
| - name: Generate Python documentation | |
| run: sphinx-build -M html . _build/ -T | |
| working-directory: docs | |
| - name: Upload docs artefact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: docs | |
| path: docs/_build/html | |
| - name: Upload Jupyter Book cache | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: doctrees | |
| path: docs/_build/doctrees | |
| - name: Upload Jupyter Book cache | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: jupyter-cache | |
| path: docs/_build/.jupyter_cache | |
| branch-preview: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-docs] | |
| name: Deploy Branch Preview | |
| # We can only deploy for PRs on host repo | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| S3_BUCKET: "preview.awkward-array.org" | |
| DEPLOY_URL: "http://preview.awkward-array.org.s3-website.us-east-1.amazonaws.com" | |
| environment: | |
| name: docs | |
| url: "${{ env.DEPLOY_URL }}/${{ github.head_ref }}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: eu-west-2 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} | |
| - name: Download rendered docs | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: docs | |
| path: built-docs | |
| - name: Sync artefacts | |
| run: | | |
| aws s3 sync built-docs/ "s3://${S3_BUCKET}/${{ github.head_ref }}" | |
| deploy: | |
| runs-on: ubuntu-22.04 | |
| needs: [ build-docs ] | |
| # We can only deploy for PRs on host repo | |
| if: github.event_name == 'push' || github.event_name == 'release' | |
| name: Deploy | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| S3_BUCKET: "awkward-array.org" | |
| PRODUCTION_URL: "http://awkward-array.org" | |
| CLOUDFRONT_ID: "EFM4QVENUIXHS" | |
| environment: | |
| name: docs | |
| url: ${{ env.PRODUCTION_URL }}${{ steps.sync-main.outputs.path || steps.sync-stable.outputs.path }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: eu-west-2 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} | |
| - name: Download rendered docs | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: docs | |
| path: built-docs | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| # Pushes to main trigger latest | |
| - name: Sync `main` | |
| if: github.event_name == 'push' | |
| id: sync-main | |
| run: | | |
| aws s3 sync built-docs/ "s3://${S3_BUCKET}/doc/main/" | |
| aws cloudfront create-invalidation --distribution-id "${CLOUDFRONT_ID}" \ | |
| --paths "/doc/main*" | |
| echo "path=/doc/main" >> $GITHUB_OUTPUT | |
| # Releases trigger versions | |
| - name: Sync `stable` | |
| if: github.event_name == 'release' | |
| id: sync-stable | |
| run: | | |
| # Take only leading version | |
| version=$(echo "${GITHUB_REF_NAME}" | sed -n -E "s/v?([0-9]+\.[0-9]+)\.[0-9]+/\1/p") | |
| aws s3 cp docs/switcher.json "s3://${S3_BUCKET}/doc/" | |
| aws s3 sync built-docs/ "s3://${S3_BUCKET}/doc/$version/" | |
| aws s3 sync built-docs/ "s3://${S3_BUCKET}/doc/stable/" | |
| aws cloudfront create-invalidation --distribution-id "${CLOUDFRONT_ID}" \ | |
| --paths "/doc/$version*" "/doc/stable*" "/doc/switcher.json" | |
| echo "path=/doc/stable" >> $GITHUB_OUTPUT |