Docker (#262)

* Revert "Reapply "test no docker (#250)"" This reverts commit 83a85a3. * rm Literate * try jill * Official source * fix * install sys packages * copy from julia image * matching * install nbconvert * KeyValueFormat * remove warning * fix * fix * fix * no custom depot = faster startup * binderhub
sosiristseng · Oct 15, 2024 · 48b7aad · 48b7aad
1 parent dac3d26
commit 48b7aad
Show file tree

Hide file tree

Showing 14 changed files with 114 additions and 220 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,3 @@
+.git/
+.cache/
+.github/workflows
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -8,3 +8,9 @@ updates:
     directory: "/"
     schedule:
       interval: daily
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: daily
+    labels:
+      - automerge
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,65 +11,36 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  ALLOWERRORS: 'false'
-  NBCACHE: '.cache'
-  JULIA_CONDAPKG_BACKEND: 'Null'
-  JULIA_NUM_THREADS: 'auto'
-  JULIA_CI: 'true'
-  UV_SYSTEM_PYTHON: 1
-
 jobs:
   setup:
+    permissions:
+      packages: write
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
-      hash: ${{ steps.hash.outputs.value }}
-      ver: ${{ steps.hash.outputs.ver }}
+      hash: ${{ steps.img.outputs.hash }}
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      id: setup-python
-      with:
-        python-version: '3.x'
-    - name: Read Julia version
-      uses: SebRollen/[email protected]
-      id: read_toml
+    - name: Login to ghcr.io
+      uses: docker/login-action@v3
       with:
-        file: 'Manifest.toml'
-        field: 'julia_version'
-    - name: Setup Julia
-      uses: julia-actions/setup-julia@v2
-      with:
-        version: ${{ steps.read_toml.outputs.value }}
-    - name: Get environment hash
-      id: hash
+        registry: ghcr.io
+        username: ${{ github.repository_owner	}}
+        password: ${{ github.token }}
+    - name: Get docker image hash
+      id: img
+      run: echo "hash=${{ hashFiles('requirements.txt', 'Project.toml', 'Manifest.toml', 'src/**', 'env.Dockerfile') }}" >> "$GITHUB_OUTPUT"
+    - name: Build Docker container
+      env:
+        IMG: ghcr.io/${{ github.repository }}:${{ steps.img.outputs.hash }}
       run: |
-        echo "value=${{ hashFiles('Project.toml', 'Manifest.toml', 'src/**') }}" >> "$GITHUB_OUTPUT"
-        echo "ver=${{ runner.os }}-julia-${{ steps.read_toml.outputs.value }}" >> "$GITHUB_OUTPUT"
-    - name: Cache Julia packages
-      if: ${{ contains(runner.name, 'GitHub Actions') }}
-      uses: actions/cache@v4
-      id: cache-julia
-      with:
-        path: ~/.julia
-        key: ${{ steps.hash.outputs.ver }}-${{ steps.hash.outputs.value }}
-        restore-keys: |
-          ${{ steps.hash.outputs.ver }}-
-    - name: Install Julia packages
-      if: ${{ runner.environment == 'self-hosted' || steps.cache-julia.outputs.cache-hit != 'true' }}
-      shell: julia --color=yes {0}
-      run: |
-        using Pkg
-        Pkg.add(["IJulia", "Literate", "JSON"])
-        Pkg.activate(".")
-        Pkg.instantiate()
-        Pkg.precompile()
+        docker manifest inspect ${IMG} && exit 0
+        docker build -f env.Dockerfile -t ${IMG} .
+        docker push ${IMG}
     - name: List notebooks as a JSON array
       id: set-matrix
-      run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True) + glob.glob("**/*.jl", root_dir="docs",recursive=True)))')" >> "$GITHUB_OUTPUT"
+      run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True)))')" >> "$GITHUB_OUTPUT"
 
   execute:
     needs: setup
@@ -81,52 +52,48 @@ jobs:
     runs-on: ubuntu-latest
     env:
       NB: docs/${{ matrix.notebook }}
+      IMG: ghcr.io/${{ github.repository }}:${{ needs.setup.outputs.hash }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Cache notebook
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         id: nb-cache
         with:
-          path: ${{ env.NBCACHE }}
-          key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
-      - name: Setup Python
-        uses: actions/setup-python@v5
+          path: ${{ env.NB }}
+          key: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
+      - name: Pull docker image
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        with:
-          python-version: '3.x'
-      - name: Install uv
+        run: |
+          docker pull ${{ env.IMG }}
+          docker images ${{ env.IMG }}
+      - name: Get Julia kernel name
+        id: jl-kernel
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        run: pip install uv
-      - name: Install Python dependencies
+        run: docker run ${{ env.IMG }} julia -e 'print("name=--ExecutePreprocessor.kernel_name=julia-1.", VERSION.minor)' >> "$GITHUB_OUTPUT"
+      - name: Execute notebook
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        run: uv pip install -r requirements.txt
-      - name: Read Julia version
+        run: >
+          docker run -w /app -v ${{ github.workspace }}:/app
+          ${{ env.IMG }}
+          jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=-1 ${{ steps.jl-kernel.outputs.name }} ${{ env.NB }}
+      - name: Claim output notebook
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        uses: SebRollen/[email protected]
-        id: read_toml
-        with:
-          file: 'Manifest.toml'
-          field: 'julia_version'
-      - name: Setup Julia
+        run: sudo chown $USER ${{ env.NB }}
+      - name: Cache notebook
+        uses: actions/cache/save@v4
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        uses: julia-actions/setup-julia@v2
-        with:
-          version: ${{ steps.read_toml.outputs.value }}
-      - name: Restore Julia packages
-        if: ${{ runner.environment == 'github-hosted' && steps.nb-cache.outputs.cache-hit != 'true'}}
-        uses: actions/cache@v4
         with:
-          path: ~/.julia
-          key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}
-      - name: Execute notebook
-        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        run: julia --project=@. execute.jl
+          path: ${{ env.NB }}
+          key: ${{ steps.nb-cache.outputs.cache-primary-key }}
+      - name: Convert artifact Name
+        id: art
+        run: echo "name=$(echo ${{ env.NB }} | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
       - name: Upload Notebook
         uses: actions/upload-artifact@v4
         with:
-          name: notebook-${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
-          path: ${{ env.NBCACHE }}
+          name: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
+          path: docs/*${{ matrix.notebook }}
           include-hidden-files: true
           retention-days: 1
 
@@ -139,19 +106,20 @@ jobs:
       - name: Download notebooks
         uses: actions/download-artifact@v4
         with:
-          path: ${{ env.NBCACHE }}/
+          path: out/
           pattern: notebook-*
           merge-multiple: true
       - name: Copy back built notebooks
-        run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
+        run: cp --verbose -rf out/* docs/
       - name: Setup Python
         uses: actions/setup-python@v5
+        id: setup-python
         with:
           python-version: '3.x'
-      - name: Install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Install Python dependencies
-        run: uv pip install -r requirements-jb.txt
+        run: |
+          pip install uv
+          uv pip install --system jupyter-book
       - name: Build website
         run: jupyter-book build docs/
       - name: Upload pages artifact

diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml
@@ -24,31 +24,13 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Setup Julia
-        uses: julia-actions/setup-julia@v2
-      - name: Convert literate notebooks to ipynb files
-        shell: julia --color=yes {0}
-        working-directory: ${{ env.DIR }}
-        run: |
-          import Pkg
-          Pkg.add("Literate")
-          using Literate
-          for (root, dirs, files) in walkdir(pwd())
-            for file in files
-              if endswith(file, ".jl")
-                nb = joinpath(root, file)
-                Literate.notebook(nb, dirname(nb); mdstrings=true, execute=false)
-              end
-            end
-          end
       - name: Setup Python
         uses: actions/setup-python@v5
-        id: setup-python
         with:
           python-version: '3.x'
-      - name: Install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Install Python dependencies
-        run: uv pip install -r requirements-jb.txt
+        run: |
+          pip install uv
+          uv pip install --system jupyter-book
       - name: Build website
         run: jupyter-book build ${DIR} --builder linkcheck
diff --git a/.github/workflows/rm-old-image.yml b/.github/workflows/rm-old-image.yml
@@ -0,0 +1,20 @@
+name: Docker image retention
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 1' # Every week
+
+jobs:
+  clean:
+    name: Delete old images
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+    - uses: snok/[email protected]
+      with:
+        account: user
+        token: ${{ secrets.GITHUB_TOKEN }}
+        image-names: ${{ github.event.repository.name }}
+        cut-off: 2w
+        dry-run: false
diff --git a/.github/workflows/update-manifest.yml b/.github/workflows/update-manifest.yml
@@ -26,14 +26,13 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Setup Julia
-        uses: julia-actions/setup-julia@v2
-        with:
-          version: '1'
       - name: Update Julia dependencies
-        env:
-          JULIA_PKG_PRECOMPILE_AUTO: '0'
-        run: julia --project=@. --color=yes -e 'using Pkg; Pkg.update()'
+        run: >
+          docker run --rm -w /tmp
+          -v ${{ github.workspace }}:/tmp
+          -e JULIA_PKG_PRECOMPILE_AUTO=0
+          julia:latest
+          julia --project=@. --color=yes -e 'import Pkg; Pkg.update()'
       # Authenticate with a custom GitHub APP
       # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens
       - name: Generate token for PR

diff --git a/docs/_config.yml b/docs/_config.yml
@@ -56,7 +56,7 @@ bibtex_bibfiles:
 # Launch button settings
 launch_buttons:
   notebook_interface: jupyterlab # The interface interactive links will activate ["classic", "jupyterlab"]
-  binderhub_url: "" # The URL of the BinderHub (e.g., https://mybinder.org)
+  binderhub_url: "https://mybinder.org" # The URL of the BinderHub (e.g., https://mybinder.org)
   jupyterhub_url: "" # The URL of the JupyterHub (e.g., https://datahub.berkeley.edu)
   thebe: false # Add a thebe button to pages (requires the repository to run on Binder)
   colab_url: "" # The URL of Google Colab (https://colab.research.google.com)
@@ -77,7 +77,7 @@ html:
   use_multitoc_numbering: true # Continuous numbering across parts/chapters
   extra_navbar: "" # Will be displayed underneath the left navbar.
   extra_footer: "" # Will be displayed underneath the footer.
-  google_analytics_id: "" # A GA id that can be used to track book views.
+  # google_analytics_id: "" # A GA id that can be used to track book views.
   home_page_in_navbar: true # Whether to include your home page in the left Navigation Bar
   baseurl: "" # The base URL where your book will be hosted. Used for creating image previews and social links. e.g.: https://mypage.com/mybook/
   # analytics:

diff --git a/docs/_toc.yml b/docs/_toc.yml
@@ -5,6 +5,4 @@ format: jb-book
 root: index
 chapters:
   - file: pyplot
-  - file: pythonplot
   - file: sub/plots
-  - file: sub/plots-lit
diff --git a/docs/pythonplot.jl b/docs/pythonplot.jl
diff --git a/docs/sub/plots-lit.jl b/docs/sub/plots-lit.jl
diff --git a/env.Dockerfile b/env.Dockerfile
@@ -0,0 +1,23 @@
+FROM python:3.12.5-slim
+
+# System config
+ENV JULIA_CI='true'
+ENV JULIA_NUM_THREADS='auto'
+# Let PythonCall use built-in python
+ENV JULIA_CONDAPKG_BACKEND='Null'
+ENV JULIA_PATH='/usr/local/julia/'
+ENV PATH=${JULIA_PATH}/bin:${PATH}
+COPY --from=julia:1.11.0 ${JULIA_PATH} ${JULIA_PATH}
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+WORKDIR /app
+
+# Python dependencies
+COPY requirements.txt .
+RUN uv pip install --system --no-cache nbconvert -r requirements.txt
+
+# Julia dependencies
+COPY Project.toml Manifest.toml ./
+COPY src/ src
+RUN julia --color=yes -e 'using Pkg; Pkg.add("IJulia"); import IJulia; IJulia.installkernel("Julia", "--project=@.")' && \
+    julia --color=yes --project=@. -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'