From 48b7aada7ce24811c4015856ebe5060357469ac2 Mon Sep 17 00:00:00 2001 From: Wen Wei Tseng Date: Tue, 15 Oct 2024 20:11:52 +0800 Subject: [PATCH] Docker (#262) * Revert "Reapply "test no docker (#250)"" This reverts commit 83a85a3ef05ed99981cda6cb45043ba605dd17f5. * rm Literate * try jill * Official source * fix * install sys packages * copy from julia image * matching * install nbconvert * KeyValueFormat * remove warning * fix * fix * fix * no custom depot = faster startup * binderhub --- .dockerignore | 3 + .github/dependabot.yml | 6 ++ .github/workflows/ci.yml | 134 ++++++++++---------------- .github/workflows/linkcheck.yml | 24 +---- .github/workflows/rm-old-image.yml | 20 ++++ .github/workflows/update-manifest.yml | 13 ++- docs/_config.yml | 4 +- docs/_toc.yml | 2 - docs/pythonplot.jl | 21 ---- docs/sub/plots-lit.jl | 19 ---- env.Dockerfile | 23 +++++ execute.jl | 63 ------------ requirements-jb.txt | 1 - requirements.txt | 1 - 14 files changed, 114 insertions(+), 220 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/rm-old-image.yml delete mode 100644 docs/pythonplot.jl delete mode 100644 docs/sub/plots-lit.jl create mode 100644 env.Dockerfile delete mode 100644 execute.jl delete mode 100644 requirements-jb.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2311a50 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git/ +.cache/ +.github/workflows diff --git a/.github/dependabot.yml b/.github/dependabot.yml index afaf0bb..f278062 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,3 +8,9 @@ updates: directory: "/" schedule: interval: daily + - package-ecosystem: docker + directory: "/" + schedule: + interval: daily + labels: + - automerge diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a49214..a4bd7e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,65 +11,36 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - ALLOWERRORS: 'false' - NBCACHE: '.cache' - JULIA_CONDAPKG_BACKEND: 'Null' - JULIA_NUM_THREADS: 'auto' - JULIA_CI: 'true' - UV_SYSTEM_PYTHON: 1 - jobs: setup: + permissions: + packages: write runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} - hash: ${{ steps.hash.outputs.value }} - ver: ${{ steps.hash.outputs.ver }} + hash: ${{ steps.img.outputs.hash }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - id: setup-python - with: - python-version: '3.x' - - name: Read Julia version - uses: SebRollen/toml-action@v1.2.0 - id: read_toml + - name: Login to ghcr.io + uses: docker/login-action@v3 with: - file: 'Manifest.toml' - field: 'julia_version' - - name: Setup Julia - uses: julia-actions/setup-julia@v2 - with: - version: ${{ steps.read_toml.outputs.value }} - - name: Get environment hash - id: hash + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + - name: Get docker image hash + id: img + run: echo "hash=${{ hashFiles('requirements.txt', 'Project.toml', 'Manifest.toml', 'src/**', 'env.Dockerfile') }}" >> "$GITHUB_OUTPUT" + - name: Build Docker container + env: + IMG: ghcr.io/${{ github.repository }}:${{ steps.img.outputs.hash }} run: | - echo "value=${{ hashFiles('Project.toml', 'Manifest.toml', 'src/**') }}" >> "$GITHUB_OUTPUT" - echo "ver=${{ runner.os }}-julia-${{ steps.read_toml.outputs.value }}" >> "$GITHUB_OUTPUT" - - name: Cache Julia packages - if: ${{ contains(runner.name, 'GitHub Actions') }} - uses: actions/cache@v4 - id: cache-julia - with: - path: ~/.julia - key: ${{ steps.hash.outputs.ver }}-${{ steps.hash.outputs.value }} - restore-keys: | - ${{ steps.hash.outputs.ver }}- - - name: Install Julia packages - if: ${{ runner.environment == 'self-hosted' || steps.cache-julia.outputs.cache-hit != 'true' }} - shell: julia --color=yes {0} - run: | - using Pkg - Pkg.add(["IJulia", "Literate", "JSON"]) - Pkg.activate(".") - Pkg.instantiate() - Pkg.precompile() + docker manifest inspect ${IMG} && exit 0 + docker build -f env.Dockerfile -t ${IMG} . + docker push ${IMG} - name: List notebooks as a JSON array id: set-matrix - run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True) + glob.glob("**/*.jl", root_dir="docs",recursive=True)))')" >> "$GITHUB_OUTPUT" + run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True)))')" >> "$GITHUB_OUTPUT" execute: needs: setup @@ -81,52 +52,48 @@ jobs: runs-on: ubuntu-latest env: NB: docs/${{ matrix.notebook }} + IMG: ghcr.io/${{ github.repository }}:${{ needs.setup.outputs.hash }} steps: - name: Checkout uses: actions/checkout@v4 - name: Cache notebook - uses: actions/cache@v4 + uses: actions/cache/restore@v4 id: nb-cache with: - path: ${{ env.NBCACHE }} - key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} - - name: Setup Python - uses: actions/setup-python@v5 + path: ${{ env.NB }} + key: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} + - name: Pull docker image if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - with: - python-version: '3.x' - - name: Install uv + run: | + docker pull ${{ env.IMG }} + docker images ${{ env.IMG }} + - name: Get Julia kernel name + id: jl-kernel if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - run: pip install uv - - name: Install Python dependencies + run: docker run ${{ env.IMG }} julia -e 'print("name=--ExecutePreprocessor.kernel_name=julia-1.", VERSION.minor)' >> "$GITHUB_OUTPUT" + - name: Execute notebook if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - run: uv pip install -r requirements.txt - - name: Read Julia version + run: > + docker run -w /app -v ${{ github.workspace }}:/app + ${{ env.IMG }} + jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=-1 ${{ steps.jl-kernel.outputs.name }} ${{ env.NB }} + - name: Claim output notebook if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - uses: SebRollen/toml-action@v1.2.0 - id: read_toml - with: - file: 'Manifest.toml' - field: 'julia_version' - - name: Setup Julia + run: sudo chown $USER ${{ env.NB }} + - name: Cache notebook + uses: actions/cache/save@v4 if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - uses: julia-actions/setup-julia@v2 - with: - version: ${{ steps.read_toml.outputs.value }} - - name: Restore Julia packages - if: ${{ runner.environment == 'github-hosted' && steps.nb-cache.outputs.cache-hit != 'true'}} - uses: actions/cache@v4 with: - path: ~/.julia - key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }} - - name: Execute notebook - if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - run: julia --project=@. execute.jl + path: ${{ env.NB }} + key: ${{ steps.nb-cache.outputs.cache-primary-key }} + - name: Convert artifact Name + id: art + run: echo "name=$(echo ${{ env.NB }} | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" - name: Upload Notebook uses: actions/upload-artifact@v4 with: - name: notebook-${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} - path: ${{ env.NBCACHE }} + name: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} + path: docs/*${{ matrix.notebook }} include-hidden-files: true retention-days: 1 @@ -139,19 +106,20 @@ jobs: - name: Download notebooks uses: actions/download-artifact@v4 with: - path: ${{ env.NBCACHE }}/ + path: out/ pattern: notebook-* merge-multiple: true - name: Copy back built notebooks - run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/ + run: cp --verbose -rf out/* docs/ - name: Setup Python uses: actions/setup-python@v5 + id: setup-python with: python-version: '3.x' - - name: Install uv - run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Install Python dependencies - run: uv pip install -r requirements-jb.txt + run: | + pip install uv + uv pip install --system jupyter-book - name: Build website run: jupyter-book build docs/ - name: Upload pages artifact diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index f86193d..9e50f81 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -24,31 +24,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Julia - uses: julia-actions/setup-julia@v2 - - name: Convert literate notebooks to ipynb files - shell: julia --color=yes {0} - working-directory: ${{ env.DIR }} - run: | - import Pkg - Pkg.add("Literate") - using Literate - for (root, dirs, files) in walkdir(pwd()) - for file in files - if endswith(file, ".jl") - nb = joinpath(root, file) - Literate.notebook(nb, dirname(nb); mdstrings=true, execute=false) - end - end - end - name: Setup Python uses: actions/setup-python@v5 - id: setup-python with: python-version: '3.x' - - name: Install uv - run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Install Python dependencies - run: uv pip install -r requirements-jb.txt + run: | + pip install uv + uv pip install --system jupyter-book - name: Build website run: jupyter-book build ${DIR} --builder linkcheck diff --git a/.github/workflows/rm-old-image.yml b/.github/workflows/rm-old-image.yml new file mode 100644 index 0000000..e053549 --- /dev/null +++ b/.github/workflows/rm-old-image.yml @@ -0,0 +1,20 @@ +name: Docker image retention +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' # Every week + +jobs: + clean: + name: Delete old images + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - uses: snok/container-retention-policy@v3.0.0 + with: + account: user + token: ${{ secrets.GITHUB_TOKEN }} + image-names: ${{ github.event.repository.name }} + cut-off: 2w + dry-run: false diff --git a/.github/workflows/update-manifest.yml b/.github/workflows/update-manifest.yml index 32d69c7..bf51de2 100644 --- a/.github/workflows/update-manifest.yml +++ b/.github/workflows/update-manifest.yml @@ -26,14 +26,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Julia - uses: julia-actions/setup-julia@v2 - with: - version: '1' - name: Update Julia dependencies - env: - JULIA_PKG_PRECOMPILE_AUTO: '0' - run: julia --project=@. --color=yes -e 'using Pkg; Pkg.update()' + run: > + docker run --rm -w /tmp + -v ${{ github.workspace }}:/tmp + -e JULIA_PKG_PRECOMPILE_AUTO=0 + julia:latest + julia --project=@. --color=yes -e 'import Pkg; Pkg.update()' # Authenticate with a custom GitHub APP # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens - name: Generate token for PR diff --git a/docs/_config.yml b/docs/_config.yml index 33d1907..d5a8f0a 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -56,7 +56,7 @@ bibtex_bibfiles: # Launch button settings launch_buttons: notebook_interface: jupyterlab # The interface interactive links will activate ["classic", "jupyterlab"] - binderhub_url: "" # The URL of the BinderHub (e.g., https://mybinder.org) + binderhub_url: "https://mybinder.org" # The URL of the BinderHub (e.g., https://mybinder.org) jupyterhub_url: "" # The URL of the JupyterHub (e.g., https://datahub.berkeley.edu) thebe: false # Add a thebe button to pages (requires the repository to run on Binder) colab_url: "" # The URL of Google Colab (https://colab.research.google.com) @@ -77,7 +77,7 @@ html: use_multitoc_numbering: true # Continuous numbering across parts/chapters extra_navbar: "" # Will be displayed underneath the left navbar. extra_footer: "" # Will be displayed underneath the footer. - google_analytics_id: "" # A GA id that can be used to track book views. + # google_analytics_id: "" # A GA id that can be used to track book views. home_page_in_navbar: true # Whether to include your home page in the left Navigation Bar baseurl: "" # The base URL where your book will be hosted. Used for creating image previews and social links. e.g.: https://mypage.com/mybook/ # analytics: diff --git a/docs/_toc.yml b/docs/_toc.yml index 32516b7..541611e 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -5,6 +5,4 @@ format: jb-book root: index chapters: - file: pyplot - - file: pythonplot - file: sub/plots - - file: sub/plots-lit diff --git a/docs/pythonplot.jl b/docs/pythonplot.jl deleted file mode 100644 index c1c9a7e..0000000 --- a/docs/pythonplot.jl +++ /dev/null @@ -1,21 +0,0 @@ -#=== -# Plotting with PythonPlot.jl - -Using Literate.jl -===# -import PythonPlot as plt -using Random -Random.seed!(2022) - -#--- -plt.figure() -plt.plot(1:5, rand(1:6, 5)) -plt.gcf() - -# ## Runtime information -import Pkg -Pkg.status() - -#--- -import InteractiveUtils -InteractiveUtils.versioninfo() diff --git a/docs/sub/plots-lit.jl b/docs/sub/plots-lit.jl deleted file mode 100644 index 79e58b4..0000000 --- a/docs/sub/plots-lit.jl +++ /dev/null @@ -1,19 +0,0 @@ -#=== -# Plotting by Plots.jl - -Using Literate.jl -===# -using Plots -using Random -Random.seed!(2022) - -#--- -plot(rand(1:6, 5)) - -# ## Runtime information -import Pkg -Pkg.status() - -#--- -import InteractiveUtils -InteractiveUtils.versioninfo() diff --git a/env.Dockerfile b/env.Dockerfile new file mode 100644 index 0000000..7b40fb2 --- /dev/null +++ b/env.Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12.5-slim + +# System config +ENV JULIA_CI='true' +ENV JULIA_NUM_THREADS='auto' +# Let PythonCall use built-in python +ENV JULIA_CONDAPKG_BACKEND='Null' +ENV JULIA_PATH='/usr/local/julia/' +ENV PATH=${JULIA_PATH}/bin:${PATH} +COPY --from=julia:1.11.0 ${JULIA_PATH} ${JULIA_PATH} +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +WORKDIR /app + +# Python dependencies +COPY requirements.txt . +RUN uv pip install --system --no-cache nbconvert -r requirements.txt + +# Julia dependencies +COPY Project.toml Manifest.toml ./ +COPY src/ src +RUN julia --color=yes -e 'using Pkg; Pkg.add("IJulia"); import IJulia; IJulia.installkernel("Julia", "--project=@.")' && \ + julia --color=yes --project=@. -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' diff --git a/execute.jl b/execute.jl deleted file mode 100644 index a7a6c91..0000000 --- a/execute.jl +++ /dev/null @@ -1,63 +0,0 @@ -using Literate -using JSON -using Pkg -using IJulia - -ENV["GKSwstype"] = "100" - -function main(; rmsvg=true) - file = get(ENV, "NB", "test.ipynb") - if endswith(file, ".jl") - run_literate(file; rmsvg) - elseif endswith(file, ".ipynb") - IJulia.installkernel("Julia", "--project=@.") - run_ipynb(file) - else - error("$(file) is not a valid notebook file!") - end -end - -# Strip SVG output from a Jupyter notebook -function strip_svg(ipynb) - @info "Stripping SVG in $(ipynb)" - nb = open(JSON.parse, ipynb, "r") - for cell in nb["cells"] - !haskey(cell, "outputs") && continue - for output in cell["outputs"] - !haskey(output, "data") && continue - datadict = output["data"] - if haskey(datadict, "image/png") || haskey(datadict, "image/jpeg") - delete!(datadict, "text/html") - delete!(datadict, "image/svg+xml") - end - end - end - rm(ipynb) - open(ipynb, "w") do io - JSON.print(io, nb, 1) - end - return ipynb -end - -function run_literate(file; rmsvg=true) - cachedir = get(ENV, "NBCACHE", ".cache") - outpath = joinpath(abspath(pwd()), cachedir, dirname(file)) - mkpath(outpath) - ipynb = Literate.notebook(file, outpath; mdstrings=true, execute=true) - rmsvg && strip_svg(ipynb) - return ipynb -end - -function run_ipynb(file) - cachedir = get(ENV, "NBCACHE", ".cache") - outpath = joinpath(abspath(pwd()), cachedir, file) - mkpath(dirname(outpath)) - kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)" - execute = get(ENV, "ALLOWERRORS", " ") == "true" ? "--execute --allow-errors" : "--execute" - timeout = "--ExecutePreprocessor.timeout=" * get(ENV, "TIMEOUT", "-1") - cmd = `jupyter nbconvert --to notebook $(execute) $(timeout) $(kernelname) --output $(outpath) $(file)` - run(cmd) - return outpath -end - -main() diff --git a/requirements-jb.txt b/requirements-jb.txt deleted file mode 100644 index d4e4641..0000000 --- a/requirements-jb.txt +++ /dev/null @@ -1 +0,0 @@ -jupyter-book==1.0.3 diff --git a/requirements.txt b/requirements.txt index 1807e69..3357350 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ matplotlib==3.9.2 -nbconvert==7.16.4