Merge remote-tracking branch 'tobiasdiez/rel_import_modules' into pr/…

…tobiasdiez/36524

.ci/merge-fixes.sh

@@ -14,10 +14,10 @@ else
     git tag -f test_base
     git commit -q -m "Uncommitted changes" --no-allow-empty -a
     for a in $PRs; do
+        git fetch --unshallow --all > /dev/null 2>&1 && echo "Unshallowed."
         echo "::group::Merging PR https://github.com/$REPO/pull/$a"
         git tag -f test_head
         $GH pr checkout -b pr-$a $a
-        git fetch --unshallow --all
         git checkout -q test_head
         if git merge --no-edit --squash -q pr-$a; then
             echo "::endgroup::"

.github/workflows/build.yml

@@ -193,11 +193,12 @@ jobs:
         run: |
           ./venv/bin/python3 -m coverage combine src/.coverage/
           ./venv/bin/python3 -m coverage xml
-          find . -name *coverage*
+          mkdir -p coverage-report
+          mv coverage.xml coverage-report/
         working-directory: ./worktree-image
 
       - name: Upload coverage to codecov
         if: always() && steps.build.outcome == 'success'
         uses: codecov/codecov-action@v3
         with:
-          files: ./worktree-image/coverage.xml
+          directory: ./worktree-image/coverage-report

.github/workflows/ci-linux.yml

@@ -34,39 +34,20 @@ permissions:
 
 jobs:
 
-  # standard-pre for the default platform (used by build.yml etc.)
-  default-pre:
+  # standard-pre and standard (without ptest) for the default platform (used by build.yml etc.)
+  default:
     uses: ./.github/workflows/docker.yml
     with:
       # Build from scratch
-      docker_targets: "with-system-packages configured with-targets-pre"
+      docker_targets: "with-system-packages configured with-targets-pre with-targets"
       # FIXME: duplicated from env.TARGETS
       targets_pre: all-sage-local
-      tox_system_factors: >-
-        ["ubuntu-focal"]
-      tox_packages_factors: >-
-        ["standard"]
-      docker_push_repository: ghcr.io/${{ github.repository }}/
-
-  # standard for the default platform (used by build.yml etc.)
-  default:
-    if: ${{ success() || failure() }}
-    needs: [default-pre]
-    uses: ./.github/workflows/docker.yml
-    with:
-      # Build incrementally from previous stage (pre)
-      incremental: true
-      free_disk_space: true
-      from_docker_repository: ghcr.io/${{ github.repository }}/
-      from_docker_target: "with-targets-pre"
-      docker_targets: "with-targets with-targets-optional"
-      # FIXME: duplicated from env.TARGETS
       targets: build doc-html
       targets_optional: ptest
       tox_system_factors: >-
         ["ubuntu-focal"]
       tox_packages_factors: >-
-          ["standard"]
+        ["standard"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
 
   # All platforms. This duplicates the default platform, but why not,
@@ -83,6 +64,9 @@ jobs:
       tox_packages_factors: >-
           ["standard"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
+      # Make sure that all "standard-pre" jobs can start simultaneously,
+      # so that runners are available by the time that "default" starts.
+      max_parallel: 50
 
   standard:
     if: ${{ success() || failure() }}
@@ -101,6 +85,8 @@ jobs:
       tox_packages_factors: >-
           ["standard"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
+      # Reduce from 30 to 25 because it runs in parallel with 'standard-sitepackages' below
+      max_parallel: 25
 
   standard-sitepackages:
     if: ${{ success() || failure() }}
@@ -153,8 +139,6 @@ jobs:
 
   minimal-pre:
     if: ${{ success() || failure() }}
-    # It does not really "need" it.
-    needs: [standard]
     uses: ./.github/workflows/docker.yml
     with:
       # Build from scratch
@@ -164,6 +148,9 @@ jobs:
       tox_packages_factors: >-
           ["minimal"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
+      # Reduced from 30 because it may run in parallel with 'standard-sitepackages' above.
+      # Calibrated for clogging the job pipeline until the "default" job has finished.
+      max_parallel: 20
 
   minimal:
     if: ${{ success() || failure() }}

.github/workflows/ci-macos.yml

@@ -85,7 +85,7 @@ jobs:
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "Sage GitHub CI"
-          SAGE_ROOT=. SAGE_SRC=./src src/bin/sage-update-version $(git describe) || echo "(ignoring error)"
+          SAGE_ROOT=. SAGE_SRC=./src src/bin/sage-update-version $(git describe --tags) || echo "(ignoring error)"
       - name: make dist
         run: |
           ./configure --enable-download-from-upstream-url && make dist

.github/workflows/dist.yml

@@ -99,7 +99,7 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.SAGEMATH_PYPI_API_TOKEN }}
-          skip_existing: true
+          skip-existing: true
           verbose: true
         if: env.CAN_DEPLOY == 'true'
 
@@ -180,7 +180,7 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.SAGEMATH_PYPI_API_TOKEN }}
-          packages_dir: wheelhouse/
-          skip_existing: true
+          packages-dir: wheelhouse/
+          skip-existing: true
           verbose: true
         if: env.CAN_DEPLOY == 'true'

.github/workflows/doc-build-pdf.yml

@@ -66,8 +66,6 @@ jobs:
           git config --global user.email "[email protected]"
           git config --global user.name "Build & Test workflow"
           .ci/retrofit-worktree.sh worktree-image /sage
-          # Keep track of changes to built HTML
-          new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old")
 
       - name: Download upstream artifact
         uses: actions/download-artifact@v3
@@ -107,19 +105,20 @@ jobs:
         id: docbuild
         if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
         run: |
-          make doc-clean doc-uninstall; make doc-pdf
+          make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps
         working-directory: ./worktree-image
         env:
           MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Copy docs
+        id: copy
         if: always() && steps.docbuild.outcome == 'success'
         run: |
           # For some reason the deploy step below cannot find /sage/...
           # So copy everything from there to local folder
           mkdir -p ./docs
-          cp -r -L /sage/local/share/doc/sage/pdf/en/* ./docs
+          cp -r -L /sage/local/share/doc/sage/pdf ./docs
           # Zip everything for increased performance
           zip -r docs-pdf.zip docs
 

.github/workflows/doc-build.yml

@@ -63,12 +63,12 @@ jobs:
           mathjax_path_to=$(SAGE_USE_CDNS=yes /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)")
           new_version=$(cat src/VERSION.txt)
           # Wipe out chronic diffs between old doc and new doc
-          (cd /sage/local/share/doc/sage/html/en && \
+          (cd /sage/local/share/doc/sage/html && \
            find . -name "*.html" | xargs sed -i -e '/class="sidebar-brand-text"/ s/Sage [0-9a-z.]* /Sage '"$new_version"' /' \
                                                 -e 's;'"$mathjax_path_from"';'"$mathjax_path_to"';' \
                                                 -e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d')
           # Create git repo from old doc
-          (cd /sage/local/share/doc/sage/html/en && \
+          (cd /sage/local/share/doc/sage/html && \
            git init && \
            (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && \
            (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; \
@@ -116,9 +116,9 @@ jobs:
         run: |
           set -ex
           export SAGE_USE_CDNS=yes
-          mv /sage/local/share/doc/sage/html/en/.git /sage/.git-doc
+          mv /sage/local/share/doc/sage/html/.git /sage/.git-doc
           make doc-clean doc-uninstall
-          mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git
+          mkdir -p /sage/local/share/doc/sage/html/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/.git
           ./config.status && make sagemath_doc_html-no-deps
         working-directory: ./worktree-image
         env:
@@ -131,9 +131,9 @@ jobs:
         run: |
           set -ex
           mkdir -p ./docs
-          (cd /sage/local/share/doc/sage/html/en && git commit -a -m 'new')
+          (cd /sage/local/share/doc/sage/html && git commit -a -m 'new')
           # Wipe out chronic diffs between old doc and new doc
-          (cd /sage/local/share/doc/sage/html/en && \
+          (cd /sage/local/share/doc/sage/html && \
            find . -name "*.html" | xargs sed -i -e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d')
           # Create CHANGES.html
           echo '<html>' > ./docs/CHANGES.html
@@ -162,7 +162,7 @@ jobs:
           EOF
           echo '</head>' >> ./docs/CHANGES.html
           echo '<body>' >> ./docs/CHANGES.html
-          (cd /sage/local/share/doc/sage/html/en && git diff HEAD^; rm -rf .git) > ./docs/diff.txt
+          (cd /sage/local/share/doc/sage/html && git diff HEAD^ -- *.html; rm -rf .git) > ./docs/diff.txt
           /sage/sage -python - << EOF
           import re, html
           with open('./docs/diff.txt', 'r') as f:
@@ -172,7 +172,7 @@ jobs:
               for block in diff_blocks:
                   match = re.search(r'^diff --git a/(.*) b/\1', block, flags=re.MULTILINE)
                   if match:
-                      path = match.group(1)
+                      path = 'html/' + match.group(1)
                       out_blocks.append(f'<p class="diff"><a href="{path}">{path}</a>&emsp;</p>\n<pre><code class="language-diff">' + html.escape(block).strip() + '</code></pre>')
               output_text = '\n'.join(out_blocks)
           with open('./docs/diff.html', 'w') as f:
@@ -182,11 +182,11 @@ jobs:
           echo '</body>' >> ./docs/CHANGES.html
           echo '</html>' >>./docs/CHANGES.html
           rm ./docs/diff.txt ./docs/diff.html
-          (cd /sage/local/share/doc/sage/html/en && git reset --hard HEAD)
           # For some reason the deploy step below cannot find /sage/...
           # So copy everything from there to local folder
           # We also need to replace the symlinks because netlify is not following them
-          cp -r -L /sage/local/share/doc/sage/html/en/* ./docs
+          cp -r -L /sage/local/share/doc/sage/html ./docs
+          cp  /sage/local/share/doc/sage/index.html ./docs
           # Zip everything for increased performance
           zip -r docs.zip docs
 
@@ -196,3 +196,36 @@ jobs:
         with:
           name: docs
           path: docs.zip
+
+      - name: Build live doc
+        id: buildlivedoc
+        if: (success() || failure()) && steps.copy.outcome == 'success' && github.repository == 'sagemath/sage' && github.ref == 'refs/heads/develop'
+        run: |
+          set -ex
+          export SAGE_USE_CDNS=yes
+          export SAGE_LIVE_DOC=yes
+          export SAGE_JUPYTER_SERVER=binder:sagemath/sage-binder-env/dev
+          make doc-clean doc-uninstall
+          ./config.status && make sagemath_doc_html-no-deps
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
+      - name: Copy live doc
+        id: copylivedoc
+        if: (success() || failure()) && steps.buildlivedoc.outcome == 'success'
+        run: |
+          set -ex
+          mkdir -p ./livedoc
+          cp -r -L /sage/local/share/doc/sage/html ./livedoc
+          cp  /sage/local/share/doc/sage/index.html ./livedoc
+          zip -r livedoc.zip livedoc
+
+      - name: Upload live doc
+        if: (success() || failure()) && steps.copylivedoc.outcome == 'success'
+        uses: actions/upload-artifact@v3
+        with:
+          name: livedoc
+          path: livedoc.zip
+

.github/workflows/doc-publish.yml

@@ -72,7 +72,7 @@ jobs:
           header: preview-comment
           recreate: true
           message: |
-            [Documentation preview for this PR](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}) (built with commit ${{ steps.source-run-info.outputs.sourceHeadSha }}; [changes](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}/CHANGES.html)) is ready! :tada:
+            [Documentation preview for this PR](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}/html/en) (built with commit ${{ steps.source-run-info.outputs.sourceHeadSha }}; [changes](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}/CHANGES.html)) is ready! :tada:
 
       - name: Update deployment status PR check
         uses: myrotvorets/[email protected]
@@ -90,5 +90,48 @@ jobs:
 
       - name: Report deployment url
         run: |
-          echo "::notice::The documentation has being automatically deployed to Netlify. %0A ✅ Preview: ${{ steps.deploy-netlify.outputs.NETLIFY_URL }}"
+          echo "::notice::The documentation has been deployed - ${{ steps.deploy-netlify.outputs.NETLIFY_URL }}"
+
+  publish-live-doc:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion == 'success' && github.repository == 'sagemath/sage' && github.event.workflow_run.head_branch == 'develop'
+    steps:
+      - name: Download live doc
+        uses: actions/[email protected]
+        with:
+          script: |
+            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{github.event.workflow_run.id }},
+            });
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "livedoc"
+            })[0];
+            var download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/livedoc.zip', Buffer.from(download.data));
+
+      - name: Extract live doc
+        run: unzip livedoc.zip -d doc && unzip doc/livedoc.zip -d doc/doc
+
+      - name: Deploy to Netlify
+        id: deploy-netlify
+        uses: netlify/actions/cli@master
+        with:
+          args: deploy --dir=doc/doc/livedoc --message ${NETLIFY_MESSAGE} --alias ${NETLIFY_ALIAS}
+        env:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+          NETLIFY_MESSAGE: Deployed live doc
+          NETLIFY_ALIAS: deploy-livedoc
+
+      - name: Report deployment url
+        run: |
+          echo "::notice::The live documentation has been deployed - ${{ steps.deploy-netlify.outputs.NETLIFY_URL }}"
 

.github/workflows/docker.yml

@@ -51,14 +51,13 @@ on:
            "almalinux-9-python3.11",
            "gentoo-python3.10",
            "gentoo-python3.11",
-           "gentoo-python3.12",
            "archlinux-latest",
            "opensuse-15.3-gcc_11-python3.9",
            "opensuse-15.4-gcc_11-python3.10",
            "opensuse-15.5-gcc_11-python3.11",
            "opensuse-tumbleweed-python3.10",
            "opensuse-tumbleweed",
-           "conda-forge",
+           "conda-forge-python3.11",
            "ubuntu-bionic-gcc_8-i386",
            "debian-bullseye-i386",
            ]
@@ -191,6 +190,9 @@ jobs:
               echo "DOCKER_PUSH_REPOSITORY=$(echo ${{ inputs.docker_push_repository }} | tr "[:upper:]" "[:lower:]")" >> $GITHUB_ENV
               echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
             fi
+
+      - name: Determine Docker tags to use
+        run: |
             # This line needs to be run before the step "Merge CI fixes from sagemath/sage".
             DOCKER_TAG="$(git describe --dirty --always)"
             echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV

.github/workflows/macos.yml

@@ -21,16 +21,16 @@ on:
       osversion_xcodeversion_toxenv_tuples:
         description: 'Stringified JSON object'
         default: >-
-          [["latest", "",           "homebrew-macos-usrlocal-minimal"],
-           ["latest", "",           "homebrew-macos-usrlocal-standard"],
-           ["11",     "xcode_11.7", "homebrew-macos-usrlocal-standard"],
-           ["12",     "",           "homebrew-macos-usrlocal-standard"],
-           ["13",     "xcode_15.0", "homebrew-macos-usrlocal-standard"],
-           ["latest", "",           "homebrew-macos-usrlocal-maximal"],
-           ["latest", "",           "homebrew-macos-usrlocal-python3_xcode-standard"],
-           ["latest", "",           "conda-forge-macos-minimal"],
-           ["latest", "",           "conda-forge-macos-standard"],
-           ["latest", "",           "conda-forge-macos-maximal"]]
+          [["latest", "",             "homebrew-macos-usrlocal-minimal"],
+           ["latest", "",             "homebrew-macos-usrlocal-standard"],
+           ["11",     "xcode_13.2.1", "homebrew-macos-usrlocal-standard"],
+           ["12",     "",             "homebrew-macos-usrlocal-standard"],
+           ["13",     "xcode_15.0",   "homebrew-macos-usrlocal-standard"],
+           ["latest", "",             "homebrew-macos-usrlocal-maximal"],
+           ["latest", "",             "homebrew-macos-usrlocal-python3_xcode-standard"],
+           ["latest", "",             "conda-forge-macos-minimal"],
+           ["latest", "",             "conda-forge-macos-standard"],
+           ["latest", "",             "conda-forge-macos-maximal"]]
         type: string
       extra_sage_packages:
         description: 'Extra Sage packages to install as system packages'

CITATION.cff

@@ -4,8 +4,8 @@ title: SageMath
 abstract: SageMath is a free open-source mathematics software system.
 authors:
 - name: "The SageMath Developers"
-version: 10.2.beta9
+version: 10.2.rc1
 doi: 10.5281/zenodo.593563
-date-released: 2023-10-30
+date-released: 2023-11-10
 repository-code: "https://github.com/sagemath/sage"
 url: "https://www.sagemath.org/"