Minimal scraper working from end-to-end

openzim · Sep 27, 2024 · aecd0a3 · aecd0a3
1 parent 2c4a12a
commit aecd0a3
Show file tree

Hide file tree

Showing 80 changed files with 9,860 additions and 2 deletions.
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: kiwix # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # https://kiwix.org/support-us/
diff --git a/.github/stale.yml b/.github/stale.yml
@@ -0,0 +1,15 @@
+daysUntilClose: false
+staleLabel: stale
+
+issues:
+  daysUntilStale: 60
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be now be reviewed manually. Thank you
+    for your contributions.
+pulls:
+   daysUntilStale: 7
+   markComment: >
+     This pull request has been automatically marked as stale because it has not had
+     recent activity. It will be now be reviewed manually. Thank you
+     for your contributions.
diff --git a/.github/workflows/Publish.yml b/.github/workflows/Publish.yml
@@ -0,0 +1,45 @@
+name: Build and upload to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-24.04
+    permissions:
+      id-token: write  # mandatory for PyPI trusted publishing
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: scraper/pyproject.toml
+          architecture: x64
+
+      - name: Build packages
+        working-directory: scraper
+        run: |
+          pip install -U pip build
+          python -m build --sdist --wheel
+
+      - name: Upload to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1.10
+        with:
+          packages-dir: scraper/dist/
+
+      - name: Build and push Docker image
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: openzim/libretexts
+          tag-pattern: /^v([0-9.]+)$/
+          latest-on-tag: true
+          restrict-to: openzim/libretexts
+          registries: ghcr.io
+          credentials:
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+          repo_description: auto
+          repo_overview: auto
diff --git a/.github/workflows/PublishDockerDevImage.yaml b/.github/workflows/PublishDockerDevImage.yaml
@@ -0,0 +1,27 @@
+name: Publish Docker dev image
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  publish:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build and push Docker image
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: openzim/libretexts
+          manual-tag: dev
+          latest-on-tag: false
+          restrict-to: openzim/libretexts
+          registries: ghcr.io
+          credentials:
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+          repo_description: auto
+          repo_overview: auto
diff --git a/.github/workflows/QA.yml b/.github/workflows/QA.yml
@@ -0,0 +1,62 @@
+name: QA
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  check-scraper-qa:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: scraper/pyproject.toml
+          architecture: x64
+
+      - name: Install dependencies
+        working-directory: scraper
+        run: |
+          pip install -U pip
+          pip install -e .[lint,check,scripts,test]
+
+      - name: Check black formatting
+        working-directory: scraper
+        run: inv lint-black
+
+      - name: Check ruff
+        working-directory: scraper
+        run: inv lint-ruff
+
+      - name: Check pyright
+        working-directory: scraper
+        run: inv check-pyright
+
+  check-zimui-qa:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: zimui/.node-version
+
+      - name: Install JS dependencies
+        working-directory: zimui
+        run: |
+          yarn install
+      - name: Check prettier
+        working-directory: zimui
+        run: |
+          yarn format
+      - name: Check eslint
+        working-directory: zimui
+        run: |
+          yarn lint
diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml
@@ -0,0 +1,108 @@
+name: Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  test-scraper:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: scraper/pyproject.toml
+          architecture: x64
+
+      - name: Install dependencies (and project)
+        working-directory: scraper
+        run: |
+          pip install -U pip
+          pip install -e .[test,scripts]
+
+      - name: Run the tests
+        working-directory: scraper
+        run: inv coverage --args "-vvv"
+
+      - name: Upload coverage report to codecov
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+  build-scraper:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: scraper/pyproject.toml
+          architecture: x64
+
+      - name: Ensure we can build Python targets
+        working-directory: scraper
+        run: |
+          pip install -U pip build
+          python3 -m build --sdist --wheel
+
+  build-and-test-zimui:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: zimui/.node-version
+
+      - name: Install dependencies
+        working-directory: zimui
+        run: |
+          yarn install
+
+      - name: Test
+        working-directory: zimui
+        run: |
+          yarn test:unit:run
+
+      - name: Build
+        working-directory: zimui
+        run: |
+          yarn build
+
+      - name: Start web server
+        working-directory: zimui
+        run: |
+          yarn preview &
+
+      - name: Wait for web server to be ready
+        run: |
+          npx wait-on http://localhost:4173
+
+      - name: Run frontend tests
+        working-directory: zimui
+        run: |
+          $(yarn bin)/cypress run
+
+  # this job replaces the standard "build_docker" job since it builds the docker image
+  run-integration-tests:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Ensure we can build the Docker image
+        run: |
+          docker build -t libretexts2zim .
+
+      - name: Run scraper
+        run: docker run -v $PWD/output:/output libretexts2zim libretexts2zim --library-slug geo --library-name Geosciences --file-name-format "tests_en_libretexts-geo"
+
+      - name: Run integration test suite
+        run: docker run -v $PWD/scraper/tests-integration:/src/scraper/tests-integration -v $PWD/output:/output libretexts2zim bash -c "pip install pytest; pytest -v /src/scraper/tests-integration"