From 0aa9376392bfe870d1728a79982707e4c7e596f3 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 24 Apr 2024 19:07:35 +1200 Subject: [PATCH 1/9] ci: Use xml2rfc-fonts for tests --- .github/workflows/checks.yml | 54 +++++++++++++----------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 73b2637c..b2cb4442 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -56,30 +56,23 @@ jobs: - name: Download Fonts if: steps.cache-fonts-linux.outputs.cache-hit != 'true' run: | - echo "Downloading Noto Font..." - mkdir -p ~/.fonts/opentype/noto - wget -q https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip - unzip -q Noto-unhinted.zip -d ~/.fonts/opentype/noto/ - mkdir -p /usr/share/fonts/truetype/noto/ - ln -sf ~/.fonts/opentype/noto/*.[to]tf /usr/share/fonts/truetype/noto/ - echo "Downloading Roboto Mono Font..." - mkdir -p ~/.fonts/opentype/roboto - wget -q https://fonts.google.com/download?family=Roboto%20Mono -O roboto-mono.zip - unzip -q roboto-mono.zip -d ~/.fonts/opentype/roboto/ - mkdir -p /usr/share/fonts/truetype/roboto/ - ln -sf ~/.fonts/opentype/roboto/*.[to]tf /usr/share/fonts/truetype/roboto/ + echo "Downloading xml2rfc-fonts" + mkdir -p ~/.fonts/opentype ~/fonts + wget -q -O fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.20.0.tar.gz + tar zxf fonts.tar.gz -C ~/fonts + mv ~/fonts/*/noto/* ~/.fonts/opentype/ + mv ~/fonts/*/roboto_mono/* ~/.fonts/opentype/ + mkdir -p /usr/share/fonts/truetype + ln -sf ~/.fonts/opentype/*.[to]tf /usr/share/fonts/truetype/ echo "Reloading Font Cache..." fc-cache -f -v - name: Link Fonts if: steps.cache-fonts-linux.outputs.cache-hit == 'true' run: | - echo "Linking Noto Font..." - mkdir -p /usr/share/fonts/truetype/noto/ - ln -sf ~/.fonts/opentype/noto/*.[to]tf /usr/share/fonts/truetype/noto/ - echo "Linking Roboto Mono Font..." - mkdir -p /usr/share/fonts/truetype/roboto/ - ln -sf ~/.fonts/opentype/roboto/*.[to]tf /usr/share/fonts/truetype/roboto/ + echo "Linking Fonts..." + mkdir -p /usr/share/fonts/truetype/ + ln -sf ~/.fonts/opentype/*.[to]tf /usr/share/fonts/truetype/ echo "Reloading Font Cache..." fc-cache -f -v @@ -124,30 +117,21 @@ jobs: id: cache-fonts-mac uses: pat-s/always-upload-cache@v2.1.5 with: - path: ~/new-fonts + path: ~/fonts key: fonts-macos - name: Download Fonts - if: steps.cache-fonts-mac.outputs.cache-hit != 'true' run: | - mkdir -p ~/new-fonts - cd ~/new-fonts - echo "Downloading Noto Font..." - wget -q https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip - unzip -o -q Noto-unhinted.zip -d /Library/Fonts/ - echo "Downloading Roboto Mono Font..." - wget -q https://fonts.google.com/download?family=Roboto%20Mono -O roboto-mono.zip - unzip -o -q roboto-mono.zip -d /Library/Fonts/ - - - name: Extract Fonts - if: steps.cache-fonts-mac.outputs.cache-hit == 'true' - run: | - unzip -o -q ~/new-fonts/Noto-unhinted.zip -d /Library/Fonts/ - unzip -o -q ~/new-fonts/roboto-mono.zip -d /Library/Fonts/ + echo "Downloading xml2rfc-fonts" + mkdir -p ~/fonts /Library/Fonts + wget -q -O fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.20.0.tar.gz + tar zxf fonts.tar.gz -C ~/fonts + mv ~/fonts/*/noto/* /Library/Fonts/ + mv ~/fonts/*/roboto_mono/* /Library/Fonts/ - name: Install dependencies run: | - brew install pango libffi groff make diffutils + brew install pango pillow libffi groff make diffutils echo "Installing pip + wheel..." python -m pip install --upgrade pip wheel echo "Installing requirements.txt + test dependencies..." From 5a1f971111ea5730a15a8db13b85de7b74b2bf96 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Mon, 17 Jun 2024 22:19:22 +1200 Subject: [PATCH 2/9] ci: Bump up setup-python action to v5 --- .github/workflows/checks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b2cb4442..2d447a0f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -42,7 +42,7 @@ jobs: uses: actions/checkout@v2 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v2.3.1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -109,7 +109,7 @@ jobs: uses: actions/checkout@v2 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v2.3.1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -164,7 +164,7 @@ jobs: # uses: actions/checkout@v2 # - name: Setup Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2.3.1 +# uses: actions/setup-python@v5 # with: # python-version: ${{ matrix.python-version }} From a7459ea37508e631a5ddbb8c206b9fa857a622af Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Mon, 17 Jun 2024 22:23:41 +1200 Subject: [PATCH 3/9] ci: Bump up checout action to v4 --- .github/workflows/checks.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2d447a0f..4467560f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v1 @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -106,7 +106,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -161,7 +161,7 @@ jobs: # steps: # - name: Checkout repository -# uses: actions/checkout@v2 +# uses: actions/checkout@v4 # - name: Setup Python ${{ matrix.python-version }} # uses: actions/setup-python@v5 From a082ad017a745909913fe9a1c6eeb44cb6fc1618 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Mon, 17 Jun 2024 22:32:07 +1200 Subject: [PATCH 4/9] ci: Restrict macOS testing to intel processor See #1129 --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 4467560f..7f6efe5d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -97,7 +97,7 @@ jobs: tests-macos: name: Unit Tests (macOS) - runs-on: macos-latest + runs-on: macos-12 if: github.event_name == 'pull_request' strategy: From 79ce4a7f547f7286711c8e62f11719189c26f39d Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 26 Jun 2024 00:44:06 +1200 Subject: [PATCH 5/9] fix: Fix text extraction issues with updated fonts --- xml2rfc/walkpdf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xml2rfc/walkpdf.py b/xml2rfc/walkpdf.py index 665e363d..d94c83bd 100644 --- a/xml2rfc/walkpdf.py +++ b/xml2rfc/walkpdf.py @@ -22,7 +22,7 @@ def walk(obj, seen): dobj[k] = d iobj += i if hasattr(obj, 'extract_text'): - dobj['text'] = obj.extract_text() + dobj['text'] = obj.extract_text(extraction_mode="layout") elif isinstance(obj, pypdf.generic.ArrayObject): dobj = [] for o in obj: From b87991e5a1c44386b565700d6133ccec69079e6f Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 26 Jun 2024 00:55:00 +1200 Subject: [PATCH 6/9] build: Use xml2rfc-fonts in docker image --- docker/base.Dockerfile | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 78dfaf72..ce9f6c2f 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -34,17 +34,12 @@ RUN apt-get update --fix-missing && \ apt-get clean -y # Install required fonts -RUN mkdir -p ~/.fonts/opentype && \ - wget -q https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip && \ - unzip -q Noto-unhinted.zip -d ~/.fonts/opentype/ && \ - rm Noto-unhinted.zip && \ - wget -q https://fonts.google.com/download?family=Roboto%20Mono -O roboto-mono.zip && \ - unzip -q roboto-mono.zip -d ~/.fonts/opentype/ && \ - rm roboto-mono.zip && \ - wget -q https://fonts.google.com/download?family=Noto%20Sans%20Math -O noto-sans-math.zip && \ - unzip -q noto-sans-math.zip -d ~/.fonts/opentype/ && \ - rm noto-sans-math.zip && \ - ln -sf ~/.fonts/opentype/*.[to]tf /usr/share/fonts/truetype/ && \ +RUN mkdir -p ~/.fonts/opentype /tmp/fonts && \ + wget -q -O /tmp/fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.21.0.tar.gz && \ + tar zxf /tmp/fonts.tar.gz -C /tmp/fonts && \ + mv /tmp/fonts/*/noto/* ~/.fonts/opentype/ && \ + mv /tmp/fonts/*/roboto_mono/* ~/.fonts/opentype/ && \ + rm -rf /tmp/fonts.tar.gz /tmp/fonts/ && \ fc-cache -f # Copy everything required to build xml2rfc From 811d03db19abc99e1ee4bea3eb4e40b3c970f6c3 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 26 Jun 2024 01:17:09 +1200 Subject: [PATCH 7/9] fix: Use xml2rfc-fonts xml2rfc compatible version --- .github/workflows/checks.yml | 4 ++-- docker/base.Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7f6efe5d..1768901f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -58,7 +58,7 @@ jobs: run: | echo "Downloading xml2rfc-fonts" mkdir -p ~/.fonts/opentype ~/fonts - wget -q -O fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.20.0.tar.gz + wget -q -O fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz tar zxf fonts.tar.gz -C ~/fonts mv ~/fonts/*/noto/* ~/.fonts/opentype/ mv ~/fonts/*/roboto_mono/* ~/.fonts/opentype/ @@ -124,7 +124,7 @@ jobs: run: | echo "Downloading xml2rfc-fonts" mkdir -p ~/fonts /Library/Fonts - wget -q -O fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.20.0.tar.gz + wget -q -O fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz tar zxf fonts.tar.gz -C ~/fonts mv ~/fonts/*/noto/* /Library/Fonts/ mv ~/fonts/*/roboto_mono/* /Library/Fonts/ diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index ce9f6c2f..66c382ee 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update --fix-missing && \ # Install required fonts RUN mkdir -p ~/.fonts/opentype /tmp/fonts && \ - wget -q -O /tmp/fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.21.0.tar.gz && \ + wget -q -O /tmp/fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz && \ tar zxf /tmp/fonts.tar.gz -C /tmp/fonts && \ mv /tmp/fonts/*/noto/* ~/.fonts/opentype/ && \ mv /tmp/fonts/*/roboto_mono/* ~/.fonts/opentype/ && \ From d09d41ea69892a4ddddf82fafb42e2e14ab0c93a Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 26 Jun 2024 01:35:54 +1200 Subject: [PATCH 8/9] docs: Update documentation to use xml2rfc-fonts --- README.md | 14 ++++---------- xml2rfc/run.py | 22 ++++++---------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 97fa3e26..5afc412e 100644 --- a/README.md +++ b/README.md @@ -44,16 +44,10 @@ In order to generate PDFs, **xml2rfc** uses the [WeasyPrint] module, which depen ```sh pip install 'weasyprint>=53.0,!=57.0,!=60.0' ``` -3. Finally, install the full **Noto Font** and **Roboto Mono** packages: - * Download the full font file from: - https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip - * Follow the installation instructions at - https://www.google.com/get/noto/help/install/ - * Go to https://fonts.google.com/specimen/Roboto+Mono, and download the - font. Follow the installation instructions above, as applied to this download. - * Go to https://fonts.google.com/noto/specimen/Noto+Sans+Math, and - download the font. Follow the installation instructions above, as - applied to this download. +3. Finally, install the required fonts: + * Download [xml2rfc-fonts v3.22.0](https://github.com/ietf-tools/xml2rfc-fonts/releases/tag/3.22.0) [tar.gz](https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz) [zip](https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.zip) + * Extract the contents of the downloaded xml2rfc-fonts archive. + * Install the fonts found in the `noto` and `roboto_mono` directories to your operating system. With these installed and available to **xml2rfc**, the `--pdf` switch will be enabled. diff --git a/xml2rfc/run.py b/xml2rfc/run.py index ea2e2074..984f8f3b 100755 --- a/xml2rfc/run.py +++ b/xml2rfc/run.py @@ -89,25 +89,15 @@ def get_pdf_help(missing_libs=""): pip install 'weasyprint>=53.0,!=57.0,!=60.0' + 3. Finally, install the required fonts: + * Download xml2rfc-fonts v3.22.0. + https://github.com/ietf-tools/xml2rfc-fonts/releases/tag/3.22.0 - 3. Finally, install the full Noto Font and Roboto Mono packages: + * Extract the contents of the downloaded xml2rfc-fonts archive. - * Download the full font file from: - https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip - or follow the 'DOWNLOAD ALL FONTS' link on this page: - https://www.google.com/get/noto/ + * Install the fonts found in the `noto` and `roboto_mono` directories + to your operating system. - * Follow the installation instructions at - https://www.google.com/get/noto/help/install/ - - * Go to https://fonts.google.com/specimen/Roboto+Mono, and download the - font. Follow the installation instructions above, as applied to this - download. - - * Go to https://fonts.google.com/noto/specimen/Noto+Sans+Math, and - download the font. Follow the installation instructions above, as - applied to this download. - With these libraries, modules, and fonts installed and available to xml2rfc, the --pdf switch will be enabled. """ From a62627fed94ad49c2ca8d01c30bafd810eb67468 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 26 Jun 2024 08:52:35 +1200 Subject: [PATCH 9/9] docs: Update font install instructions --- README.md | 5 +++-- xml2rfc/run.py | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5afc412e..a9f1540f 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,9 @@ In order to generate PDFs, **xml2rfc** uses the [WeasyPrint] module, which depen pip install 'weasyprint>=53.0,!=57.0,!=60.0' ``` 3. Finally, install the required fonts: - * Download [xml2rfc-fonts v3.22.0](https://github.com/ietf-tools/xml2rfc-fonts/releases/tag/3.22.0) [tar.gz](https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz) [zip](https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.zip) - * Extract the contents of the downloaded xml2rfc-fonts archive. + * Download latest fonts from [xml2rfc-fonts](https://github.com/ietf-tools/xml2rfc-fonts/releases/latest). + * In the **Assets** section, download either the `tar.gz` or the `zip` archive. + * Extract the contents of the downloaded `xml2rfc-fonts` archive. * Install the fonts found in the `noto` and `roboto_mono` directories to your operating system. With these installed and available to **xml2rfc**, the `--pdf` switch will be enabled. diff --git a/xml2rfc/run.py b/xml2rfc/run.py index 984f8f3b..576ef520 100755 --- a/xml2rfc/run.py +++ b/xml2rfc/run.py @@ -90,12 +90,15 @@ def get_pdf_help(missing_libs=""): pip install 'weasyprint>=53.0,!=57.0,!=60.0' 3. Finally, install the required fonts: - * Download xml2rfc-fonts v3.22.0. - https://github.com/ietf-tools/xml2rfc-fonts/releases/tag/3.22.0 + * Download latest fonts from xml2rfc-fonts. + https://github.com/ietf-tools/xml2rfc-fonts/releases/latest + + * In the "Assets" section, download either the tar.gz or + the zip archive. * Extract the contents of the downloaded xml2rfc-fonts archive. - * Install the fonts found in the `noto` and `roboto_mono` directories + * Install the fonts found in the noto and roboto_mono directories to your operating system. With these libraries, modules, and fonts installed and available to