Migrate to poetry and optimize dependencies

.github/workflows/python-package.yml

@@ -19,50 +19,55 @@ jobs:
  - name: Install packages
  run: |
  sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr
+
  - name: Compile and install yara
  run: |
  sudo apt-get install automake libtool make gcc pkg-config
- wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.5.0.tar.gz
- tar xzf *.tar.gz
+ wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.5.0.tar.gz -O yara.tar.gz
+ tar xzf yara.tar.gz
  pushd yara-*
  ./bootstrap.sh
  ./configure
  make
  sudo make install
  popd
- - uses: actions/checkout@v4
+
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
  - name: Set up Python ${{ matrix.python-version }}
  uses: actions/setup-python@v5
  with:
  python-version: ${{ matrix.python-version }}
  cache: 'pip'
+
  - name: Install dependencies
  run: |
- python -m pip install --upgrade pip
- pip install flake8 pytest
- pip install pipenv
- sed -i "s/python_version.*/python_version = \"${{ matrix.python-version }}\"/" Pipfile
- pipenv lock
- pipenv requirements > requirements.txt
- # pyfaul must be installed manually (?)
- pip install -r requirements.txt pyfaup
- pip install .
+ python -m pip install --upgrade poetry
+ poetry install
+ # Tell poetry to not use a virtual environment
+ # poetry config virtualenvs.create false
+
  - name: Lint with flake8
  run: |
  # stop the build if there are Python syntax errors or undefined names
- flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+ poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
  # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+ poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
  - name: Run server in background
  run: |
- misp-modules -l 127.0.0.1 -s 2>error.log &
- sleep 3
+ poetry run misp-modules -l 127.0.0.1 -s 2>error.log &
+ sleep 10
+
  - name: Check if server is running
  run: |
  curl -sS localhost:6666/healthcheck
+
  - name: Test with pytest
  run: |
- pytest tests
+ poetry run pytest
+
  - name: Show error log
  if: always()
  run: |

.gitignore

@@ -24,3 +24,4 @@ venv*
 .vscode*
 *.sqlite
 website/conf/config.cfg
+wheels

Makefile

@@ -1,38 +1,61 @@
-# https://www.mkdocs.org/user-guide/deploying-your-docs/
+# See: https://www.mkdocs.org/user-guide/deploying-your-docs/
+# Running 'make' uses poetry-installed mkdocs
+# Running 'USE_DOCKER=true make' uses docker mkdocs
 
-.PHONY: prepare_docs generate_docs ci_generate_docs test_docs
+.PHONY: prepare_docs generate_docs deploy test_docs
 
-prepare_docs:
- pip3 install .
- cd documentation; python3 generate_documentation.py
- mkdir -p docs/expansion/logos docs/export_mod/logos docs/import_mod/logos
- mkdir -p docs/logos
- cd documentation; cp -R ./logos/* ../docs/logos
- cd documentation; cp -R ./logos/* ../docs/expansion/logos
- cd documentation; cp -R ./logos/* ../docs/export_mod/logos
- cd documentation; cp -R ./logos/* ../docs/import_mod/logos
- cp ./documentation/mkdocs/*.md ./docs
- cp LICENSE ./docs/license.md
-
-install_requirements:
- pip install -r docs/REQUIREMENTS.txt
+MKDOCS_DOCKER_IMAGE := squidfunk/mkdocs-material
 
-generate_docs: prepare_docs
- mkdocs build
+DOCS_DIST_DIR := ./docs
 
-deploy:
- mkdocs gh-deploy
+DOCS_SRC_DIR := ./documentation
 
-test_docs: prepare_docs
- mkdocs serve
+USE_DOCKER ?=
 
+.DEFAULT_GOAL := generate_docs
 
-# DOCKER make commands
-generate_docs_docker: prepare_docs
- docker run --rm -it -v $(PWD):/docs squidfunk/mkdocs-material build
 
-deploy_docker:
- docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro squidfunk/mkdocs-material gh-deploy
+prepare_docs:
+ @echo "Preparing documentation."
+ poetry install --only docs
+ poetry run python $(DOCS_SRC_DIR)/generate_documentation.py
+ mkdir -p $(DOCS_DIST_DIR)/logos
+ mkdir -p $(DOCS_DIST_DIR)/expansion/logos
+ mkdir -p $(DOCS_DIST_DIR)/export_mod/logos
+ mkdir -p $(DOCS_DIST_DIR)/import_mod/logos
+ cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/logos
+ cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/expansion/logos
+ cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/export_mod/logos
+ cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/import_mod/logos
+ cp $(DOCS_SRC_DIR)/mkdocs/*.md $(DOCS_DIST_DIR)
+ cp LICENSE $(DOCS_DIST_DIR)/license.md
+
 
-test_docs_docker: prepare_docs
- docker run --rm -it -p 8000:8000 -v $(PWD):/docs squidfunk/mkdocs-material
+generate_docs: prepare_docs
+ifeq ($(USE_DOCKER), true)
+ @echo "Generating documentation using '$(MKDOCS_DOCKER_IMAGE)'."
+ docker run --rm -it -v $(PWD):/docs $(MKDOCS_DOCKER_IMAGE) build
+else
+ @echo "Generating docunentation."
+ poetry run mkdocs build
+endif
+
+
+deploy: generate_docs
+ifeq ($(USE_DOCKER), true)
+ @echo "Deploying documentation using '$(MKDOCS_DOCKER_IMAGE)'."
+ docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro $(MKDOCS_DOCKER_IMAGE) gh-deploy
+else
+ @echo "Deploying docunentation."
+ poetry run mkdocs gh-deploy
+endif
+
+
+test_docs: prepare_docs
+ifeq ($(USE_DOCKER), true)
+ @echo "Serving documentation using '$(MKDOCS_DOCKER_IMAGE)'."
+ docker run --rm -it -v $(PWD):/docs -p 8000:8000 $(MKDOCS_DOCKER_IMAGE)
+else
+ @echo "Serving docunentation."
+ poetry run mkdocs serve
+endif