Real python package

.github/workflows/tests.yml

@@ -2,60 +2,75 @@ name: tests
 
 on: [push, pull_request]
 
-env:
- DOCKER_REGISTRY: ghcr.io/commaai
- RUN: docker run -e PYTHONWARNINGS=error --shm-size 1G --name msgq msgq /bin/sh -c
- RUN_NAMED: docker run -e PYTHONWARNINGS=error --shm-size 1G --rm msgq /bin/sh -c
- CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm msgqci /bin/bash -c
- BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $DOCKER_REGISTRY/msgq:latest -t msgq -f Dockerfile .
- PYTHONWARNINGS: error
-
 jobs:
- build:
-  name: build
- runs-on: ubuntu-latest
+
+ static_analysis:
+ runs-on: ubuntu-24.04
  steps:
- - uses: actions/checkout@v3
- - name: Build docker image
- run: eval "$BUILD"
- - name: Push to dockerhub
- if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/msgq'
- run: |
- docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
- docker tag msgq $DOCKER_REGISTRY/msgq:latest
- docker push $DOCKER_REGISTRY/msgq:latest
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+ - name: Installing requirements
+ run:
+ python -m pip install -r requirements.txt
+ - name: Installing ubuntu requirements
+ run: "sudo apt-get install -y cppcheck"
+ - name: Static analysis
+ run: pre-commit run --all
+
+ scons_test:
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"]
+ python: ["3.11", "3.12"]
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Installing python requirements
+ run:
+ python -m pip install -r requirements.txt
+ - name: Installing ubuntu requirements
+ if: ${{startsWith(matrix.os, 'ubuntu')}}
+ run:
+ scripts/ubuntu_dependencies.sh
+ - name: Installing macos requirements
+ if: ${{startsWith(matrix.os, 'macos')}}
+ run:
+ scripts/macos_dependencies.sh
+ - name: Building
+ run: scons -j$(nproc || sysctl -n hw.logicalcpu)
+
 
- unit_tests:
- name: unit tests
- runs-on: ubuntu-latest
+ full_tests:
+ runs-on: ${{ matrix.os }}
  strategy:
  matrix:
+ os: ["ubuntu-20.04", "ubuntu-24.04"]
+ python: ["3.11", "3.12"]
  flags: ['', '--asan', '--ubsan']
  backend: ['MSGQ', 'ZMQ']
  steps:
- - uses: actions/checkout@v3
- - name: Build docker image
- run: eval "$BUILD"
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Installing ubuntu requirements
+ run:
+ scripts/ubuntu_dependencies.sh
+ - name: Building msgq
+ run: |
+ pip3 install -e .[dev]
+ - name: Python tests
+ run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append
  - name: C++ tests
  run: |
- $RUN "export ${{ matrix.backend }}=1 && \
- scons ${{ matrix.flags }} -j$(nproc) && \
- msgq/test_runner && \
- msgq/visionipc/test_runner"
- - name: python tests
- run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m pytest"
+ export ${{ matrix.backend }}=1
+ scons ${{ matrix.flags }} -j$(nproc)
+ msgq/test_runner
+ msgq/visionipc/test_runner
  - name: Upload coverage
- run: |
- docker commit msgq msgqci
- $CI_RUN "cd /project/msgq && bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"
-
- static_analysis:
- name: static analysis
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - name: Build docker image
- run: eval "$BUILD"
- - name: Static analysis
- # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
- run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all"
+ run: "bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"

.github/workflows/wheels.yml

@@ -0,0 +1,50 @@
+name: wheels
+
+on: [push, pull_request]
+
+jobs:
+ wheels:
+ runs-on: ${{ matrix.platform.os }}
+ strategy:
+ fail-fast: true
+ matrix:
+ platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64", arch: "x86_64"},
+ {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"},
+ {os: "macos-14", target: "macosx_arm64", arch: "arm64"} ]
+ python: [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ]
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up QEMU
+ if: runner.os == 'Linux'
+ uses: docker/setup-qemu-action@v3
+ with:
+ platforms: all
+
+ - name: Building wheel
+ uses: pypa/[email protected]
+ env:
+ CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}"
+ CIBW_ARCHS: "${{ matrix.platform.arch }}"
+ CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh"
+ CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
+ CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
+ CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
+ MACOSX_DEPLOYMENT_TARGET: "14.0"
+
+ - uses: actions/setup-python@v5
+ if: ${{ matrix.platform.arch != 'aarch64' }}
+ with:
+ python-version: ${{ matrix.python.py }}
+
+ - name: Installing the wheel
+ if: ${{ matrix.platform.arch != 'aarch64' }}
+ run: |
+ pip install --break-system-packages ./wheelhouse/*.whl
+
+ - name: Saving wheel
+ uses: actions/upload-artifact@v4
+ with:
+ name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }}
+ path: ./wheelhouse/*.whl

.gitignore

@@ -17,3 +17,7 @@ services.h
 .sconsign.dblite
 libcereal_shared.*
 .mypy_cache/
+msgq.egg-info
+build/
+dist/
+catch2/

MANIFEST.in

@@ -0,0 +1,4 @@
+exclude msgq/**/*.o
+include SConscript
+include SConstruct
+recursive-include site_scons *

SConstruct

@@ -61,6 +61,7 @@ env = Environment(
  "-Werror",
  "-Wshadow",
  "-Wno-vla-cxx-extension",
+ "-Wno-unknown-warning-option",
  ] + ccflags,
  LDFLAGS=ldflags,
  LINKFLAGS=ldflags,

pyproject.toml

@@ -1,3 +1,30 @@
+[project]
+name = "msgq"
+requires-python = ">= 3.11"
+version = "0.1.0"
+
+dependencies = [
+ "numpy",
+ "Cython",
+ "pycapnp",
+ "setuptools",
+]
+
+[project.optional-dependencies]
+dev = [
+ "scons",
+ "pre-commit",
+ "ruff",
+ "parameterized",
+ "coverage",
+ "pytest",
+ "pytest-cov",
+]
+
+[build-system]
+requires = ["setuptools","scons","Cython","numpy"]
+build-backend = "setuptools.build_meta"
+
 # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
 [tool.ruff]
 lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"]

requirements.txt

@@ -0,0 +1,50 @@
+# This file was autogenerated by uv via the following command:
+# uv pip compile --all-extras -o requirements.txt pyproject.toml
+cfgv==3.4.0
+ # via pre-commit
+coverage==7.6.0
+ # via
+ # msgq (pyproject.toml)
+ # pytest-cov
+cython==3.0.10
+ # via msgq (pyproject.toml)
+distlib==0.3.8
+ # via virtualenv
+filelock==3.15.4
+ # via virtualenv
+identify==2.6.0
+ # via pre-commit
+iniconfig==2.0.0
+ # via pytest
+nodeenv==1.9.1
+ # via pre-commit
+numpy==2.0.0
+ # via msgq (pyproject.toml)
+packaging==24.1
+ # via pytest
+parameterized==0.9.0
+ # via msgq (pyproject.toml)
+platformdirs==4.2.2
+ # via virtualenv
+pluggy==1.5.0
+ # via pytest
+pre-commit==3.7.1
+ # via msgq (pyproject.toml)
+pycapnp==2.0.0
+ # via msgq (pyproject.toml)
+pytest==8.2.2
+ # via
+ # msgq (pyproject.toml)
+ # pytest-cov
+pytest-cov==5.0.0
+ # via msgq (pyproject.toml)
+pyyaml==6.0.1
+ # via pre-commit
+ruff==0.5.3
+ # via msgq (pyproject.toml)
+scons==4.8.0
+ # via msgq (pyproject.toml)
+setuptools==71.0.4
+ # via msgq (pyproject.toml)
+virtualenv==20.26.3
+ # via pre-commit

scripts/macos_dependencies.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+ROOT="$(cd $DIR/../ && pwd)"
+ARCH=$(uname -m)
+
+if [[ $SHELL == "/bin/zsh" ]]; then
+ RC_FILE="$HOME/.zshrc"
+elif [[ $SHELL == "/bin/bash" ]]; then
+ RC_FILE="$HOME/.bash_profile"
+fi
+
+# Install brew if required
+if [[ $(command -v brew) == "" ]]; then
+ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+
+ if [[ $ARCH == "x86_64" ]]; then
+ echo 'eval "$(/usr/local/homebrew/bin/brew shellenv)"' >> $RC_FILE
+ eval "$(/usr/local/homebrew/bin/brew shellenv)"
+ else
+ echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> $RC_FILE
+ eval "$(/opt/homebrew/bin/brew shellenv)"
+ fi
+fi
+
+brew bundle --file=- <<-EOS
+brew "zeromq"
+cask "gcc-arm-embedded"
+brew "gcc@13"
+EOS
+
+if [[ -z "$NO_CATCH2" ]]; then
+ cd /tmp
+ git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
+ cd Catch2
+ mv single_include/* "$ROOT"
+ rm -rf Catch2
+fi

scripts/manylinux_dependencies.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+dnf install -y clang opencl-headers ocl-icd-devel cppcheck wget
+
+wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz
+tar -xvf zeromq-4.3.5.tar.gz
+cd zeromq-4.3.5
+./configure
+make -j$(nproc)
+make install