ci: update CI workflows (#101)

* Don't build/install OpenSSL * Build Python extension * Automatically upload Python extension to PyPI
VirusTotal · Apr 29, 2024 · 4173748 · 4173748
1 parent fa628cf
commit 4173748
Show file tree

Hide file tree

Showing 6 changed files with 192 additions and 105 deletions.
diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml
@@ -31,28 +31,6 @@ jobs:
       with:
         toolchain: stable
 
-    # Install OpenSSL on Windows
-    - name: Install OpenSSL
-      if: runner.os == 'Windows'
-      id: vcpkg
-      uses: johnwason/vcpkg-action@v5
-      with:
-        pkgs: openssl
-        triplet: x64-windows-static
-        token: ${{ github.token }}
-
-    - name: Set OPENSSL_DIR environment variable
-      if: runner.os == 'Windows'
-      shell: bash
-      run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-static" >> $GITHUB_ENV
-
-    # Install OpenSSL on Linux
-    - name: Install dependencies
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libssl-dev
-
     - name: Install cargo-c
       run: cargo install cargo-c
 

diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
@@ -3,8 +3,8 @@ name: Python extension
 on:
   push:
     paths:
-      - 'py/**'
-      - '.github/workflows/python.yaml'
+    - 'py/**'
+    - '.github/workflows/python.yaml'
 
 permissions:
   contents: read
@@ -14,50 +14,35 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, "3.10", "3.11"]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [ 3.8, 3.9, "3.10", "3.11" ]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
+    - name: Checkout
+      uses: actions/checkout@v4
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
 
-      # Windows
-      - name: Install OpenSSL
-        if: runner.os == 'Windows'   
-        id: vcpkg
-        uses: johnwason/vcpkg-action@v5
-        with:
-          pkgs: openssl
-          triplet: x64-windows-static
-          token: ${{ github.token }}
-
-      - name: Set OPENSSL_DIR environment variable
-        if: runner.os == 'Windows'   
-        shell: bash
-        run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-static" >> $GITHUB_ENV
-
-      - name: Test Python - Windows
-        if: runner.os == 'Windows'
-        run: |
-          pip install virtualenv
-          virtualenv venv; venv\Scripts\activate.ps1
-          pwd
-          python -m pip install --upgrade pip maturin
-          python -m pip install pytest
-          maturin develop --manifest-path py/Cargo.toml
-          pytest py
-          
-      # Non-windows
-      - name: Test Python - Non-Windows
-        if: runner.os != 'Windows'
-        run: |
-          pip install virtualenv
-          virtualenv venv; source venv/bin/activate
-          python -m pip install --upgrade pip maturin
-          python -m pip install pytest
-          maturin develop --manifest-path py/Cargo.toml
-          pytest py
+    - name: Test Python - Windows
+      if: runner.os == 'Windows'
+      run: |
+        pip install virtualenv
+        virtualenv venv; venv\Scripts\activate.ps1
+        pwd
+        python -m pip install --upgrade pip maturin
+        python -m pip install pytest
+        maturin develop --manifest-path py/Cargo.toml
+        pytest py
+    
+    # Non-windows
+    - name: Test Python - Non-Windows
+      if: runner.os != 'Windows'
+      run: |
+        pip install virtualenv
+        virtualenv venv; source venv/bin/activate
+        python -m pip install --upgrade pip maturin
+        python -m pip install pytest
+        maturin develop --manifest-path py/Cargo.toml
+        pytest py
 
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -4,11 +4,12 @@ on:
     tags:
     - 'v*'
 jobs:
-  build:
+  build-cli:
     env:
       CARGO_TERM_COLOR: always
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         build:
         - linux
@@ -17,30 +18,24 @@ jobs:
         include:
         - build: linux
           os: ubuntu-latest
-          rust: stable
           target: x86_64-unknown-linux-gnu
-          vcpkg_openssl_triplet: x64-linux-release
 
         - build: macos
           os: macos-latest
-          rust: stable
           target: x86_64-apple-darwin
-          vcpkg_openssl_triplet: x64-osx-release
 
         - build: windows
           os: windows-latest
-          rust: stable
           target: x86_64-pc-windows-msvc
-          vcpkg_openssl_triplet: x64-windows-static
 
     steps:
     - name: Checkout sources
       uses: actions/checkout@v4
 
     - name: Install Rust toolchain
-      uses: dtolnay/rust-toolchain@master
+      uses: dtolnay/rust-toolchain@stable
       with:
-        toolchain: ${{ matrix.rust }}
+        toolchain: stable
 
     - name: Check version number
       shell: bash
@@ -51,19 +46,10 @@ jobs:
           exit 1
         fi
 
-    - name: Install dependencies
-      id: vcpkg
-      uses: johnwason/vcpkg-action@v6
-      with:
-        pkgs: openssl
-        triplet: ${{ matrix.vcpkg_openssl_triplet }}
-        token: ${{ github.token }}
-
     - name: Build
       run: cargo build --bin yr --profile release-lto --target ${{ matrix.target }}
       env:
         RUSTFLAGS: "-C target-feature=+crt-static"
-        OPENSSL_DIR: "${{ github.workspace }}/vcpkg/installed/${{ matrix.vcpkg_openssl_triplet }}"
 
     - name: Build archive
       shell: bash
@@ -82,8 +68,111 @@ jobs:
         name: yr-${{ matrix.target }}
         path: yara-x-*
 
-  publish:
-    needs: [ build ]
+  build-py:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+        - cp39
+        - cp310
+        - cp311
+        - cp312
+        - pp38
+        - pp39
+        - pp310
+        build:
+        - linux
+        - macos
+        - windows
+        include:
+        - build: linux
+          os: ubuntu-latest
+
+        - build: macos
+          os: macos-latest
+          arch: 'arm64 x86_64'
+
+        - build: windows
+          os: windows-latest
+          arch: 'x86 AMD64'
+
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v3
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+      with:
+        toolchain: stable
+
+    - name: Check version number
+      shell: bash
+      run: |
+        set -ex
+        version=`cargo pkgid --manifest-path lib/Cargo.toml | cut -d "@" -f2`
+        if [ "${{ github.ref_name }}" != "v$version" ]; then
+          exit 1
+        fi
+
+    - name: Install Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+
+    - run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
+      if: matrix.build == 'macos'
+
+    #- run: rustup toolchain install stable-i686-pc-windows-msvc
+    #  if: matrix.build == 'windows'
+
+    #- run: rustup target add i686-pc-windows-msvc
+    #  if: matrix.build == 'windows'
+
+    - name: Install Python dependencies
+      run: pip install -U setuptools wheel twine cibuildwheel platformdirs
+
+    - name: Build sdist
+      if: matrix.build == 'linux' && matrix.python-version == 'cp312'
+      run: |
+        pip install maturin build
+        python -m build --sdist -o wheelhouse py
+
+    - name: Build ${{ matrix.platform || matrix.os }} binaries
+      run: cibuildwheel --output-dir wheelhouse py
+      env:
+        CIBW_BUILD: '${{ matrix.python-version }}-*'
+        # wasmtime doesn't support i686
+        CIBW_SKIP: '*_i686 *-musllinux* *-win32'
+        # we build for "arch" if it exists, else 'auto'
+        CIBW_ARCHS: ${{ matrix.arch || 'auto' }}
+
+        CIBW_BEFORE_BUILD_LINUX: >
+          curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=minimal -y &&
+          rustup show
+
+        CIBW_ENVIRONMENT_LINUX: 'PATH="$HOME/.cargo/bin:$PATH" '
+        CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"'
+
+        CIBW_TEST_REQUIRES: pytest
+        CIBW_TEST_COMMAND: 'pytest {project}/py'
+        CIBW_TEST_SKIP: '*-macosx_arm64 *-macosx_universal2:arm64'
+        CIBW_BUILD_VERBOSITY: 1
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: pypi-${{ matrix.build }}-${{ matrix.python-version }}
+        path: ./wheelhouse/*.whl
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: pypi-source-${{ strategy.job-index }}
+        path: ./wheelhouse/*.tar.gz
+
+  publish-cli:
+    needs: [ build-cli ]
     runs-on: ubuntu-latest
 
     steps:
@@ -100,4 +189,26 @@ jobs:
       uses: softprops/action-gh-release@v2
       with:
         draft: true
-        files: yr-*/yara-x-*
+        files: yr-*/yara-x-*
+
+  publish-py:
+    needs: [ build-py ]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/yara-x
+    permissions:
+      id-token: write
+    steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v4
+      with:
+        pattern: pypi-*
+        merge-multiple: true
+        path: dist
+
+    - name: Publish package distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        #repository-url: https://test.pypi.org/legacy/
+        skip-existing: true
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -75,21 +75,7 @@ jobs:
       if: runner.os == 'Linux'
       run: |
         sudo apt-get update
-        sudo apt-get install -y libssl-dev libmagic-dev
-
-    - name: Install dependencies
-      id: vcpkg
-      if: runner.os == 'Windows'
-      uses: johnwason/vcpkg-action@v5
-      with:
-        pkgs: openssl
-        triplet: x64-windows-release
-        token: ${{ github.token }}
-
-    - name: Set OPENSSL_DIR environment variable
-      if: runner.os == 'Windows'
-      shell: bash
-      run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-release" >> $GITHUB_ENV
+        sudo apt-get install -y libmagic-dev
 
     - name: Install Rust toolchain
       uses: dtolnay/rust-toolchain@master

diff --git a/py/README.md b/py/README.md
@@ -0,0 +1,21 @@
+This Python package allows using [YARA-X](https://virustotal.github.com/yara-x)
+from your Python programs.
+
+```python
+import yara_x
+
+rules = yara_x.compile('''
+  rule test { 
+    strings: 
+      $a = "foobar" 
+    condition: 
+      $a
+  }''')
+
+results = rules.scan(b"foobar")
+
+assert results.matching_rules[0].identifier == "test"
+assert results.matching_rules[0].patterns[0].identifier == "$a"
+assert results.matching_rules[0].patterns[0].matches[0].offset == 0
+assert results.matching_rules[0].patterns[0].matches[0].length == 6
+```
diff --git a/py/pyproject.toml b/py/pyproject.toml
@@ -3,15 +3,21 @@ requires = ["maturin>=1.0,<2.0"]
 build-backend = "maturin"
 
 [project]
-name = "yara-x-py"
+name = "yara-x"
+description = "Python bindings for YARA-X"
 requires-python = ">=3.8"
+readme = "README.md"
+keywords = ["pattern-matching", "cybersecurity", "forensics", "malware", "yara"]
 classifiers = [
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
-    "License :: OSI Approved :: MIT License",
+    "License :: OSI Approved :: BSD License",
 ]
 
+[project.urls]
+homepage = "https://virustotal.github.io/yara-x"
+repository = "https://github.com/VirusTotal/yara-x.git"
 
 [tool.maturin]
 features = ["pyo3/extension-module"]