too recent symbols, unable to repair the wheel

[alexlnkp]

this isn't as much of an "issue" as it is a plea for help

i'm trying to setup the deployment of a python module written in C that uses OpenMP for parallel computation
the issue at hand is that i'm stumped about how to even convert a linux-arch-specific wheel to a manylinux wheel without issues. i don't care if my python module ends up being distributed as a binary or source code that gets compiled on the user machine, in fact i'd prefer if it were the latter, however i'm simply unable to push a wheel to pypi because it's not a manylinux wheel...

workflow file:

name: Publish Python distribution to TestPyPI

on: push

jobs:
  build:
    name: Build distribution 📦
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: "3.x"
    - name: Pull docker image
      run: docker pull quay.io/pypa/manylinux2014_x86_64
    - name: Run docker container
      run: docker run --rm -v `pwd`:/io quay.io/pypa/manylinux2014_x86_64 /bin/bash
    - name: Install pypa/build
      run: >-
        python3 -m
        pip install
        build auditwheel
        --user
    - name: Build a binary wheel and a source tarball
      run: python -m build
    - name: Convert binary wheel to manylinux
      run: auditwheel repair dist/*.whl --plat manylinux2014_x86_64
    - name: Store the distribution packages
      uses: actions/upload-artifact@v3
      with:
        name: python-package-distributions
        path: dist/
  publish-to-pypi:
    name: >-
      Publish Python 🐍 distribution 📦 to PyPI
    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
    needs:
    - build
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/p/bulkhasher
    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing
    steps:
    - name: Download all the dists
      uses: actions/download-artifact@v3
      with:
        name: python-package-distributions
        path: dist/
    - name: Publish distribution 📦 to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
  github-release:
    name: >-
      Sign the Python 🐍 distribution 📦 with Sigstore
      and upload them to GitHub Release
    needs:
    - publish-to-pypi
    runs-on: ubuntu-latest

    permissions:
      contents: write  # IMPORTANT: mandatory for making GitHub Releases
      id-token: write  # IMPORTANT: mandatory for sigstore

    steps:
    - name: Download all the dists
      uses: actions/download-artifact@v3
      with:
        name: python-package-distributions
        path: dist/
    - name: Sign the dists with Sigstore
      uses: sigstore/[email protected]
      with:
        inputs: >-
          ./dist/*.tar.gz
          ./dist/*.whl
    - name: Create GitHub Release
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: >-
        gh release create
        '${{ github.ref_name }}'
        --repo '${{ github.repository }}'
        --notes ""
    - name: Upload artifact signatures to GitHub Release
      env:
        GITHUB_TOKEN: ${{ github.token }}
      # Upload to GitHub Release using the `gh` CLI.
      # `dist/` contains the built packages, and the
      # sigstore-produced signatures and certificates.
      run: >-
        gh release upload
        '${{ github.ref_name }}' dist/**
        --repo '${{ github.repository }}'
  publish-to-testpypi:
    name: Publish Python 🐍 distribution 📦 to TestPyPI
    needs:
    - build
    runs-on: ubuntu-latest

    environment:
      name: testpypi
      url: https://test.pypi.org/p/bulkhasher

    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing

    steps:
    - name: Download all the dists
      uses: actions/download-artifact@v3
      with:
        name: python-package-distributions
        path: dist/
    - name: Publish distribution 📦 to TestPyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        repository-url: https://test.pypi.org/legacy/

please keep in mind i'm not that smart, especially when it comes to distributing packages and such, so i'm bound to make extremely silly mistakes, but here i'm genuinely stumped as to what the issue is; it looks like i'm doing everything according to instructions, pulling docker image, opening the container, building using setup.py, but still it just wont work for some reason...

just in case, here's my CMakeLists.txt:

cmake_minimum_required(VERSION 3.12 FATAL_ERROR)

project(RVC_Hasher)

find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
find_package(OpenMP REQUIRED)

# Define the new shared library target
add_library(bulkhasher MODULE hash.c sha2.c)
target_compile_options(bulkhasher PRIVATE -Wall -Wextra -Wpedantic -O3 -flto -fopenmp -Wno-unused-parameter)

# Include Python headers and OpenMP
target_include_directories(bulkhasher PRIVATE ${Python3_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(bulkhasher PRIVATE Python3::Python Python3::Module OpenMP::OpenMP_C)

# Set the output name of the library to 'bulkhasher' without any prefix (like 'lib')
set_target_properties(bulkhasher PROPERTIES PREFIX "")
set_target_properties(bulkhasher PROPERTIES OUTPUT_NAME "bulkhasher")

# Optionally, set the suffix to .pyd for Windows or .so (the default) for Unix
if(WIN32)
    set_target_properties(bulkhasher PROPERTIES SUFFIX ".pyd")
endif()

# Set the C standard and required flag
set(CMAKE_C_STANDARD 17)
set(CMAKE_C_STANDARD_REQUIRED ON)

target_compile_options(bulkhasher PRIVATE -Wall -Wextra -Wpedantic -O3 -fopenmp)

and the setup.py:

from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext
import subprocess
import platform
import sys
import os

LIBRARY_SUFFIX = 'so' if platform.system() == 'Linux' else 'pyd'

class CMakeExtension(Extension):
    def __init__(self, name, sourcedir='.'):
        Extension.__init__(self, name, sources=[])
        self.sourcedir = os.path.abspath(sourcedir)

class CMakeBuild(build_ext):
    def run(self):
        for ext in self.extensions:
            self.build_extension(ext)

    def build_extension(self, ext):
        extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
        cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
                      '-DPYTHON_EXECUTABLE=' + sys.executable]

        cfg = 'Debug' if self.debug else 'Release'
        build_args = ['--config', cfg]

        os.makedirs(self.build_temp, exist_ok=True)
        os.makedirs(self.build_lib, exist_ok=True)

        subprocess.check_call(['cmake', '-B', self.build_temp, '-DCMAKE_BUILD_TYPE=' + cfg, ext.sourcedir] + cmake_args, cwd=self.build_temp)
        subprocess.check_call(['cmake', '--build', self.build_temp] + build_args, cwd=self.build_temp)


setup(
    name = 'bulkhasher',
    version = '0.0.4',
    description = 'Module for bulk hashing of files',
    long_description="Made on C",
    author = 'Alex Murkoff',
    author_email = '[email protected]',
    packages=['bulkhasher'],
    package_dir={'': '.'},
    package_data={
        'bulkhasher': [
            f'bulkhasher.{LIBRARY_SUFFIX}'
        ]
    },
    ext_modules=[CMakeExtension('bulkhasher')],
    cmdclass=dict(build_ext=CMakeBuild),
    zip_safe=False,
)

i'm not sure what to do at this point, auditwheel doesn't like openmp's library usage in my module, also it doesn't like Python's .so usage too, for some reason?

the module itself works fine and builds without any issues, but pypi just wont accept it

i've looked into other issues of the same topic, but none of them are working for me.

[calendulish]

manylinux2014_x86_64 is too old for your application and will reach the end of life on June 30th, 2024.
use a more recent one, as manylinux_2_28_x86_64

[alexlnkp]

manylinux2014_x86_64 is too old for your application and will reach the end of life on June 30th, 2024. use a more recent one, as manylinux_2_28_x86_64

I think this helped? Though still getting the same error

error: cannot repair "dist/bulkhasher-0.0.4-cp312-cp312-linux_x86_64.whl" to "manylinux_2_28_x86_64" ABI because of the presence of too-recent versioned symbols. You'll need to compile the wheel on an older toolchain.

[mayeut]

If I'm reading the workflow correctly, the build and repair steps are actually being run on the GHA runner rather than in the container.
You'll have to run the build inside the manylinux container if you want to achieve some kind of compatibility.
By default, when not running inside a manylinux container that exposes the correct AUDITWHEEL_PLAT environment variable, auditwheel targets manylinux1. Given it's being run on the GHA runner (ubuntu 22.04), it won't be able to achieve manylinux1 compatibility.
You can probably have a look at https://github.com/pypa/cibuildwheel to help you setup a workflow to build wheels.

[alexlnkp]

I tried cibuildwheel, but it seems to be unable to locate python library?

   CMake Error at /opt/_internal/pipx/venvs/cmake/lib/python3.10/site-packages/cmake/data/share/cmake-3.29/Modules/FindPackageHandleStandardArgs.cmake:230 (message):
    Could NOT find Python3 (missing: Python3_LIBRARIES Development
    Development.Embed) (found version "3.13.0")
  Call Stack (most recent call first):
    /opt/_internal/pipx/venvs/cmake/lib/python3.10/site-packages/cmake/data/share/cmake-3.29/Modules/FindPackageHandleStandardArgs.cmake:600 (_FPHSA_FAILURE_MESSAGE)
    /opt/_internal/pipx/venvs/cmake/lib/python3.10/site-packages/cmake/data/share/cmake-3.29/Modules/FindPython/Support.cmake:3867 (find_package_handle_standard_args)
    /opt/_internal/pipx/venvs/cmake/lib/python3.10/site-packages/cmake/data/share/cmake-3.29/Modules/FindPython3.cmake:545 (include)
    CMakeLists.txt:7 (find_package)

I even started passing the python's directories explicitly to CMake, still to no avail

[mayeut]

You should probably be using Development.Module rather than Development.Embed (or just Development).
There are no libraries needed for Development.Module on Linux (one should not linked against the library as the module wouldn't work with static interpreters).

Edit: Just saw the CMakeLists.txt
Try replacing Development by Development.Module. This will need a bump to the minimum cmake version (I think 3.18).
You might also be interested in scikit-build-core as a backend rather than setuptools.

[alexlnkp]

You should probably be using Development.Module rather than Development.Embed (or just Development). There are no libraries needed for Development.Module on Linux (one should not linked against the library as the module wouldn't work with static interpreters).

Edit: Just saw the CMakeLists.txt Try replacing Development by Development.Module. This will need a bump to the minimum cmake version (I think 3.18). You might also be interested in scikit-build-core as a backend rather than setuptools.

That... Worked!!!
I'm sorry if this mistake of mine was quite silly... I'm new to making Python modules. When I see a linker angry I just link wherever I find the functions it can't, didn't think such semantics could matter...

Regardless, thank You very much for the help, I don't think I would be able to solve this on my own (heck, I've even considered just shipping python code with my Python module as a git submodule, so it would build itself for the user's platform regardless; glad I didn't end up doing that).

Scikit-Build-Core looks promising, but even scarier than setuptools... I'll look into it though since a lot of C/C++ projects use it to build python modules.

[alexlnkp]

Answered by @mayeut !
Thank You all for help :)