diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 6492b81..9fdadee 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -1,29 +1,93 @@ --- name: Publish OpenSTL -on: push +on: + push: + branches: + - main + tags: + - 'v*' + pull_request: + branches: + - main + jobs: build: - name: Build distribution 📦 - timeout-minutes: 30 - if: startsWith(github.ref, 'refs/tags/') - runs-on: - - self-hosted - - manylinux - env: - PLAT: manylinux_2_28_x86_64 + if: "github.event_name == 'pull_request'" + name: Build and Test on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-13, macos-14] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.8' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install . + + - name: Run tests + run: | + python3 -m pip install numpy pytest + pytest + + bump_version_and_tag: + if: "github.event_name == 'push' && github.ref == 'refs/heads/main' && !startsWith(github.event.head_commit.message, 'bump:')" + name: Bump version and tag on main + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ssh-key: "${{ secrets.COMMIT_KEY }}" + - name: Create bump and changelog + uses: commitizen-tools/commitizen-action@master + with: + push: false + - name: Print Version + run: echo "Bumped to version ${{ steps.cz.outputs.version }}" + - name: Push using ssh + if: github.event_name == 'push' + run: | + git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}" + git config --global user.email "ruelj2@users.noreply.github.com" + git push --tags + git push + + build_wheels: + if: "startsWith(github.event.head_commit.message, 'bump:')" + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-13, macos-14] steps: - uses: actions/checkout@v4 - - name: Build a binary wheel and a source tarball -> manylinux_2_28_x86_64 - run: bash python/build-wheels.sh - - name: Store the distribution packages - uses: actions/upload-artifact@v3 + - uses: actions/setup-python@v5 + - name: Print the arch and system + run: | + python -c "import platform; print(f'System: {platform.system()}'); print(f'Architecture: {platform.architecture()[0]}')" + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.18.1 + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + - uses: actions/upload-artifact@v4 with: - name: python-package-distributions - path: dist/ + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + publish-to-testpypi: name: Publish Python 🐍 distribution 📦 to TestPyPI needs: - - build + - build_wheels runs-on: ubuntu-latest environment: name: testpypi @@ -32,37 +96,20 @@ jobs: id-token: write steps: - name: Download all the dists - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: python-package-distributions - path: dist/ + pattern: cibw-* + path: dist + merge-multiple: true - name: Publish distribution 📦 to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ - tests: - name: Test openstl testpypi distrib - needs: - - publish-to-testpypi - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["pypy3.7", "pypy3.8", "pypy3.9", "pypy3.10", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Test the python package - run: | - python3 -m pip install --index-url https://test.pypi.org/simple/ openstl - python3 -m pip install numpy pytest - pytest ./tests/python + publish-to-pypi: name: Publish Python 🐍 distribution 📦 to PyPI needs: - - tests + - publish-to-testpypi runs-on: ubuntu-latest environment: name: pypi @@ -71,12 +118,14 @@ jobs: id-token: write steps: - name: Download all the dists - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: python-package-distributions - path: dist/ + pattern: cibw-* + path: dist + merge-multiple: true - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + github-release: name: Sign the Python 🐍 distribution 📦 with Sigstore and upload them to GitHub Release @@ -88,10 +137,11 @@ jobs: id-token: write steps: - name: Download all the dists - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: python-package-distributions - path: dist/ + pattern: cibw-* + path: dist + merge-multiple: true - name: Sign the dists with Sigstore uses: sigstore/gh-action-sigstore-python@v1.2.3 with: diff --git a/CMakeLists.txt b/CMakeLists.txt index f3e37df..1e25b40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,10 +24,16 @@ set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -Wextra -std=c++11 -lstdc++ -pthread") -set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fno-math-errno") -set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lm ") +# Compiler-specific flags +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -pthread") + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fno-math-errno") + set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /EHsc") + set(CMAKE_CXX_FLAGS_RELEASE "/O2") + set(CMAKE_CXX_FLAGS_DEBUG "/Od /Zi") +endif() # Do not allow to build in main repo file(TO_CMAKE_PATH "${PROJECT_BINARY_DIR}/CMakeLists.txt" LOC_PATH) diff --git a/pyproject.toml b/pyproject.toml index 25739b0..568977b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,40 @@ [build-system] -requires = ["setuptools>=42", "wheel" , "ninja", "cmake>=3.15.3"] +requires = ["setuptools>=42", "wheel", "ninja", "cmake>=3.15.3"] build-backend = "setuptools.build_meta" +[tool.mypy] +files = "setup.py" +python_version = "3.7" +strict = true +show_error_codes = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +warn_unreachable = true + +[[tool.mypy.overrides]] +module = ["ninja"] +ignore_missing_imports = true + [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra -q" -testpaths = [ - "tests/python", +addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] +xfail_strict = true +filterwarnings = [ + "error", + "ignore:(ast.Str|Attribute s|ast.NameConstant|ast.Num) is deprecated:DeprecationWarning:_pytest", ] +testpaths = ["tests/python"] -[tool] [tool.commitizen] name = "cz_conventional_commits" version = "1.1.1" tag_format = "v$version" +[tool.cibuildwheel] +test-command = "pytest {project}/tests" +test-extras = ["test"] +test-skip = ["*universal2:arm64", "pp*", "cp{38,39,310,311,312}-manylinux_i686", "cp38-macosx_arm64", "*musllinux*", "*ppc64le", "*s390x"] +# Setuptools bug causes collision between pypy and cpython artifacts +before-build = "rm -rf {project}/build" [tool.poetry.dependencies] pybind11 = "v2.11.1" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 990344e..aae2c99 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,12 +1,5 @@ message(STATUS "Adding OpenSTL python binding") -#------------------------------------------------------------------------------- -# External libraries -#------------------------------------------------------------------------------- -FIND_PACKAGE(PythonInterp 3) -FIND_PACKAGE(PythonLibs 3) -find_package(Python COMPONENTS Interpreter Development) -find_package(pybind11 CONFIG) #------------------------------------------------------------------------------- # Internal libraries @@ -18,22 +11,12 @@ endif() #------------------------------------------------------------------------------- # Build Python Binding #------------------------------------------------------------------------------- -if(NOT PYTHONINTERP_FOUND) - message( FATAL_ERROR "PYTHONINTERP could not be found") -endif() -if(NOT PYTHONLIBS_FOUND) - message( FATAL_ERROR "PYTHONLIBS could not be found") -endif() - - -IF(PYTHONINTERP_FOUND AND PYTHONLIBS_FOUND) - file(GLOB_RECURSE python_SRC core/*.cpp) - pybind11_add_module(openstl MODULE ${python_SRC}) - target_include_directories(openstl PRIVATE ${PYBIND11_SUBMODULE}/include) - target_link_libraries(openstl PRIVATE openstl::core pybind11::headers) - target_compile_definitions(openstl PRIVATE VERSION_INFO=${PROJECT_VERSION}) - set_target_properties(openstl PROPERTIES - INTERPROCEDURAL_OPTIMIZATION ON - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN ON) -ENDIF() \ No newline at end of file +file(GLOB_RECURSE python_SRC core/*.cpp) +pybind11_add_module(openstl MODULE ${python_SRC}) +target_include_directories(openstl PRIVATE ${PYBIND11_SUBMODULE}/include) +target_link_libraries(openstl PRIVATE openstl::core pybind11::headers) +target_compile_definitions(openstl PRIVATE VERSION_INFO=${PROJECT_VERSION}) +set_target_properties(openstl PROPERTIES + INTERPROCEDURAL_OPTIMIZATION ON + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) diff --git a/python/build-wheels.sh b/python/build-wheels.sh deleted file mode 100644 index 609b956..0000000 --- a/python/build-wheels.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e -u -x - -function repair_wheel { - wheel="$1" - if ! auditwheel show "$wheel"; then - echo "Skipping non-platform wheel $wheel" - else - auditwheel repair "$wheel" --plat "$PLAT" -w ./dist/ - fi -} - -# Compile wheels -for PYBIN in /opt/python/*/bin; do - OPENSTL_SOURCE_DIR=$PWD "${PYBIN}/pip" wheel . --no-deps -w dist/ -done - -# Bundle external shared libraries into the wheels -for whl in dist/*.whl; do - repair_wheel "$whl" - rm $whl -done \ No newline at end of file diff --git a/python/core/src/stl.cpp b/python/core/src/stl.cpp index 5fc59a6..507643f 100644 --- a/python/core/src/stl.cpp +++ b/python/core/src/stl.cpp @@ -73,7 +73,7 @@ namespace pybind11 { namespace detail { bool load(handle src, bool convert) { - if ( !convert and !py::array_t::check_(src) ) + if ( (!convert) && (!py::array_t::check_(src)) ) return false; auto buf = py::array_t::ensure(src); diff --git a/setup.py b/setup.py index 2ddd616..687e6b4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import os, re, sys import subprocess -import sysconfig from pathlib import Path from setuptools import Extension, setup @@ -21,6 +20,14 @@ def read_version_from_pyproject(file_path): return None +# Convert distutils Windows platform specifiers to CMake -A arguments +PLAT_TO_CMAKE = { + "win32": "Win32", + "win-amd64": "x64", + "win-arm32": "ARM", + "win-arm64": "ARM64", +} + class CMakeExtension(Extension): def __init__(self, name: str, sourcedir: str = "", cmake: str = "cmake") -> None: super().__init__(name, sources=[]) @@ -29,42 +36,94 @@ def __init__(self, name: str, sourcedir: str = "", cmake: str = "cmake") -> None class CMakeBuild(build_ext): - # Inspired by pybind/cmake_example - # https://github.com/pybind/cmake_example/blob/835e1a81b01d06097ccbb7b8f214ef9bd2d0c159/setup.py + # Inspired from https://github.com/pybind/cmake_example/blob/master/setup.py def build_extension(self, ext: CMakeExtension) -> None: ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) extdir = ext_fullpath.parent.resolve() + debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug + cfg = "Debug" if debug else "Release" + + # CMake lets you override the generator - we need to check this. + # Can be set with Conda-Build, for example. + cmake_generator = os.environ.get("CMAKE_GENERATOR", "") + cmake_args = [ f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}", f"-DPYTHON_EXECUTABLE={sys.executable}", - f"-DPYTHON_INCLUDE_DIR={sysconfig.get_path('include')}", - f"-DPYTHON_LIBRARY={sysconfig.get_config_var('LIBDIR')}", - f"-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_BUILD_TYPE={cfg}", + "-DCMAKE_BUILD_TYPE=Release", '-DCMAKE_INSTALL_RPATH=$ORIGIN', '-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON', '-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=OFF', '-DOPENSTL_BUILD_PYTHON:BOOL=ON' ] + build_args = [] + # Adding CMake arguments set as environment variable + # (needed e.g. to build for ARM OSx on conda-forge) if "CMAKE_ARGS" in os.environ: cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] - + if self.compiler.compiler_type != "msvc": + # Using Ninja-build since it a) is available as a wheel and b) + # multithreads automatically. MSVC would require all variables be + # exported for Ninja to pick it up, which is a little tricky to do. + # Users can override the generator with CMAKE_GENERATOR in CMake + # 3.15+. + if not cmake_generator or cmake_generator == "Ninja": + try: + import ninja + + ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" + cmake_args += [ + "-GNinja", + f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", + ] + except ImportError: + pass + + else: + # Single config generators are handled "normally" + single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) + + # CMake allows an arch-in-generator style for backward compatibility + contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) + + # Specify the arch if using MSVC generator, but only if it doesn't + # contain a backward-compatibility arch spec already in the + # generator name. + if not single_config and not contains_arch: + cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] + + # Multi-config generators have a different way to specify configs + if not single_config: + cmake_args += [ + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" + ] + build_args += ["--config", cfg] + + if sys.platform.startswith("darwin"): + # Cross-compile support for macOS - respect ARCHFLAGS if set + archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", "")) + if archs: + cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] + + # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level + # across all generators. if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: + # self.parallel is a Python 3 only way to set parallel jobs by hand + # using -j in the build_ext call, not supported by pip or PyPA-build. if hasattr(self, "parallel") and self.parallel: + # CMake 3.12+ only. build_args += [f"-j{self.parallel}"] build_temp = Path(self.build_temp) / ext.name if not build_temp.exists(): build_temp.mkdir(parents=True) - subprocess.run( - [ext.cmake, ext.sourcedir] + cmake_args, cwd=build_temp, check=True - ) - subprocess.run( - [ext.cmake, "--build", "."] + build_args, cwd=build_temp, check=True - ) + subprocess.run([ext.cmake, ext.sourcedir] + cmake_args, cwd=build_temp, check=True) + subprocess.run([ext.cmake, "--build", "."] + build_args, cwd=build_temp, check=True) test_deps = [ 'coverage', @@ -100,6 +159,10 @@ def build_extension(self, ext: CMakeExtension) -> None: "Topic :: Scientific/Engineering :: Visualization", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7",