Merge pull request #58 from haosulab/pybind11_mkdoc

Automatically generate pybind11 docstring from C++ code comments

.github/workflows/build_and_publish.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        pyver: [cp36, cp37, cp38, cp39, cp310, cp311, cp312]
+        pyver: [cp37, cp38, cp39, cp310, cp311, cp312]
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.gitignore

@@ -37,3 +37,5 @@ imgui.ini
 
 # Converted URDF that used to contain package:// paths
 **/*_package_keyword_replaced.urdf
+dev/drake
+dev/pybind11_mkdoc

CMakeLists.txt

@@ -23,6 +23,7 @@ endif(CCACHE_FOUND)
 # Pinocchio uses its own FindCppAD, but does not provide it.
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
 
+set(Boost_NO_WARN_NEW_VERSIONS 1)  # silence Boost CMake warnings
 find_package(Eigen3 3.4.0 REQUIRED)
 find_package(Boost COMPONENTS system filesystem REQUIRED)
 find_package(ompl REQUIRED)
@@ -32,22 +33,36 @@ find_package(assimp REQUIRED)
 find_package(orocos_kdl REQUIRED)
 find_package(urdfdom REQUIRED)
 
-include_directories("/usr/include/eigen3")
 include_directories(${OMPL_INCLUDE_DIRS} ${urdfdom_INCLUDE_DIRS})
 include_directories("src")
 
 # store libries in a variable
 set(LIBS ompl fcl assimp orocos-kdl Boost::system Boost::filesystem urdfdom_model urdfdom_world)
 
+# pymp
 file(GLOB_RECURSE PROJECT_SRC "src/*.h" "src/*.cpp" "src/*.hpp")
 add_library(mp STATIC ${PROJECT_SRC})
 target_link_libraries(mp PRIVATE ${LIBS})
 set_target_properties(mp PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
 
+# pybind11_mkdoc
+add_custom_target(
+  pybind11_mkdoc ALL
+  COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/dev/mkdoc.sh"
+               "-I$<JOIN:$<TARGET_PROPERTY:mp,INCLUDE_DIRECTORIES>,;-I>"
+  BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/python/docstring/*.h"
+  DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/dev/mkdoc.sh" "${CMAKE_CURRENT_SOURCE_DIR}/src/*.h"
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND_EXPAND_LISTS
+  VERBATIM
+)
+
+# Pybind11
 add_subdirectory("third_party/pybind11")
 include_directories("python")
 pybind11_add_module(pymp python/pybind.cpp)
 target_link_libraries(pymp PRIVATE mp)
+add_dependencies(pymp pybind11_mkdoc)
 
 # compile test_articulated_model and run the test
 add_executable(test_articulated_model tests/test_articulated_model.cpp)

README.md

@@ -17,7 +17,7 @@ functionalities in robot manipulation.
 
 ## Installation
 
-Pre-built pip packages support Ubuntu 18.04+ with Python 3.6+.
+Pre-built pip packages support Ubuntu 18.04+ with Python 3.7+.
 
 ```
 pip install mplib

dev/README.md

@@ -8,8 +8,10 @@ Most dependencies have been installed.
 The docker file can be found [here](../docker/Dockerfile).
 
 To building python wheels, run `./dev/build_wheels.sh --py 310`  
-This will install [`cibuildwheel`](https://cibuildwheel.readthedocs.io/en/stable/#how-it-works) via pip and use it to build wheel for the specified python version.  
-The wheel will be generated in the `wheelhouse/`
+This will create a docker container from [`kolinguo/mplib-build`](https://hub.docker.com/r/kolinguo/mplib-build)
+and use it to build wheel for the specified python version.  
+The wheel will be generated in the `wheelhouse/` and the generated pybind11 docstring
+will be in `python/docstring/`.
 
 If you want to start a docker container for debugging, run `./dev/docker_setup.sh`
 
@@ -69,13 +71,29 @@ Depending on your python version, you will get a file called `pymp.cpython-310-x
 
 To install the entire package along with python glue code, do `python3.[version] -m pip install .` inside the root directory of the project.
 
+## Docstring Generation from C++ Comments
+Based on [`pybind11_mkdoc`](https://github.com/pybind/pybind11_mkdoc), we created
+[`./dev/mkdoc.py`](./mkdoc.py) to automatically generate pybind11 docstrings from
+C++ code comments. In [`CMakeLists.txt`](../CMakeLists.txt), a custom target will run
+[`./dev/mkdoc.sh`](./mkdoc.sh) which calls `./dev/mkdoc.py` for all header files.
+The generated docstrings will be stored in [`./python/docstring/`](../python/docstring/).
+
+The expected C++ code comments should follow the [`doxygen`](https://doxygen.nl/manual/docblocks.html) format
+([an example header file](./test_mkdoc/mplib_sample/sample_header.h))
+and the generated pybind11 docstrings (which will be used to generate python docstrings)
+will be in Sphinx [`reStructuredText (reST)`](https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html) format.
+
+To run on local environment, please install `python3 -m pip install libclang=={clang-version}`
+where the `clang-version` should match your local LLVM installation
+`/usr/lib/llvm-{ver}/lib/clang/{clang-version}/`.
+
 ## Stubs & Documentation Generation
 
-To generate stubs and documentations, run `./dev/generate_stub_and_doc.sh`.
+To generate stubs and documentations, run [`./dev/generate_stub_and_doc.sh`](./generate_stub_and_doc.sh).
 By default it uses `python3.10` in docker image [`kolinguo/mplib-build`](https://hub.docker.com/r/kolinguo/mplib-build).
 
 The script does the following:
-* Build a python wheel using [`cibuildwheel`](https://cibuildwheel.readthedocs.io/en/stable/#how-it-works).
+* Build a python wheel using [`./dev/build_wheels.sh`](./build_wheels.sh).
 * In a docker container, install the python wheel and
 use [`pybind11-stubgen`](https://github.com/sizmailov/pybind11-stubgen)
 to generate stubs.  
@@ -86,6 +104,7 @@ Copy the generated docs into [`docs`](../docs/).
 
 ## GitHub Action CI/CD
 Currently, a GitHub action is setup to build / release / publish python wheels.
+Building wheels are done using [`cibuildwheel`](https://cibuildwheel.readthedocs.io/en/stable/#how-it-works).
 
 ### Push to `main` branch
 This triggers building wheels for all supported python versions and

dev/build_wheels.sh

@@ -1,12 +1,40 @@
 #!/bin/bash
 
+# Docker image name to build wheel
+IMGNAME="kolinguo/mplib-build:latest"
+
+############################################################
+# Section 0: Bash Error Handling                           #
+############################################################
 set -eEu -o pipefail
+trap 'catch' ERR  # Trap all errors (status != 0) and call catch()
+catch() {
+  local err="$?"
+  local err_command="$BASH_COMMAND"
+  set +xv  # disable trace printing
+
+  echo -e "\n\e[1;31mCaught error in ${BASH_SOURCE[1]}:${BASH_LINENO[0]} ('${err_command}' exited with status ${err})\e[0m" >&2
+  echo "Traceback (most recent call last, command might not be complete):" >&2
+  for ((i = 0; i < ${#FUNCNAME[@]} - 1; i++)); do
+    local funcname="${FUNCNAME[$i]}"
+    [ "$i" -eq "0" ] && funcname=$err_command
+    echo -e "  ($i) ${BASH_SOURCE[$i+1]}:${BASH_LINENO[$i]}\t'${funcname}'" >&2
+  done
+  exit "$err"
+}
 
+############################################################
+# Section 1: Build python wheel in a docker container      #
+############################################################
 # Move to the repo folder, so later commands can use relative paths
 SCRIPT_PATH=$(readlink -f "$0")
 REPO_DIR=$(dirname "$(dirname "$SCRIPT_PATH")")
 cd "$REPO_DIR"
 
+echo_info() {
+  echo -e "\n\e[1;36m$1 ...\e[0m"
+}
+
 PY_VERSION=
 while (("$#")); do
   case "$1" in
@@ -26,17 +54,47 @@ while (("$#")); do
   esac
 done
 
+# Actual function to build wheel
+build_wheel() {
+  BUILD_WHEEL_CMD="\
+    export PATH=\"\$(find /opt/python -name \"cp${PY_VERSION}*\")/bin:\${PATH}\" \
+    && rm -rf build dist/* wheelhouse/* \
+    && git config --global --add safe.directory '*' \
+    && python3 -m build --wheel \
+    && auditwheel repair \$(find dist/ -name \"*.whl\")
+  "
+
+  echo_info "Building wheel for python${PY_VERSION} in docker '${IMGNAME}'"
+  local temp_cont_name="mplib_build_$(date "+%Y%m%d_%H%M%S")"
+  docker create --name="$temp_cont_name" \
+    "$IMGNAME" \
+    bash -c "$BUILD_WHEEL_CMD"
+  docker cp . "${temp_cont_name}:/MPlib"
+  docker start -a "$temp_cont_name"
+  docker cp "${temp_cont_name}:/MPlib/wheelhouse" .
+  docker cp "${temp_cont_name}:/MPlib/python/docstring" ./python
+  docker rm -f "$temp_cont_name"
+}
+
 if [ -z "$PY_VERSION" ]; then
   echo "Error: No python version is provided"
   exit 3
 fi
 
-if ! command -v "cibuildwheel" &>/dev/null; then
-  python3 -m pip install cibuildwheel
-fi
-
 if [ "$PY_VERSION" == "all" ]; then
-  python3 -m cibuildwheel --platform linux
+  # python3 -m cibuildwheel --platform linux
+  for PY_VERSION in 37 38 39 310 311 312; do
+    build_wheel
+  done
 else
-  CIBW_BUILD="cp${PY_VERSION}-*" python3 -m cibuildwheel --platform linux
+  # CIBW_BUILD="cp${PY_VERSION}-*" python3 -m cibuildwheel --platform linux
+  case "$PY_VERSION" in
+    37|38|39|310|311|312) ;;
+    *)
+      echo "Error: Python version($PY_VERSION) not supported" >&2
+      exit 4
+      ;;
+  esac
+
+  build_wheel
 fi

dev/generate_stub_and_doc.sh

@@ -47,13 +47,16 @@ dev/build_wheels.sh --py "$PY_VERSION"
 ############################################################
 # Section 2: Build stubs                                   #
 ############################################################
+# Build stub and run ruff isort / formatter
 BUILD_STUB_CMD="\
-  export PATH=\"/opt/python/cp${PY_VERSION}-cp${PY_VERSION}/bin:\${PATH}\" \
+  export PATH=\"\$(find /opt/python -name \"cp${PY_VERSION}*\")/bin:\${PATH}\" \
   && python3 -m pip install pybind11-stubgen \
   && python3 -m pip install wheelhouse/mplib*.whl \
-  && python3 dev/stubgen.py
+  && python3 dev/stubgen.py \
+  && python3 -m pip install ruff \
+  && ruff check --select I --fix ./stubs \
+  && ruff format ./stubs
 "
-# TODO: add ruff
 
 echo_info "Building stubs in docker '${IMGNAME}'"
 docker run -it --rm \
@@ -74,7 +77,7 @@ rm -rfv stubs/
 # TODO: switch to other tools to generate docs, README.md should not be included in wheels
 # TODO: do we must install sapien to generate doc?
 BUILD_DOC_CMD="\
-  export PATH=\"/opt/python/cp${PY_VERSION}-cp${PY_VERSION}/bin:\${PATH}\" \
+  export PATH=\"\$(find /opt/python -name \"cp${PY_VERSION}*\")/bin:\${PATH}\" \
   && python3 -m pip install pdoc \
   && python3 -m pip install sapien==3.0.0.dev0 \
   && python3 -m pip install wheelhouse/mplib*.whl \