bindings CHANGE rewrite python binding from scratch

Signed-off-by: Robin Jarry <[email protected]>

.travis.yml

@@ -44,7 +44,7 @@ before_install:
   - cmake .. && make -j2 && sudo make install
   - cd ../..
   - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew update; fi
-  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq; sudo apt-get install -y valgrind libpcre3-dev python3-dev swig; fi
+  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq; sudo apt-get install -y valgrind libpcre3-dev python3-dev swig python3-cffi python3-setuptools; fi
   - if [ "$TRAVIS_OS_NAME" = "linux" -a "$CC" = "gcc" -a "$TRAVIS_ARCH" = "amd64" ]; then pip install --user codecov; export CFLAGS="-coverage"; fi
 
 script:

CMakeLists.txt

@@ -106,9 +106,7 @@ set(USER_TYPES_PLUGINS_DIR_MACRO "${PLUGINS_DIR}/user_types")
 # setup bindings
 set(GEN_LANGUAGE_BINDINGS 0 CACHE BOOL "Enable language bindings generation.")
 set(GEN_CPP_BINDINGS 1 CACHE BOOL "Enable C++ bindings.")
-# Python bindings depend on C++ bindings because of SWIG
 set(GEN_PYTHON_BINDINGS 1 CACHE BOOL "Enable Python bindings.")
-set(GEN_PYTHON_VERSION "3" CACHE STRING "Python version")
 set(GEN_JAVASCRIPT_BINDINGS 0 CACHE BOOL "Enable JavaScript bindings.")
 set(GEN_JAVA_BINDINGS 0 CACHE BOOL "Enable Java bindings.")
 
@@ -376,3 +374,11 @@ endif()
 if(GEN_LANGUAGE_BINDINGS AND GEN_CPP_BINDINGS)
     add_subdirectory(swig)
 endif()
+
+if(GEN_LANGUAGE_BINDINGS AND GEN_PYTHON_BINDINGS)
+    if(ENABLE_STATIC)
+        message(WARNING "Python bindings do not work with a static build.")
+    else()
+        add_subdirectory(python)
+    endif()
+endif()

packages/CMakeLists.txt

@@ -1,6 +1,6 @@
 set(PACKAGE "libyang")
 set(CPP_PACKAGE "libyang-cpp")
-set(PYTHON_PACKAGE "python3-yang")
+set(PYTHON_PACKAGE "python3-libyang")
 
 find_program(DEB_BUILDER NAMES debuild)
 find_program(RPM_BUILDER NAMES rpmbuild)
@@ -22,9 +22,8 @@ configure_file(${PROJECT_SOURCE_DIR}/packages/debian.${CPP_PACKAGE}.install
                ${PROJECT_BINARY_DIR}/build-packages/debian.${CPP_PACKAGE}.install COPYONLY)
 configure_file(${PROJECT_SOURCE_DIR}/packages/debian.${CPP_PACKAGE}-dev.install
                ${PROJECT_BINARY_DIR}/build-packages/debian.${CPP_PACKAGE}-dev.install COPYONLY)
-# no python package for Debian because there is only SWIG 3.10 on Debian 9 :-/
-#configure_file(${PROJECT_SOURCE_DIR}/packages/debian.${PYTHON_PACKAGE}.install
-#               ${PROJECT_BINARY_DIR}/build-packages/debian.${PYTHON_PACKAGE}.install COPYONLY)
+configure_file(${PROJECT_SOURCE_DIR}/packages/debian.${PYTHON_PACKAGE}.install
+               ${PROJECT_BINARY_DIR}/build-packages/debian.${PYTHON_PACKAGE}.install COPYONLY)
 
 if(NOT DEB_BUILDER)
     message(STATUS "Missing tools (devscripts, debhelper package) for building DEB package.")

packages/debian.control.in

@@ -2,7 +2,13 @@ Source: @PACKAGE@
 Maintainer: CESNET <[email protected]>
 Priority: extra
 Standards-Version: 3.8.2
-Build-Depends: debhelper (>= 9), gcc
+Build-Depends:
+ debhelper (>= 9),
+ gcc,
+ cmake,
+ make,
+ python3 (>= 3.5),
+ dh-python (>= 3.20180313~),
 Homepage: https://github.com/CESNET/libyang
 
 Package: @PACKAGE@
@@ -40,3 +46,18 @@ Depends: @CPP_PACKAGE@ (=@LIBYANG_VERSION@)
 Section: debug
 Architecture: any
 Description: Debug symbols of C++ bidings of libyang library.
+
+Package: @PYTHON_PACKAGE@
+Depends:
+ ${misc:Depends},
+ ${python3:Depends},
+ @PACKAGE@ (=@LIBYANG_VERSION@),
+Section: libs
+Architecture: any
+Description: Bindings of libyang library to Python language.
+
+Package: @PYTHON_PACKAGE@-dbg
+Depends: @PYTHON_PACKAGE@ (=@LIBYANG_VERSION@)
+Section: debug
+Architecture: any
+Description: Debug symbols of Python bindings of libyang library.

packages/debian.python3-libyang.install

@@ -0,0 +1 @@
+usr/lib/python3*/dist-packages/*

packages/debian.rules.in

@@ -4,16 +4,16 @@
 export DH_VERBOSE=1
 
 %:
-	dh $@
+	dh $@ --with python3
 
 override_dh_strip:
 	dh_strip -p@PACKAGE@  --dbg-package=@PACKAGE@-dbg
 	dh_strip -p@CPP_PACKAGE@ --dbg-package=@CPP_PACKAGE@-dbg
-#	dh_strip -p@PYTHON_PACKAGE@ --dbg-package=@PYTHON_PACKAGE@-dbg
+	dh_strip -p@PYTHON_PACKAGE@ --dbg-package=@PYTHON_PACKAGE@-dbg
 
 override_dh_auto_configure:
 	cmake -DCMAKE_INSTALL_PREFIX=/usr -DFORCE_INSRC_BUILD=ON -DCMAKE_BUILD_TYPE="Package" -DENABLE_LYD_PRIV=ON \
-	-DGEN_LANGUAGE_BINDINGS=ON -DGEN_PYTHON_BINDINGS=OFF .
+	-DGEN_LANGUAGE_BINDINGS=ON -DGEN_PYTHON_BINDINGS=ON .
 
 override_dh_auto_test:
 	ctest --output-on-failure

python/CMakeLists.txt

@@ -0,0 +1,86 @@
+unset(PYTHON_LIBRARY CACHE)
+unset(PYTHON_EXECUTABLE CACHE)
+unset(PYTHON_INCLUDE_DIR CACHE)
+
+find_package(PythonInterp 3 REQUIRED)
+find_package(PythonLibs 3 REQUIRED)
+foreach(dep setuptools cffi)
+    execute_process(
+        COMMAND ${PYTHON_EXECUTABLE} -B -c "import ${dep}"
+        RESULT_VARIABLE PYTHON_DEP_RESULT)
+    if(NOT ${PYTHON_DEP_RESULT} EQUAL 0)
+        message(FATAL_ERROR "Missing required python module: ${dep}")
+    endif()
+endforeach()
+
+set(PYTHON_MODULE_PATH_CMD "
+from distutils.sysconfig import get_python_lib
+print(get_python_lib(prefix='${CMAKE_INSTALL_PREFIX}', plat_specific=True))
+")
+execute_process(
+    COMMAND ${PYTHON_EXECUTABLE} -B -c "${PYTHON_MODULE_PATH_CMD}"
+    OUTPUT_VARIABLE PYTHON_MODULE_PATH
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+set(PYTHON_LIB_BUILDDIR_CMD "
+import sys
+from distutils.util import get_platform
+print('lib.' + get_platform() + '-{}.{}'.format(*sys.version_info[:2]))
+")
+execute_process(
+    COMMAND ${PYTHON_EXECUTABLE} -B -c "${PYTHON_LIB_BUILDDIR_CMD}"
+    OUTPUT_VARIABLE PYTHON_LIB_BUILDDIR
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+configure_file(pydistutils.cfg.in .pydistutils.cfg @ONLY)
+
+configure_file(
+    ${CMAKE_SOURCE_DIR}/src/libyang.h.in
+    ${CMAKE_BINARY_DIR}/python/include/libyang/libyang.h @ONLY)
+foreach(header ${headers})
+    configure_file(
+        ${CMAKE_SOURCE_DIR}/${header}
+        ${CMAKE_BINARY_DIR}/python/include/libyang/ COPYONLY)
+endforeach()
+
+# Cmake does not support exporting specific environment variables for
+# custom_target commands. The only way is to use a shell script.
+configure_file(run_python.sh.in run_python.sh @ONLY)
+add_custom_target(
+    libyang_python ALL
+    COMMAND ${CMAKE_BINARY_DIR}/python/run_python.sh ./setup.py build
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python
+    DEPENDS yang)
+add_custom_target(
+    libyang_python_sdist
+    COMMAND ${CMAKE_BINARY_DIR}/python/run_python.sh ./setup.py sdist
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python)
+
+if(ENABLE_BUILD_TESTS)
+    set(PYTHON_TESTS
+        tests/test_context.py
+        tests/test_data.py
+        tests/test_diff.py
+        tests/test_schema.py)
+    foreach(test_file ${PYTHON_TESTS})
+        string(
+            REGEX REPLACE "tests/test_([a-z]+)\\.py" "test_python_\\1"
+            test_name ${test_file})
+        add_test(
+            NAME ${test_name}
+            COMMAND ${PYTHON_EXECUTABLE} -B -m unittest -cv ${test_file}
+            WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python)
+        set_property(
+            TEST ${test_name}
+            PROPERTY ENVIRONMENT
+                PYTHONPATH=${CMAKE_BINARY_DIR}/python/${PYTHON_LIB_BUILDDIR}
+                LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}
+                LIBYANG_EXTENSIONS_PLUGINS_DIR=${CMAKE_BINARY_DIR}/src/extensions
+                LIBYANG_USER_TYPES_PLUGINS_DIR=${CMAKE_BINARY_DIR}/src/user_types)
+    endforeach()
+endif()
+
+install(CODE "
+message(STATUS \"${CMAKE_BINARY_DIR}/python/run_python.sh ./setup.py install --root=\$ENV{DESTDIR}/\")
+execute_process(
+    COMMAND ${CMAKE_BINARY_DIR}/python/run_python.sh ./setup.py install --root=\$ENV{DESTDIR}/
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python)
+")

python/LICENSE

@@ -0,0 +1 @@
+../LICENSE

python/MANIFEST.in

@@ -0,0 +1,20 @@
+graft cffi
+graft clib
+prune clib/.git
+prune clib/doc
+prune clib/python
+prune clib/swig
+prune clib/tests
+prune clib/tools/lint/examples
+graft libyang
+prune libyang/_lib
+prune libyang/_include
+include build-so.sh
+include setup.py
+include setup.cfg
+include LICENSE
+include README.md
+global-exclude *.py[co]
+global-exclude *.so
+global-exclude *.o
+global-exclude *.a

python/README.md

@@ -0,0 +1,146 @@
+# libyang
+
+Python CFFI bindings to [libyang][github-link].
+
+[![pypi project][pypi-img]][pypi-link] [![supported python versions][pyversions-img]][pypi-link] [![build status][travis-img]][travis-link] [![license][license-img]][github-link]
+
+[pypi-img]: https://img.shields.io/pypi/v/libyang.svg
+[pypi-link]: https://pypi.org/project/libyang
+[pyversions-img]: https://img.shields.io/pypi/pyversions/libyang.svg
+[license-img]: https://img.shields.io/github/license/CESNET/libyang.svg
+[travis-img]: https://api.travis-ci.com/CESNET/libyang.svg
+[travis-link]: https://travis-ci.com/CESNET/libyang
+[github-link]: https://github.com/CESNET/libyang
+
+## Installation
+
+```bash
+pip install libyang
+```
+
+This assumes `libyang.so` is installed in the system and that `libyang.h` is
+available in the system include dirs.
+
+You need the following system dependencies installed:
+
+- Python development headers
+- GCC
+- Python CFFI module
+
+On a Debian/Ubuntu system::
+
+```
+sudo apt-get install python3-dev gcc python3-cffi
+```
+
+### Compilation Flags
+
+If libyang headers and libraries are installed in a non-standard location, you
+can specify them with the `LIBYANG_HEADERS` and `LIBYANG_LIBRARIES` variables.
+Additionally, for finer control, you may use `LIBYANG_EXTRA_CFLAGS` and
+`LIBYANG_EXTRA_LDFLAGS`:
+
+```bash
+LIBYANG_HEADERS=/home/build/opt/ly/include \
+LIBYANG_LIBRARIES=/home/build/opt/ly/lib \
+LIBYANG_EXTRA_CFLAGS="-O3" \
+LIBYANG_EXTRA_LDFLAGS="-rpath=/opt/ly/lib" \
+       pip install libyang
+```
+
+### Embedding `libyang.so`
+
+If libyang headers and libraries are not installed on the system, you may
+build `libyang.so` and embed it into the `libyang` package before linking
+the CFFI extension against it (with a custom `RPATH`).
+
+To do so, you must export the `LIBYANG_INSTALL=embed` variable when running
+pip:
+
+```bash
+LIBYANG_INSTALL=embed pip install libyang
+```
+
+This requires additional system dependencies in order to build the libyang
+C code:
+
+- cmake
+- Lib PCRE development headers
+
+On a Debian/Ubuntu system:
+
+```bash
+sudo apt-get install cmake build-essential libpcre3-dev
+```
+
+## Examples
+
+```pycon
+>>> import libyang
+>>> ctx = libyang.Context('/usr/local/share/yang/modules')
+>>> module = ctx.load_module('ietf-system')
+>>> print(module)
+module: ietf-system
+  +--rw system
+  |  +--rw contact?          string
+  |  +--rw hostname?         ietf-inet-types:domain-name
+  |  +--rw location?         string
+  |  +--rw clock
+  |  |  +--rw (timezone)?
+  |  |     +--:(timezone-utc-offset)
+  |  |        +--rw timezone-utc-offset?   int16
+  |  +--rw dns-resolver
+  |     +--rw search*    ietf-inet-types:domain-name
+  |     +--rw server* [name]
+  |     |  +--rw name          string
+  |     |  +--rw (transport)
+  |     |     +--:(udp-and-tcp)
+  |     |        +--rw udp-and-tcp
+  |     |           +--rw address    ietf-inet-types:ip-address
+  |     +--rw options
+  |        +--rw timeout?    uint8 <5>
+  |        +--rw attempts?   uint8 <2>
+  +--ro system-state
+     +--ro platform
+     |  +--ro os-name?      string
+     |  +--ro os-release?   string
+     |  +--ro os-version?   string
+     |  +--ro machine?      string
+     +--ro clock
+        +--ro current-datetime?   ietf-yang-types:date-and-time
+        +--ro boot-datetime?      ietf-yang-types:date-and-time
+
+  rpcs:
+    +---x set-current-datetime
+    |  +---- input
+    |     +---w current-datetime    ietf-yang-types:date-and-time
+    +---x system-restart
+    +---x system-shutdown
+
+>>> xpath = '/ietf-system:system/ietf-system:dns-resolver/ietf-system:server'
+>>> dnsserver = next(ctx.find_path(xpath))
+>>> dnsserver
+<libyang.schema.SList: server [name]>
+>>> print(dnsserver.description())
+List of the DNS servers that the resolver should query.
+
+When the resolver is invoked by a calling application, it
+sends the query to the first name server in this list.  If
+no response has been received within 'timeout' seconds,
+the resolver continues with the next server in the list.
+If no response is received from any server, the resolver
+continues with the first server again.  When the resolver
+has traversed the list 'attempts' times without receiving
+any response, it gives up and returns an error to the
+calling application.
+
+Implementations MAY limit the number of entries in this
+list.
+>>> dnsserver.ordered()
+True
+>>> for node in dnsserver:
+...     print(repr(node))
+...
+<libyang.schema.SLeaf: name string>
+<libyang.schema.SContainer: udp-and-tcp>
+```

python/build-so.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -e
+
+src_dir="$1"
+build_dir="$2"
+install_dir="$3"
+
+mkdir -p "$build_dir"
+cd "$build_dir"
+
+cmake -DCMAKE_BUILD_TYPE=release \
+    -DENABLE_BUILD_TESTS=OFF \
+    -DENABLE_VALGRIND_TESTS=OFF \
+    -DENABLE_CALGRIND_TESTS=OFF \
+    -DENABLE_BUILD_FUZZ_TARGETS=OFF \
+    -DCMAKE_INSTALL_PREFIX=$install_dir \
+    -DCMAKE_INSTALL_LIBDIR=$install_dir/_lib \
+    -DCMAKE_INSTALL_INCLUDEDIR=$install_dir/_include \
+    -DPLUGINS_DIR=$install_dir/_lib \
+    -DGEN_LANGUAGE_BINDINGS=0 \
+    "$src_dir"
+
+make -s -j$(nproc)
+make -s install
+rm -rf -- $install_dir/bin $install_dir/share $install_dir/_lib/pkgconfig