Refactor CMakeList and download scripts to use versions from a single…

… source (#767)

.gitignore

@@ -30,17 +30,12 @@ install_manifest.txt
 compile_commands.json
 CTestTestfile.cmake
 _deps
-/tools/boost_1_80_0
-/tools/boost.unzipped
-/tools/boost_1_80_0.tar.gz
-/tools/pybind11-2.3.0
-/tools/pybind11_2_3_0.tar.gz
-/tools/pybind11-2.10.4
-/tools/pybind11_2_10_4.tar.gz
+/tools/boost*
+/tools/pybind11*
 /tools/protobuf*
-/tools/OpenBLAS-*/
-/tools/OpenBLASv*.tar.gz
-/tools/onnx-*/
+/tools/OpenBLAS*
+/tools/onnx*
+
 
 build
 /.idea/

CMakeLists.txt

@@ -80,7 +80,8 @@ include_directories(SYSTEM ${CVC4_DIR} ${CVC4_DIR}/include)
 ## Boost ##
 ###########
 
-set(BOOST_DIR "${TOOLS_DIR}/boost_1_80_0")
+set(BOOST_VERSION 1.80.0)
+set(BOOST_DIR "${TOOLS_DIR}/boost-${BOOST_VERSION}")
 if (MSVC)
   set(BOOST_ROOT "${BOOST_DIR}/win_installed")
   set(Boost_NAMESPACE libboost)
@@ -91,13 +92,11 @@ else()
 endif()
 
 set(Boost_USE_DEBUG_RUNTIME FALSE)
-find_package(Boost 1.80.0 COMPONENTS program_options timer chrono thread)
+find_package(Boost ${BOOST_VERSION} COMPONENTS program_options timer chrono thread)
 # Find boost
 if (NOT ${Boost_FOUND})
-  # bash file that downloads and install boost 1_80_0, the name need to match
-  # BOOST_DIR variable
-  execute_process(COMMAND ${TOOLS_DIR}/download_boost.${SCRIPT_EXTENSION})
-  find_package(Boost 1.80.0 REQUIRED COMPONENTS program_options timer chrono thread)
+  execute_process(COMMAND ${TOOLS_DIR}/download_boost.${SCRIPT_EXTENSION} ${BOOST_VERSION})
+  find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS program_options timer chrono thread)
 endif()
 set(LIBS_INCLUDES ${Boost_INCLUDE_DIRS})
 list(APPEND LIBS ${Boost_LIBRARIES})
@@ -107,7 +106,8 @@ list(APPEND LIBS ${Boost_LIBRARIES})
 ##############
 # Protobuf is needed to compile ONNX
 
-set(PROTOBUF_DEFAULT_DIR "${TOOLS_DIR}/protobuf-3.19.2")
+set(PROTOBUF_VERSION 3.19.2)
+set(PROTOBUF_DEFAULT_DIR "${TOOLS_DIR}/protobuf-${PROTOBUF_VERSION}")
 if (NOT PROTOBUF_DIR)
     set(PROTOBUF_DIR ${PROTOBUF_DEFAULT_DIR})
 endif()
@@ -116,7 +116,7 @@ if(NOT EXISTS "${PROTOBUF_DIR}/installed/lib/libprotobuf.a")
     message("Can't find protobuf, installing. If protobuf is installed please use the PROTOBUF_DIR parameter to pass the path")
     if (${PROTOBUF_DIR} STREQUAL ${PROTOBUF_DEFAULT_DIR})
         message("installing protobuf")
-        execute_process(COMMAND ${TOOLS_DIR}/download_protobuf.sh)
+        execute_process(COMMAND ${TOOLS_DIR}/download_protobuf.sh ${PROTOBUF_VERSION})
     else()
         message(FATAL_ERROR "Can't find protobuf in the supplied directory")
     endif()
@@ -133,11 +133,13 @@ list(APPEND LIBS ${PROTOBUF_LIB})
 ## ONNX ##
 ##########
 
-set(ONNX_DIR "${TOOLS_DIR}/onnx-1.15.0")
+# NOTE: The ONNX version must be kept in sync with Python `pyproject.toml` and `test_requirements.txt`.
+set(ONNX_VERSION 1.15.0)
+set(ONNX_DIR "${TOOLS_DIR}/onnx-${ONNX_VERSION}")
 
 if(NOT EXISTS "${ONNX_DIR}/onnx.proto3.pb.h")
     message("generating ONNX protobuf file")
-    execute_process(COMMAND ${TOOLS_DIR}/download_onnx.sh)
+    execute_process(COMMAND ${TOOLS_DIR}/download_onnx.sh ${ONNX_VERSION} ${PROTOBUF_VERSION})
 endif()
 file(GLOB DEPS_ONNX "${ONNX_DIR}/*.cc")
 include_directories(SYSTEM ${ONNX_DIR})
@@ -178,31 +180,31 @@ endif()
 ## OpenBLAS ##
 ##############
 
-set(OPENBLAS_LIB openblas)
-set(OPENBLAS_DEFAULT_DIR "${TOOLS_DIR}/OpenBLAS-0.3.19")
-if (NOT OPENBLAS_DIR)
-    set(OPENBLAS_DIR ${OPENBLAS_DEFAULT_DIR})
-endif()
+if (NOT MSVC AND ${ENABLE_OPENBLAS})
+  set(OPENBLAS_VERSION 0.3.19)
 
-if (NOT MSVC)
-  if (${ENABLE_OPENBLAS})
-    message(STATUS "Using OpenBLAS for matrix multiplication")
-    add_compile_definitions(ENABLE_OPENBLAS)
-    if(NOT EXISTS "${OPENBLAS_DIR}/installed/lib/libopenblas.a")
-        message("Can't find OpenBLAS, installing. If OpenBLAS is installed please use the OPENBLAS_DIR parameter to pass the path")
-        if (${OPENBLAS_DIR} STREQUAL ${OPENBLAS_DEFAULT_DIR})
-            message("Installing OpenBLAS")
-            execute_process(COMMAND ${TOOLS_DIR}/download_openBLAS.sh)
-        else()
-            message(FATAL_ERROR "Can't find OpenBLAS in the supplied directory")
-        endif()
-    endif()
+  set(OPENBLAS_LIB openblas)
+  set(OPENBLAS_DEFAULT_DIR "${TOOLS_DIR}/OpenBLAS-${OPENBLAS_VERSION}")
+  if (NOT OPENBLAS_DIR)
+      set(OPENBLAS_DIR ${OPENBLAS_DEFAULT_DIR})
+  endif()
 
-    add_library(${OPENBLAS_LIB} SHARED IMPORTED)
-    set_target_properties(${OPENBLAS_LIB} PROPERTIES IMPORTED_LOCATION ${OPENBLAS_DIR}/installed/lib/libopenblas.a)
-    list(APPEND LIBS ${OPENBLAS_LIB})
-    target_include_directories(${OPENBLAS_LIB} INTERFACE ${OPENBLAS_DIR}/installed/include)
+  message(STATUS "Using OpenBLAS for matrix multiplication")
+  add_compile_definitions(ENABLE_OPENBLAS)
+  if(NOT EXISTS "${OPENBLAS_DIR}/installed/lib/libopenblas.a")
+    message("Can't find OpenBLAS, installing. If OpenBLAS is installed please use the OPENBLAS_DIR parameter to pass the path")
+      if (${OPENBLAS_DIR} STREQUAL ${OPENBLAS_DEFAULT_DIR})
+        message("Installing OpenBLAS")
+        execute_process(COMMAND ${TOOLS_DIR}/download_openBLAS.sh ${OPENBLAS_VERSION})
+      else()
+        message(FATAL_ERROR "Can't find OpenBLAS in the supplied directory")
+      endif()
   endif()
+
+  add_library(${OPENBLAS_LIB} SHARED IMPORTED)
+  set_target_properties(${OPENBLAS_LIB} PROPERTIES IMPORTED_LOCATION ${OPENBLAS_DIR}/installed/lib/libopenblas.a)
+  list(APPEND LIBS ${OPENBLAS_LIB})
+  target_include_directories(${OPENBLAS_LIB} INTERFACE ${OPENBLAS_DIR}/installed/include)
 endif()
 
 ###########
@@ -364,11 +366,11 @@ else()
     endif()
 endif()
 
-set(MARABOU_PY MarabouCore)
-set(PYBIND11_DIR "${TOOLS_DIR}/pybind11-2.10.4")
-
 # Actually build Python
 if (${BUILD_PYTHON})
+   set(PYBIND11_VERSION 2.10.4)
+   set(PYBIND11_DIR "${TOOLS_DIR}/pybind11-${PYBIND11_VERSION}")
+
     # This is suppose to set the PYTHON_EXECUTABLE variable
     # First try to find the default python version
     find_package(PythonInterp ${DEFAULT_PYTHON_VERSION})
@@ -379,9 +381,11 @@ if (${BUILD_PYTHON})
 
     if (NOT EXISTS ${PYBIND11_DIR})
         message("didnt find pybind, getting it")
-	execute_process(COMMAND ${TOOLS_DIR}/download_pybind11.${SCRIPT_EXTENSION})
+	execute_process(COMMAND ${TOOLS_DIR}/download_pybind11.${SCRIPT_EXTENSION} ${PYBIND11_VERSION})
     endif()
     add_subdirectory(${PYBIND11_DIR})
+
+    set(MARABOU_PY MarabouCore)
     pybind11_add_module(${MARABOU_PY} ${PYTHON_API_DIR}/MarabouCore.cpp)
 
     target_link_libraries(${MARABOU_PY} PRIVATE ${MARABOU_LIB})

pyproject.toml

@@ -15,7 +15,7 @@ requires-python = ">=3.8,<3.12"
 test = [
   # NOTE: These dependencies must be kept in-sync with the legacy requirements file 'maraboupy/test_requirements.txt'
   "numpy>=1.21.0,<2",
-  # NOTE: The ONNX version must be kept in sync with 'tools/download_onnx.sh'
+  # NOTE: The ONNX version must be kept in sync with 'ONNX_VERSION' in CMakeLists
   "onnx>=1.15.0,<2",
   "onnxruntime>=1.12.0,<2",
   "pytest>=7.2.1,<8",

tools/download_boost.sh

@@ -1,22 +1,25 @@
 #!/bin/bash
 curdir=$pwd
 mydir="${0%/*}"
+version=$1
 
 cd $mydir
 
 # TODO: add progress bar, -q is quite, if removing it the progress bar is in
 # multiple lines
-echo "downloading boost"
-wget -q https://sourceforge.net/projects/boost/files/boost/1.80.0/boost_1_80_0.tar.gz/download -O boost_1_80_0.tar.gz
-echo "unzipping boost"
-tar xzvf boost_1_80_0.tar.gz >> /dev/null
-echo "installing boost"
-cd boost_1_80_0;
+echo "Downloading boost"
+underscore_version=${version//./_}
+wget -q https://sourceforge.net/projects/boost/files/boost/$version/boost_$underscore_version.tar.gz/download -O boost-$version.tar.gz
 
+echo "Unzipping boost"
+tar xzvf boost-$version.tar.gz >> /dev/null
+mv boost_$underscore_version boost-$version
+
+echo "Installing boost"
+cd boost-$version;
 if [[ $OSTYPE == 'darwin'* ]]; then
     export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
 fi
-
 mkdir installed
 ./bootstrap.sh --prefix=`pwd`/installed --with-libraries=program_options,timer,chrono,thread >> /dev/null ;
 ./b2 cxxflags=-fPIC link=static cxxflags=-std=c++11 install >> /dev/null

tools/download_onnx.sh

@@ -1,22 +1,25 @@
 #!/bin/bash
 curdir=$pwd
 mydir="${0%/*}"
-onnxdir=onnx-1.15.0
-protobufdir=protobuf-3.19.2
+version=$1
+protobuf_version=$2
 
-# This script downloads the protobuffer file from the Onnx repo.
-# There are no C++ bindings for onnx so we compile the protobuffer ourselves.
+onnxdir=onnx-$version
+protobufdir=protobuf-$protobuf_version
+
+# This script downloads the protobuffer file from the ONNX repo.
+# There are no C++ bindings for ONNX so we compile the protobuffer ourselves.
 # see https://stackoverflow.com/questions/67301475/parse-an-onnx-model-using-c-extract-layers-input-and-output-shape-from-an-on
 # for details.
 
 cd $mydir
 
 mkdir -p $onnxdir
 cd $onnxdir
-echo "downloading onnx proto file"
-wget -q https://raw.githubusercontent.com/onnx/onnx/v1.15.0/onnx/onnx.proto3 -O onnx.proto3
+echo "Downloading ONNX proto file"
+wget -q https://raw.githubusercontent.com/onnx/onnx/v$version/onnx/onnx.proto3 -O onnx.proto3
 
-echo "compiling the onnx proto file"
+echo "Compiling the ONNX proto file"
 ../$protobufdir/installed/bin/protoc --cpp_out=. onnx.proto3
 
 cd $curdir

tools/download_openBLAS.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
 curdir=$pwd
 mydir="${0%/*}"
+version=$1
 
 cd $mydir
 
 echo "Downloading openBLAS"
-wget -q https://github.com/xianyi/OpenBLAS/releases/download/v0.3.19/OpenBLAS-0.3.19.tar.gz -O OpenBLASv0.3.19.tar.gz
+wget -q https://github.com/xianyi/OpenBLAS/releases/download/v$version/OpenBLAS-$version.tar.gz -O OpenBLASv$version.tar.gz
 echo "Unzipping openBLAS"
-tar -xzf OpenBLASv0.3.19.tar.gz >> /dev/null
+tar -xzf OpenBLASv$version.tar.gz >> /dev/null
 echo "Installing openBLAS"
-cd OpenBLAS-0.3.19
+cd OpenBLAS-$version
 
 if [[ $OSTYPE == 'darwin'* ]]; then
     export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)

tools/download_protobuf.sh

@@ -7,20 +7,23 @@ get_abs_filename() {
 
 curdir=$pwd
 mydir="${0%/*}"
-installdir=$(get_abs_filename $mydir)/protobuf-3.19.2/installed
+version=$1
+installdir=$(get_abs_filename $mydir)/protobuf-$version/installed
 
 # Compatibility with MacOS
 if [[ $OSTYPE == 'darwin'* ]]; then
     export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
 fi
 
 cd $mydir
-echo "downloading protobuf-3.19.2"
-wget -q https://github.com/protocolbuffers/protobuf/releases/download/v3.19.2/protobuf-cpp-3.19.2.tar.gz -O protobuf-cpp-3.19.2.tar.gz
-echo "unzipping protobuf-3.19.2"
-tar -xzf protobuf-cpp-3.19.2.tar.gz # >> /dev/null
-echo "installing protobuf-3.19.2"
-cd protobuf-3.19.2
+echo "Downloading protobuf-$version"
+wget -q https://github.com/protocolbuffers/protobuf/releases/download/v$version/protobuf-cpp-$version.tar.gz -O protobuf-cpp-$version.tar.gz
+
+echo "Unzipping protobuf-$version"
+tar -xzf protobuf-cpp-$version.tar.gz # >> /dev/null
+
+echo "Installing protobuf-$version"
+cd protobuf-$version
 mkdir -p $installdir
 ./configure CXXFLAGS=-fPIC --disable-shared --prefix=$installdir --enable-fast-install
 make

tools/download_pybind11.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
 curdir=$pwd
 mydir="${0%/*}"
+version=$1
 
 cd $mydir
 
 # TODO: add progress bar, -q is quite, if removing it the progress bar is in
 # multiple lines
 echo "downloading pybind"
-wget -q -O pybind11_2_10_4.tar.gz https://github.com/pybind/pybind11/archive/v2.10.4.tar.gz
+wget -q https://github.com/pybind/pybind11/archive/v$version.tar.gz -O pybind11-$version.tar.gz
 
 echo "unzipping pybind"
-tar xzvf pybind11_2_10_4.tar.gz >> /dev/null
+tar xzvf pybind11-$version.tar.gz >> /dev/null
 
 cd $curdir