From 754175d5d3ed143c86102a8c015efd86ed04b241 Mon Sep 17 00:00:00 2001 From: Abdoulbari Zaher <32519851+a-zakir@users.noreply.github.com> Date: Wed, 11 Sep 2024 10:50:15 +0200 Subject: [PATCH] xpress from pip (#924) --- .github/workflows/build_centos7.yml | 39 ++++++----------- .github/workflows/build_oracle8.yml | 45 +++++++++----------- .github/workflows/build_ubuntu.yml | 32 +++++++------- .github/workflows/build_windows.yml | 31 ++++++-------- data_test/external_loop_test/lp/options.json | 3 +- docker/centos7-system-deps | 12 ++++++ tests/end_to_end/cucumber/steps/steps.py | 6 +-- 7 files changed, 78 insertions(+), 90 deletions(-) diff --git a/.github/workflows/build_centos7.yml b/.github/workflows/build_centos7.yml index 66851a7ea..3c50e2bf0 100644 --- a/.github/workflows/build_centos7.yml +++ b/.github/workflows/build_centos7.yml @@ -63,24 +63,12 @@ jobs: path: 'antares-version.json' key: 'antares_xpansion_version' + build: runs-on: ubuntu-latest needs: [ docker_publish, versions ] container: 'antaresrte/xpansion-centos7:1.1.0' - strategy: - matrix: - xprs: [ - # { value: XPRESS-ON, ref: 8.13a }, - { value: XPRESS-ON, ref: 9.2.5 }, - #{ value: XPRESS-OFF } - ] env: - XPRESSDIR: ${{ github.workspace }}/xpress - XPRESS: ${{ github.workspace }}/xpress/bin - XPRS_LIB_Path: ${{ github.workspace }}/xpress/lib - XPRESSDIR_CONTAINER: ${GITHUB_WORKSPACE}/xpress - XPRESS_CONTAINER: ${GITHUB_WORKSPACE}/xpress/bin - XPRS_LIB_Path_CONTAINER: ${GITHUB_WORKSPACE}/xpress/lib VCPKG_ROOT: ${{ github.workspace }}/vcpkg outputs: @@ -108,16 +96,6 @@ jobs: with: cmake-version: '3.28.x' - - name: Checkout xpressmp linux - uses: actions/checkout@v3 #keep v3 - with: - token: ${{ secrets.AS_TOKEN }} - repository: rte-france/xpress-mp - path: ${{ env.XPRESSDIR }} - github-server-url: https://github.com - ref: ${{matrix.xprs.ref}} - if: matrix.xprs.value == 'XPRESS-ON' - - name: Download pre-compiled librairies uses: ./.github/workflows/download-extract-precompiled-libraries-tgz with: @@ -139,6 +117,19 @@ jobs: pip3 install -r requirements-tests.txt pip3 install -r requirements-ui.txt + + - name: Set-up Xpress with pip for centos 7 + shell: bash + run: | + export PATH=/root/miniconda3/condabin:$PATH + conda install -c fico-xpress "xpress>=9.2,<9.3" + XPRESS_DIR=/root/miniconda3/lib/python3.8/site-packages/xpress + echo "XPRESSDIR=$XPRESS_DIR" >> $GITHUB_ENV + echo "XPAUTH_PATH=$XPRESS_DIR/license/community-xpauth.xpr" >> $GITHUB_ENV + echo "Create symbolic link for XPRESS library file because it is missing in the Python installation" + ln -s $XPRESS_DIR/lib/libxprs.so.42 $XPRESS_DIR/lib/libxprs.so + + - name: vcpkg install run: | git config --global safe.directory '*' @@ -196,8 +187,6 @@ jobs: shell: bash run: | export PATH=${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin/:$PATH - export LD_LIBRARY_PATH=LD_LIBRARY_PATH:${{ env.XPRS_LIB_Path_CONTAINER }} - export XPRESS=${{ env.XPRESS_CONTAINER }} cd _build ctest -C Release --output-on-failure -L "unit|benders|lpnamer|medium" diff --git a/.github/workflows/build_oracle8.yml b/.github/workflows/build_oracle8.yml index c5b700228..a568226c2 100644 --- a/.github/workflows/build_oracle8.yml +++ b/.github/workflows/build_oracle8.yml @@ -40,21 +40,7 @@ jobs: name: Build runs-on: ubuntu-latest container: 'oraclelinux:8' - strategy: - matrix: - xprs: [ - # { value: XPRESS-ON, ref: 8.13a }, - { value: XPRESS-ON, ref: 9.2.5 }, - # { value: XPRESS-OFF } - ] needs: [ versions ] - env: - XPRESSDIR: ${{ github.workspace }}/xpress - XPRESS: ${{ github.workspace }}/xpress/bin - XPRS_LIB_Path: ${{ github.workspace }}/xpress/lib - XPRESSDIR_CONTAINER: ${GITHUB_WORKSPACE}/xpress - XPRESS_CONTAINER: ${GITHUB_WORKSPACE}/xpress/bin - XPRS_LIB_Path_CONTAINER: ${GITHUB_WORKSPACE}/xpress/lib outputs: zip_name: ${{ steps.zip_name.outputs.zip_name }} @@ -81,15 +67,7 @@ jobs: with: submodules: true - - name: Checkout xpressmp linux - uses: actions/checkout@v4 - with: - token: ${{ secrets.AS_TOKEN }} - repository: rte-france/xpress-mp - path: ${{ env.XPRESSDIR }} - github-server-url: https://github.com - ref: ${{matrix.xprs.ref}} - if: matrix.xprs.value == 'XPRESS-ON' + - name: Set up Python run: | @@ -98,6 +76,25 @@ jobs: pip3 install wheel #Too late to install in requirements.txt pip3 install -r requirements-tests.txt + # the default python version (3.6) is too old to download xpress with pip + # this version of miniconda embeds python3.8 + - name: Set up conda + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.1-0-Linux-x86_64.sh + bash Miniconda3-py38_23.5.1-0-Linux-x86_64.sh -b -p /root/miniconda3 + rm -f Miniconda3-py38_23.5.1-0-Linux-x86_64.sh + + - name: Set-up Xpress with pip for Oracle 8 + shell: bash + run: | + export PATH=/root/miniconda3/condabin:$PATH + conda install -c fico-xpress "xpress>=9.2,<9.3" + XPRESS_DIR=/root/miniconda3/lib/python3.8/site-packages/xpress + echo "XPRESSDIR=$XPRESS_DIR" >> $GITHUB_ENV + echo "XPAUTH_PATH=$XPRESS_DIR/license/community-xpauth.xpr" >> $GITHUB_ENV + echo "Create symbolic link for XPRESS library file because it is missing in the Python installation" + ln -s $XPRESS_DIR/lib/libxprs.so.42 $XPRESS_DIR/lib/libxprs.so + - name: Download pre-compiled librairies uses: ./.github/workflows/download-extract-precompiled-libraries-tgz with: @@ -153,8 +150,6 @@ jobs: shell: bash run: | export PATH=${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin/:$PATH - export LD_LIBRARY_PATH=LD_LIBRARY_PATH:${{ env.XPRS_LIB_Path_CONTAINER }} - export XPRESS=${{ env.XPRESS_CONTAINER }} cd _build ctest -C Release --output-on-failure -L "unit|benders|lpnamer|medium" diff --git a/.github/workflows/build_ubuntu.yml b/.github/workflows/build_ubuntu.yml index 5e2039d01..8a820e665 100644 --- a/.github/workflows/build_ubuntu.yml +++ b/.github/workflows/build_ubuntu.yml @@ -14,6 +14,7 @@ on: env: GITHUB_TOKEN: ${{ github.token }} + PYTHON_VERSION: 3.8 jobs: build: @@ -22,15 +23,7 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - xprs: [ - #{ value: XPRESS-ON, ref: 8.13a }, - { value: XPRESS-ON, ref: 9.2.5 }, - # { value: XPRESS-OFF } - ] env: - XPRESSDIR: ${{ github.workspace }}/xpress - XPRESS: ${{ github.workspace }}/xpress/bin - XPRS_LIB_Path: ${{ github.workspace }}/xpress/lib VCPKG_BINARY_SOURCES: "clear;files,${{ github.workspace }}/vcpkg_cache,readwrite" outputs: @@ -42,14 +35,7 @@ jobs: with: submodules: true - - name: Checkout xpressmp linux - if: matrix.xprs.value == 'XPRESS-ON' - uses: actions/checkout@v4 - with: - repository: rte-france/xpress-mp - path: ${{ env.XPRESSDIR }} - ref: ${{ matrix.xprs.ref}} - token: ${{ secrets.AS_TOKEN }} #reniew token periodically + - name: Install mandatory system libraries run: | @@ -65,9 +51,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 + id: python-setup with: cache: 'pip' - python-version: 3.8 + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies run: | @@ -75,6 +62,17 @@ jobs: pip install -r requirements-tests.txt pip install -r requirements-ui.txt + - name: Set-up Xpress with pip for Ubuntu + shell: bash + run: | + python -m pip install "xpress>=9.2,<9.3" + echo ${{ env.pythonLocation }} + XPRESS_DIR=${{ env.pythonLocation }}/lib/python${{ env.PYTHON_VERSION }}/site-packages/xpress + echo "XPRESSDIR=$XPRESS_DIR" >> $GITHUB_ENV + echo "XPAUTH_PATH=$XPRESS_DIR/license/community-xpauth.xpr" >> $GITHUB_ENV + echo "Create symbolic link for XPRESS library file because it is missing in the Python installation" + ln -s $XPRESS_DIR/lib/libxprs.so.42 $XPRESS_DIR/lib/libxprs.so + - name: Update alternatives #mpicxx uses "g++" so we need g++ to be symbolic link to g++-10 run: | diff --git a/.github/workflows/build_windows.yml b/.github/workflows/build_windows.yml index 2e10d0375..e9230bfab 100644 --- a/.github/workflows/build_windows.yml +++ b/.github/workflows/build_windows.yml @@ -23,14 +23,7 @@ jobs: matrix: os: [ windows-latest ] triplet: [ x64-windows-release ] - xprs: [ #{ value: XPRESS-ON, ref: 8.13a }, - { value: XPRESS-ON, ref: 9.2.5 }, - #{ value: XPRESS-OFF } - ] env: - XPRESSDIR: ${{ github.workspace }}\xpress - XPRESS: ${{ github.workspace }}\xpress\bin - XPRS_LIB_Path: ${{ github.workspace }}\xpress\lib # Indicates the location of the vcpkg as a Git submodule of the project repository. VCPKG_ROOT: ${{ github.workspace }}/vcpkg VCPKG_BINARY_SOURCES: "clear;files,${{ github.workspace }}/vcpkg_cache,readwrite" @@ -43,19 +36,12 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout xpressmp linux - if: matrix.xprs.value == 'XPRESS-ON' - uses: actions/checkout@v4 - with: - repository: rte-france/xpress-mp-temp - path: ${{ env.XPRESSDIR }} - ref: ${{matrix.xprs.ref}} - token: ${{ secrets.AS_TOKEN }} + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.8' cache: 'pip' cache-dependency-path: requirements*.txt @@ -65,6 +51,16 @@ jobs: pip install -r requirements-tests.txt pip install -r requirements-ui.txt + - name: Set-up Xpress with pip + shell: bash + run: | + python -m pip install --no-cache-dir "xpress>=9.2,<9.3" + XPRESS_DIR="${{ env.pythonLocation }}\Lib\site-packages\xpress" + cp -r $XPRESS_DIR/lib $XPRESS_DIR/bin + cp $XPRESS_DIR/license/community-xpauth.xpr $XPRESS_DIR/bin/xpauth.xpr + echo "XPRESSDIR=$XPRESS_DIR" >> $GITHUB_ENV + echo "$XPRESS_DIR/bin" >> $GITHUB_PATH + - name: Pre-requisites shell: cmd run: | @@ -118,6 +114,7 @@ jobs: -DCMAKE_TOOLCHAIN_FILE="${{env.VCPKG_ROOT}}/scripts/buildsystems/vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} -DCMAKE_INSTALL_PREFIX=_install + -DPython3_EXECUTABLE="${{ env.Python3_ROOT_DIR }}/python.exe" -DBUILD_UI=ON - name: Build @@ -145,8 +142,6 @@ jobs: shell: cmd run: | set PATH=%PATH%;C:\Program Files\Microsoft MPI\Bin - set PATH=%PATH%;${{ env.XPRESS }} - set XPRESSDIR=${{ env.XPRESSDIR }} cd _build ctest -C Release --output-on-failure -L "medium|unit|benders|lpnamer" diff --git a/data_test/external_loop_test/lp/options.json b/data_test/external_loop_test/lp/options.json index 836cced01..0295314bd 100644 --- a/data_test/external_loop_test/lp/options.json +++ b/data_test/external_loop_test/lp/options.json @@ -18,8 +18,7 @@ "JSON_FILE": "../expansion/out.json", "LAST_ITERATION_JSON_FILE": "../expansion/last_iteration.json", "MASTER_FORMULATION": "integer", - "###UNTIL XPRESS LICENSE IS NOT UPDATED ###": "USE COIN", - "SOLVER_NAME": "COIN", + "SOLVER_NAME": "XPRESS", "TIME_LIMIT": 1000000000000.0, "LOG_LEVEL": 0, "LAST_MASTER_MPS": "master_last_iteration", diff --git a/docker/centos7-system-deps b/docker/centos7-system-deps index 29dc1a6dd..068bb455c 100644 --- a/docker/centos7-system-deps +++ b/docker/centos7-system-deps @@ -25,6 +25,18 @@ RUN \ yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm &&\ yum install -y git +# The default python version (3.6) is too old to download xpress with pip +# This version of miniconda embeds python3.8 +RUN \ + wget https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.1-0-Linux-x86_64.sh + +RUN \ + bash Miniconda3-py38_23.5.1-0-Linux-x86_64.sh -b + +RUN \ + rm -f Miniconda3-py38_23.5.1-0-Linux-x86_64.sh + + #create user RUN useradd user # USER user diff --git a/tests/end_to_end/cucumber/steps/steps.py b/tests/end_to_end/cucumber/steps/steps.py index 6a802caf5..72e0e9c14 100644 --- a/tests/end_to_end/cucumber/steps/steps.py +++ b/tests/end_to_end/cucumber/steps/steps.py @@ -43,9 +43,9 @@ def run_outer_loop(context, n): os.chdir(lp_path) process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) out, err = process.communicate() - # print(out) - # print("*****") - # print(err) + print(out) + print("*****") + print(err) context.return_code = process.returncode context.outputs = read_outputs(Path("..") / "expansion" / "out.json") os.chdir(old_cwd)