ODBC: improve Windows build process (#661)

* improve Windows build process - combine build code into one script for easier edits - introduced UNITY build for AWS SDK * update CI scripts * workflow updates * more workflow updates * workflow updates * workflow updates * update WinArch for VLD dll * print build_installer args * use Windows path separator & print CI output args for debugging * use Powershell envar syntax * workflow & ci script updates * update cmake path for 32-bit install * fix uplaod step path names * adding Env prefix for artifact steps * hardcode upload step paths * add link to custom test dataset * copy build script upgrades to release workflow
opensearch-project · Aug 18, 2020 · 0f7c261 · 0f7c261
1 parent d7b1c04
commit 0f7c261
Show file tree

Hide file tree

Showing 22 changed files with 550 additions and 612 deletions.
diff --git a/.github/workflows/sql-odbc-main.yml b/.github/workflows/sql-odbc-main.yml
@@ -2,6 +2,13 @@ name: Open Distro for Elasticsearch ODBC Driver
 
 on: [push, pull_request]
 
+env:
+  CI_OUTPUT_PATH: "sql-odbc/ci-output"
+  ODBC_LIB_PATH: "./build/odbc/lib"
+  ODBC_BIN_PATH: "./build/odbc/bin"
+  ODBC_BUILD_PATH: "./build/odbc/build"
+  AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install"
+
 jobs:    
   build-mac:
     runs-on: macos-latest
@@ -43,21 +50,21 @@ jobs:
     - name: create-output
       if: success()
       run: |
-        mkdir build
+        mkdir build-output
         mkdir test-output
         mkdir installer
-        cp ./lib64/*.dylib build
-        cp ./lib64/*.a build
-        cp $(ls -d bin64/* | grep -v "\.") build
-        cp ./cmake-build64/*.pkg installer
+        cp ./build/odbc/lib/*.dylib build-output/
+        cp ./build/odbc/lib/*.a build-output/
+        cp ./cmake-build64/*.pkg installer/
+    # cp $(ls -d ./build/odbc/bin/* | grep -v "\.") build-output
     #    cp ./bin64/*.html test-output
     #    cp ./bin64/*.log test-output
     - name: upload-build
       if: success()
       uses: actions/upload-artifact@v1
       with:
         name: mac64-build
-        path: sql-odbc/build
+        path: sql-odbc/build-output
     - name: upload-installer
       if: success()
       uses: actions/upload-artifact@v1
@@ -85,46 +92,34 @@ jobs:
     - name: build-installer
       if: success()
       run: |
-        $prefix_path = (pwd).path
-        cd cmake-build32
-        cmake ..\\src -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\bin -D BUILD_WITH_TESTS=OFF
-        msbuild .\PACKAGE.vcxproj -p:Configuration=Release
-        cd ..
+        .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH
     #- name: test
     #  run: |
     #    cp .\\libraries\\VisualLeakDetector\\bin32\\*.* .\\bin32\\Release
     #    cp .\\libraries\\VisualLeakDetector\\lib32\\*.lib .\\lib32\\Release
     #    .\run_test_runner.bat
-    - name: create-output
+    - name: prepare-output
       if: always()
       run: |
-        mkdir build
-        mkdir test-output
-        mkdir installer
-        cp .\\bin32\\Release\\*.dll build
-        cp .\\bin32\\Release\\*.exe build
-        cp .\\lib32\\Release\\*.lib build
-        cp .\\cmake-build32\\*.msi installer
-    #    cp .\\bin32\\Release\\*.log test-output
-    #    cp .\\bin32\\Release\\*.html test-output
+        .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_BUILD_PATH
     - name: upload-build
       if: always()
       uses: actions/upload-artifact@v1
       with:
         name: windows32-build
-        path: sql-odbc/build
+        path: sql-odbc/ci-output/build
     - name: upload-installer
       if: always()
       uses: actions/upload-artifact@v1
       with:
         name: windows32-installer
-        path: sql-odbc/installer
+        path: sql-odbc/ci-output/installer
     #- name: upload-test-results
     #  if: always()
     #  uses: actions/upload-artifact@v1
     #  with:
     #    name: windows-test-results
-    #    path: test-output
+    #    path: $CI_OUTPUT_PATH/test
   build-windows64:
     runs-on: windows-latest
     defaults:
@@ -140,43 +135,31 @@ jobs:
     - name: build-installer
       if: success()
       run: |
-        $prefix_path = (pwd).path
-        cd cmake-build64
-        cmake ..\\src -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\bin -D BUILD_WITH_TESTS=OFF
-        msbuild .\PACKAGE.vcxproj -p:Configuration=Release
-        cd ..
+        .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH
     #- name: test
     #  run: |
     #    cp .\\libraries\\VisualLeakDetector\\bin64\\*.* .\\bin64\\Release
     #    cp .\\libraries\\VisualLeakDetector\\lib64\\*.lib .\\lib64\\Release
     #    .\run_test_runner.bat
-    - name: create-output
+    - name: prepare-output
       if: always()
       run: |
-        mkdir build
-        mkdir test-output
-        mkdir installer
-        cp .\\bin64\\Release\\*.dll build
-        cp .\\bin64\\Release\\*.exe build
-        cp .\\lib64\\Release\\*.lib build
-        cp .\\cmake-build64\\*.msi installer
-    #    cp .\\bin64\\Release\\*.log test-output
-    #    cp .\\bin64\\Release\\*.html test-output
+        .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_WIN_BUILD_PATH
     - name: upload-build
       if: always()
       uses: actions/upload-artifact@v1
       with:
         name: windows64-build
-        path: sql-odbc/build
+        path: sql-odbc/ci-output/build
     - name: upload-installer
       if: always()
       uses: actions/upload-artifact@v1
       with:
         name: windows64-installer
-        path: sql-odbc/installer
+        path: sql-odbc/ci-output/installer
     #- name: upload-test-results
     #  if: always()
     #  uses: actions/upload-artifact@v1
     #  with:
     #    name: windows-test-results
-    #    path: test-output
+    #    path: sql-odbc/ci-output/test-output
diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml
@@ -5,6 +5,13 @@ on:
     tags:
       - 'v*'
 
+env:
+  CI_OUTPUT_PATH: "sql-odbc/ci-output"
+  ODBC_LIB_PATH: "./build/odbc/lib"
+  ODBC_BIN_PATH: "./build/odbc/bin"
+  ODBC_BUILD_PATH: "./build/odbc/build"
+  AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install"
+
 jobs:
   build-mac:
     runs-on: macos-latest
@@ -35,48 +42,38 @@ jobs:
           brew install curl
           brew install cmake
           brew install libiodbc
-      - name: aws-sdk-cpp-setup
-        run: |
-          sh aws_sdk_cpp_setup.sh
-      - name: configure
-        run: |
-          prefix_path=$(pwd)
-          mkdir cmake-build
-          cd cmake-build
-          cmake ../src -DCMAKE_INSTALL_PREFIX=$prefix_path/AWSSDK/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="core" -DCUSTOM_MEMORY_MANAGEMENT="OFF" -DENABLE_RTTI="OFF" -DENABLE_TESTING="OFF"
-          cd ..
-      - name: build-driver
+      - name: configure-and-build-driver
         run: |
-          cmake --build cmake-build
+          ./build_mac_release64.sh
       - name: build-installer
         if: success()
         run: |
-          cd cmake-build
+          cd cmake-build64
           cmake ../src
           make
           cpack .
           cd ..
       - name: create-output
         if: success()
         run: |
-          mkdir build
+          mkdir build-output
           mkdir test-output
           mkdir installer
-          cp ./lib64/*.dylib build
-          cp ./lib64/*.a build
-          cp $(ls -d bin64/* | grep -v "\.") build
-          cp ./cmake-build/*.pkg installer
+          cp ./build/odbc/lib/*.dylib build-output/
+          cp ./build/odbc/lib/*.a build-output/
+          cp ./cmake-build64/*.pkg installer/
+      # cp $(ls -d bin64/* | grep -v "\.") build
       - name: upload-build
         if: success()
         uses: actions/upload-artifact@v1
         with:
-          name: mac-build
-          path: sql-odbc/build
+          name: mac64-build
+          path: sql-odbc/build-output
       - name: upload-installer
         if: success()
         uses: actions/upload-artifact@v1
         with:
-          name: mac-installer
+          name: mac64-installer
           path: sql-odbc/installer
       - name: upload-artifacts-s3
         if: success()
@@ -106,37 +103,28 @@ jobs:
       - name: build-installer
         if: success()
         run: |
-          cd cmake-build32
-          cmake ..\\src -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\ -D BUILD_WITH_TESTS=OFF
-          msbuild .\PACKAGE.vcxproj -p:Configuration=Release
-          cd ..
-      - name: create-output
+          .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH
+      - name: prepare-output
         if: always()
         run: |
-          mkdir build
-          mkdir test-output
-          mkdir installer
-          cp .\\bin32\\Release\\*.dll build
-          cp .\\bin32\\Release\\*.exe build
-          cp .\\lib32\\Release\\*.lib build
-          cp .\\cmake-build32\\*.msi installer
+          .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_BUILD_PATH
       - name: upload-build
         if: always()
         uses: actions/upload-artifact@v1
         with:
           name: windows32-build
-          path: sql-odbc/build
+          path: sql-odbc/ci-output/build
       - name: upload-installer
         if: always()
         uses: actions/upload-artifact@v1
         with:
           name: windows32-installer
-          path: sql-odbc/installer
+          path: sql-odbc/ci-output/installer
       - name: upload-artifacts-s3
         if: success()
         shell: bash
         run: |
-          cd installer
+          cd ci-output/installer
           windows_installer=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1`
           echo $windows_installer
           aws s3 cp "$windows_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/windows/
@@ -161,37 +149,28 @@ jobs:
       - name: build-installer
         if: success()
         run: |
-          cd cmake-build64
-          cmake ..\\src -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\ -D BUILD_WITH_TESTS=OFF
-          msbuild .\PACKAGE.vcxproj -p:Configuration=Release
-          cd ..
-      - name: create-output
+          .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH
+      - name: prepare-output
         if: always()
         run: |
-          mkdir build
-          mkdir test-output
-          mkdir installer
-          cp .\\bin64\\Release\\*.dll build
-          cp .\\bin64\\Release\\*.exe build
-          cp .\\lib64\\Release\\*.lib build
-          cp .\\cmake-build64\\*.msi installer
+          .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_WIN_BUILD_PATH
       - name: upload-build
         if: always()
         uses: actions/upload-artifact@v1
         with:
           name: windows64-build
-          path: sql-odbc/build
+          path: sql-odbc/ci-output/build
       - name: upload-installer
         if: always()
         uses: actions/upload-artifact@v1
         with:
           name: windows64-installer
-          path: sql-odbc/installer
+          path: sql-odbc/ci-output/installer
       - name: upload-artifacts-s3
         if: success()
         shell: bash
         run: |
-          cd installer
+          cd ci-output/installer
           windows_installer=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1`
           echo $windows_installer
           aws s3 cp "$windows_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/windows/
diff --git a/sql-odbc/.gitignore b/sql-odbc/.gitignore
@@ -54,7 +54,10 @@ CMakeCache.txt
 CPackConfig.cmake
 CPackSourceConfig.cmake
 CTestTestfile.cmake
+/build/
+/sdk-build32/
 /sdk-build64/
+/cmake-build32/
 /cmake-build64/
 /src/PowerBIConnector/bin/Debug/
 /src/PowerBIConnector/obj/