From 0f7c2612ae25e0c35a6279ee3d511032c16daccb Mon Sep 17 00:00:00 2001 From: Jordan Wilson <37088125+jordanw-bq@users.noreply.github.com> Date: Tue, 18 Aug 2020 12:58:44 -0700 Subject: [PATCH] ODBC: improve Windows build process (#661) * improve Windows build process - combine build code into one script for easier edits - introduced UNITY build for AWS SDK * update CI scripts * workflow updates * more workflow updates * workflow updates * workflow updates * update WinArch for VLD dll * print build_installer args * use Windows path separator & print CI output args for debugging * use Powershell envar syntax * workflow & ci script updates * update cmake path for 32-bit install * fix uplaod step path names * adding Env prefix for artifact steps * hardcode upload step paths * add link to custom test dataset * copy build script upgrades to release workflow --- .github/workflows/sql-odbc-main.yml | 67 ++-- .../workflows/sql-odbc-release-workflow.yml | 81 ++-- sql-odbc/.gitignore | 3 + sql-odbc/BUILD_INSTRUCTIONS.md | 360 ------------------ sql-odbc/README.md | 2 +- sql-odbc/aws_sdk_cpp_setup.ps1 | 30 -- sql-odbc/build_win_debug32.ps1 | 29 +- sql-odbc/build_win_debug64.ps1 | 29 +- sql-odbc/build_win_release32.ps1 | 29 +- sql-odbc/build_win_release64.ps1 | 29 +- sql-odbc/docs/dev/BUILD_INSTRUCTIONS.md | 118 ++++++ .../dev/datasets/kibana_sample_data_types.md | 72 ++++ sql-odbc/docs/dev/run_tests.md | 106 ++++++ sql-odbc/docs/user/mac_configure_dsn.md | 34 +- sql-odbc/run_test_runner.bat | 2 +- sql-odbc/scripts/build_aws-sdk-cpp.ps1 | 34 ++ sql-odbc/scripts/build_driver.ps1 | 15 + sql-odbc/scripts/build_installer.ps1 | 17 + sql-odbc/scripts/build_windows.ps1 | 48 +++ sql-odbc/scripts/prepare_ci_output.ps1 | 20 + sql-odbc/src/CMakeLists.txt | 28 +- sql-odbc/src/installer/CMakeLists.txt | 9 +- 22 files changed, 550 insertions(+), 612 deletions(-) delete mode 100644 sql-odbc/BUILD_INSTRUCTIONS.md delete mode 100644 sql-odbc/aws_sdk_cpp_setup.ps1 create mode 100644 sql-odbc/docs/dev/BUILD_INSTRUCTIONS.md create mode 100644 sql-odbc/docs/dev/datasets/kibana_sample_data_types.md create mode 100644 sql-odbc/docs/dev/run_tests.md create mode 100644 sql-odbc/scripts/build_aws-sdk-cpp.ps1 create mode 100644 sql-odbc/scripts/build_driver.ps1 create mode 100644 sql-odbc/scripts/build_installer.ps1 create mode 100644 sql-odbc/scripts/build_windows.ps1 create mode 100644 sql-odbc/scripts/prepare_ci_output.ps1 diff --git a/.github/workflows/sql-odbc-main.yml b/.github/workflows/sql-odbc-main.yml index 41ca029465..4152aeea2a 100644 --- a/.github/workflows/sql-odbc-main.yml +++ b/.github/workflows/sql-odbc-main.yml @@ -2,6 +2,13 @@ name: Open Distro for Elasticsearch ODBC Driver on: [push, pull_request] +env: + CI_OUTPUT_PATH: "sql-odbc/ci-output" + ODBC_LIB_PATH: "./build/odbc/lib" + ODBC_BIN_PATH: "./build/odbc/bin" + ODBC_BUILD_PATH: "./build/odbc/build" + AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" + jobs: build-mac: runs-on: macos-latest @@ -43,13 +50,13 @@ jobs: - name: create-output if: success() run: | - mkdir build + mkdir build-output mkdir test-output mkdir installer - cp ./lib64/*.dylib build - cp ./lib64/*.a build - cp $(ls -d bin64/* | grep -v "\.") build - cp ./cmake-build64/*.pkg installer + cp ./build/odbc/lib/*.dylib build-output/ + cp ./build/odbc/lib/*.a build-output/ + cp ./cmake-build64/*.pkg installer/ + # cp $(ls -d ./build/odbc/bin/* | grep -v "\.") build-output # cp ./bin64/*.html test-output # cp ./bin64/*.log test-output - name: upload-build @@ -57,7 +64,7 @@ jobs: uses: actions/upload-artifact@v1 with: name: mac64-build - path: sql-odbc/build + path: sql-odbc/build-output - name: upload-installer if: success() uses: actions/upload-artifact@v1 @@ -85,46 +92,34 @@ jobs: - name: build-installer if: success() run: | - $prefix_path = (pwd).path - cd cmake-build32 - cmake ..\\src -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\bin -D BUILD_WITH_TESTS=OFF - msbuild .\PACKAGE.vcxproj -p:Configuration=Release - cd .. + .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH #- name: test # run: | # cp .\\libraries\\VisualLeakDetector\\bin32\\*.* .\\bin32\\Release # cp .\\libraries\\VisualLeakDetector\\lib32\\*.lib .\\lib32\\Release # .\run_test_runner.bat - - name: create-output + - name: prepare-output if: always() run: | - mkdir build - mkdir test-output - mkdir installer - cp .\\bin32\\Release\\*.dll build - cp .\\bin32\\Release\\*.exe build - cp .\\lib32\\Release\\*.lib build - cp .\\cmake-build32\\*.msi installer - # cp .\\bin32\\Release\\*.log test-output - # cp .\\bin32\\Release\\*.html test-output + .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_BUILD_PATH - name: upload-build if: always() uses: actions/upload-artifact@v1 with: name: windows32-build - path: sql-odbc/build + path: sql-odbc/ci-output/build - name: upload-installer if: always() uses: actions/upload-artifact@v1 with: name: windows32-installer - path: sql-odbc/installer + path: sql-odbc/ci-output/installer #- name: upload-test-results # if: always() # uses: actions/upload-artifact@v1 # with: # name: windows-test-results - # path: test-output + # path: $CI_OUTPUT_PATH/test build-windows64: runs-on: windows-latest defaults: @@ -140,43 +135,31 @@ jobs: - name: build-installer if: success() run: | - $prefix_path = (pwd).path - cd cmake-build64 - cmake ..\\src -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\bin -D BUILD_WITH_TESTS=OFF - msbuild .\PACKAGE.vcxproj -p:Configuration=Release - cd .. + .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH #- name: test # run: | # cp .\\libraries\\VisualLeakDetector\\bin64\\*.* .\\bin64\\Release # cp .\\libraries\\VisualLeakDetector\\lib64\\*.lib .\\lib64\\Release # .\run_test_runner.bat - - name: create-output + - name: prepare-output if: always() run: | - mkdir build - mkdir test-output - mkdir installer - cp .\\bin64\\Release\\*.dll build - cp .\\bin64\\Release\\*.exe build - cp .\\lib64\\Release\\*.lib build - cp .\\cmake-build64\\*.msi installer - # cp .\\bin64\\Release\\*.log test-output - # cp .\\bin64\\Release\\*.html test-output + .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_WIN_BUILD_PATH - name: upload-build if: always() uses: actions/upload-artifact@v1 with: name: windows64-build - path: sql-odbc/build + path: sql-odbc/ci-output/build - name: upload-installer if: always() uses: actions/upload-artifact@v1 with: name: windows64-installer - path: sql-odbc/installer + path: sql-odbc/ci-output/installer #- name: upload-test-results # if: always() # uses: actions/upload-artifact@v1 # with: # name: windows-test-results - # path: test-output + # path: sql-odbc/ci-output/test-output diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index 02e3cf25a5..62c1f48e50 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -5,6 +5,13 @@ on: tags: - 'v*' +env: + CI_OUTPUT_PATH: "sql-odbc/ci-output" + ODBC_LIB_PATH: "./build/odbc/lib" + ODBC_BIN_PATH: "./build/odbc/bin" + ODBC_BUILD_PATH: "./build/odbc/build" + AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" + jobs: build-mac: runs-on: macos-latest @@ -35,23 +42,13 @@ jobs: brew install curl brew install cmake brew install libiodbc - - name: aws-sdk-cpp-setup - run: | - sh aws_sdk_cpp_setup.sh - - name: configure - run: | - prefix_path=$(pwd) - mkdir cmake-build - cd cmake-build - cmake ../src -DCMAKE_INSTALL_PREFIX=$prefix_path/AWSSDK/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="core" -DCUSTOM_MEMORY_MANAGEMENT="OFF" -DENABLE_RTTI="OFF" -DENABLE_TESTING="OFF" - cd .. - - name: build-driver + - name: configure-and-build-driver run: | - cmake --build cmake-build + ./build_mac_release64.sh - name: build-installer if: success() run: | - cd cmake-build + cd cmake-build64 cmake ../src make cpack . @@ -59,24 +56,24 @@ jobs: - name: create-output if: success() run: | - mkdir build + mkdir build-output mkdir test-output mkdir installer - cp ./lib64/*.dylib build - cp ./lib64/*.a build - cp $(ls -d bin64/* | grep -v "\.") build - cp ./cmake-build/*.pkg installer + cp ./build/odbc/lib/*.dylib build-output/ + cp ./build/odbc/lib/*.a build-output/ + cp ./cmake-build64/*.pkg installer/ + # cp $(ls -d bin64/* | grep -v "\.") build - name: upload-build if: success() uses: actions/upload-artifact@v1 with: - name: mac-build - path: sql-odbc/build + name: mac64-build + path: sql-odbc/build-output - name: upload-installer if: success() uses: actions/upload-artifact@v1 with: - name: mac-installer + name: mac64-installer path: sql-odbc/installer - name: upload-artifacts-s3 if: success() @@ -106,37 +103,28 @@ jobs: - name: build-installer if: success() run: | - cd cmake-build32 - cmake ..\\src -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\ -D BUILD_WITH_TESTS=OFF - msbuild .\PACKAGE.vcxproj -p:Configuration=Release - cd .. - - name: create-output + .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH + - name: prepare-output if: always() run: | - mkdir build - mkdir test-output - mkdir installer - cp .\\bin32\\Release\\*.dll build - cp .\\bin32\\Release\\*.exe build - cp .\\lib32\\Release\\*.lib build - cp .\\cmake-build32\\*.msi installer + .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_BUILD_PATH - name: upload-build if: always() uses: actions/upload-artifact@v1 with: name: windows32-build - path: sql-odbc/build + path: sql-odbc/ci-output/build - name: upload-installer if: always() uses: actions/upload-artifact@v1 with: name: windows32-installer - path: sql-odbc/installer + path: sql-odbc/ci-output/installer - name: upload-artifacts-s3 if: success() shell: bash run: | - cd installer + cd ci-output/installer windows_installer=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` echo $windows_installer aws s3 cp "$windows_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/windows/ @@ -161,37 +149,28 @@ jobs: - name: build-installer if: success() run: | - cd cmake-build64 - cmake ..\\src -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\ -D BUILD_WITH_TESTS=OFF - msbuild .\PACKAGE.vcxproj -p:Configuration=Release - cd .. - - name: create-output + .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH + - name: prepare-output if: always() run: | - mkdir build - mkdir test-output - mkdir installer - cp .\\bin64\\Release\\*.dll build - cp .\\bin64\\Release\\*.exe build - cp .\\lib64\\Release\\*.lib build - cp .\\cmake-build64\\*.msi installer + .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_WIN_BUILD_PATH - name: upload-build if: always() uses: actions/upload-artifact@v1 with: name: windows64-build - path: sql-odbc/build + path: sql-odbc/ci-output/build - name: upload-installer if: always() uses: actions/upload-artifact@v1 with: name: windows64-installer - path: sql-odbc/installer + path: sql-odbc/ci-output/installer - name: upload-artifacts-s3 if: success() shell: bash run: | - cd installer + cd ci-output/installer windows_installer=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` echo $windows_installer aws s3 cp "$windows_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/windows/ diff --git a/sql-odbc/.gitignore b/sql-odbc/.gitignore index ecaefb1c61..6f408e7c21 100644 --- a/sql-odbc/.gitignore +++ b/sql-odbc/.gitignore @@ -54,7 +54,10 @@ CMakeCache.txt CPackConfig.cmake CPackSourceConfig.cmake CTestTestfile.cmake +/build/ +/sdk-build32/ /sdk-build64/ +/cmake-build32/ /cmake-build64/ /src/PowerBIConnector/bin/Debug/ /src/PowerBIConnector/obj/ diff --git a/sql-odbc/BUILD_INSTRUCTIONS.md b/sql-odbc/BUILD_INSTRUCTIONS.md deleted file mode 100644 index 36c5202b36..0000000000 --- a/sql-odbc/BUILD_INSTRUCTIONS.md +++ /dev/null @@ -1,360 +0,0 @@ -# Elasticsearch ODBC Driver Build Instructions - -The ElasticsearchODBC driver can be build on Windows and Mac. - -## Setting up Dependencies - -The driver [source code](https://github.com/opendistro-for-elasticsearch/sql/tree/master/sql-odbc) must be downloaded onto the system to build it. - -### Windows Dependencies - -Windows requires the following dependencies - -* [cmake](https://cmake.org/install/) -* [Visual Studio 2019](https://visualstudio.microsoft.com/vs/) (Other versions may work, but only 2019 has been tested) - -### Mac Dependencies - -Homebrew must be installed to manage packages, to install homebrew see the [homebrew homepage](https://brew.sh/). -Using homebrew, install the following packages using the command provided: ->brew install [package] -> ->* curl ->* cmake ->* libiodbc - -## Building the Driver - -Before building the driver, the build files for the system must be generated, this is done using cmake. - -### Providing AWS Credentials - -As project uses AWS services for AWS SIGV4 authentication, you must [provide AWS credentials](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html). - -### Setting up AWS SDK - -#### Windows -* Open Developer PowerShell for VS. -* Run aws_sdk_cpp_setup.ps1 script from the project's root directory. - -#### Mac -* Run aws_sdk_cpp_setup.sh script from the project's root directory. - -### Generating the Build Files - -Open the project's root directory in a command line interface of your choice. Execute ->**cmake ./src -D CMAKE_INSTALL_PREFIX=\/AWSSDK/** - -**Note:** It is desirable to not run cmake directly in the 'src' directory, because it will generate build files inline with code. - -### General CMake Options - -**BUILD_WITH_TESTS** - -(Defaults to ON) If disabled, all tests and and test dependencies will be excluded from build which will optimize the installer package size. This option can set with the command line (using `-D`). - -### Building with Windows - -Building the driver on Windows is done using **Visual Studio**. ->Open **global_make_list.sln** with **Visual Studio 2019**. -> ->* Set the **Solution Configuration** to **Release** ->* Set the **Solution Platform** to **x64** -> ->**Build the solution** by right clicking it in the **Solution Explorer** and selecting **Build Solution** - -### Building with Mac - -Building the driver on Mac is done using make. Using the CLI, enter: ->**make** - -## Output Files - -Building the driver will yield the driver, tests, and a library files (Windows only). - -### Output Location on Windows - -Compiling on Windows will output the tests and the driver to **bin64/Release** and the driver library file to **lib64/Release** directory. There are also some additional test infrastructure files which output to the **bin64/Release** directory and the **lib64/Release** directory. - -The driver can be consumed by linking to it using the library file (elasticodbc.lib in lib64/Release). BI tools can consume the driver by specifying the location of the dll (elasticodbc.dll in bin64/Release) in the [DSN](#setting-up-a-dsn). - -### Output Location on Mac - -Compiling on Mac will output the tests to **bin64** and the driver to **lib64**. There are also some additional test infrastructure files which output to the **lib64** directory. - -## Packaging installer - -Build the driver with `BUILD_WITH_TESTS` option disabled. - -#### Windows - -Open the project's build directory in Developer PowerShell for VS. -> msbuild .\PACKAGE.vcxproj -p:Configuration=Release - -Installer named as `Open Distro for Elasticsearch SQL ODBC Driver--Windows.msi` will be generated in the build directory. - -#### Mac - -Run below command from the project's build directory. ->cpack . - -Installer named as `Open Distro for Elasticsearch SQL ODBC Driver--Darwin.pkg` will be generated in the build directory. - -## Running Tests - -Tests can be **executed directly**, or by using the **Test Runner**. - -**NOTES:** - -* A test DSN named `test_dsn` must be set up in order for certain tests in ITODBCConnection to pass. To configure the DSN, see the instructions, below. -* Datasets must be loaded into Elasticsearch using [kibana](https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html). See the section on loading datasets below. - -### Windows Test DSN Setup - -1. Open `src/IntegrationTests/ITODBCConnection/test_dsn.reg`. - * This contains the registry entries needed for setting up `test_dsn`. -2. Do one of the following: - * As an Administrator, run a command prompt or Powershell and run `reg import <.reg-file>` to add the entries to your registry. - * Manually add the entries to your registry using Registry Editor. - -### Mac Test DSN Setup - -1. Open `src/IntegrationTests/ITODBCConnection/test_odbc.ini` and `src/IntegrationTests/ITODBCConnection/test_odbcinst.ini` - * These contain the minimal configuration necessary for setting up `test_dsn`. -2. Do one of the following: - * Add the following lines to your .bash_profile to point the driver to these files. - * `export ODBCINI=/src/IntegrationTests/ITODBCConnection/test_odbc.ini` - * `export ODBCINSTINI=/src/IntegrationTests/ITODBCConnection/test_odbcinst.ini` - * Manually add the entries to your existing `odbc.ini` and `odbcinst.ini` entries. (normally found at `~/.odbc.ini` and `~/.odbcinst.ini`) - -### Loading Test Datasets - -Loading a dataset requires an [elasticsearch](https://opendistro.github.io/for-elasticsearch-docs/docs/install/) service running with [kibana](https://opendistro.github.io/for-elasticsearch-docs/docs/kibana/). If either of these are missing, please refer to the documentation on how to set them up. - -Note, if you wish to work with SSL/TLS, you need to configure Elasticsearch and Kibana to support it. See Working With SSL/TLS below. - -First load the sample datasets provided by kibana. - -1. Select home (top left corner) -2. Select 'Load a data set and a Kibana dashboard' -3. Select 'Add data' under 'Sample flight data' -4. Select 'Add data' under 'Sample eCommerce orders' -5. Select 'Add data' under 'Sample web logs' - -Then load custom data sets using the kibana console. -Select the wrench on the left control panel. Enter the following commands into the console and hit the play button after each one. - -```json -PUT /kibana_sample_data_types - { - "mappings": { - "properties": { - "type_boolean" : { "type": "boolean"}, - "type_byte" : { "type": "byte"}, - "type_short" : { "type": "short"}, - "type_integer" : { "type": "integer"}, - "type_long" : { "type": "long"}, - "type_half_float" : { "type": "half_float"}, - "type_float" : { "type": "float"}, - "type_double" : { "type": "double"}, - "type_scaled_float" : { "type": "scaled_float", "scaling_factor": 100 }, - "type_keyword" : { "type": "keyword"}, - "type_text" : { "type": "text"}, - "type_date" : { "type": "date"}, - "type_object" : { "type": "object"}, - "type_nested" : { "type": "nested"} - } - } - } -``` - -```json -POST /kibana_sample_data_types/_doc -{ - "type_boolean": true, - "type_byte" : -120, - "type_short" : -2000, - "type_integer" :-350000000, - "type_long" : -8010000000, - "type_half_float" : -2.115, - "type_float" : -3.1512, - "type_double" : -5335.2215, - "type_scaled_float" : -100.1, - "type_keyword" : "goodbye", - "type_text" : "planet", - "type_date" : "2016-02-21T12:23:52.803Z", - "type_object" : { "foo" : "bar" }, - "type_nested" : {"foo":"bar"} -} -``` - -```json -POST /kibana_sample_data_types/_doc -{ - "type_boolean": false, - "type_byte" : 100, - "type_short" : 1000, - "type_integer" : 250000000, - "type_long" : 8000000000, - "type_half_float" : 1.115, - "type_float" : 2.1512, - "type_double" : 25235.2215, - "type_scaled_float" : 100, - "type_keyword" : "hello", - "type_text" : "world", - "type_date" : "2018-07-22T12:23:52.803Z", - "type_object" : { "foo" : "bar" }, - "type_nested" : {"foo":"bar"} -} -``` - -### Working With SSL/TLS - -To disable SSL/TLS in the tests, the main CMakeLists.txt file must be edited. This can be found in the project 'src' directory. In the 'General compiler definitions' in the CMakeLists.txt file, USE_SSL is set. Remove this from the add_compile_definitions function to stop SSL/TLS from being used in the tests. - -To enable SSL/TLS on Elasticsearch, you must edit the Elasticsearch.yml file, found in the config directory of Elasticsearch. An example Elasticsearch yml file can be found in the dev folder of this project. The certificates specified MUST be in the config directory of the Elasticsearch instance. For more information, please refer to the [Elasticsearch security settings documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html). - -If you plan to use Kibana, as suggested for this project, you must also edit the Kibana settings. Notice, when specifying a certificate for Kibana, you do not need to place it in the Kibana config directory, but instead must provide the absolute path to it. An example Kibana.yml file can be found in the dev folder of this project. For more information, please refer to the [Kibana settings documentation](https://www.elastic.co/guide/en/kibana/current/settings.html). - -### Running Tests directly on Windows - -Tests can be executed directly using **Visual Studio** by setting the desired test as a **Start up Project** - ->* **Right click** the desired test project in the **Solution Explorer** ->* Select **Set as Startup Project** ->* Run the test by selecting **Local Windows Debugger** in the toolbar at the top of the application - -For more information, see the [Visual Studio Console Application documentation](https://docs.microsoft.com/en-us/cpp/build/vscpp-step-2-build?view=vs-2019). - -### Running Tests directly on Mac - -Tests can be executed using a command line interface. From the project root directory, enter: -> **bin64/** - -To execute a test. - -### Running Tests using the Test Runner - -The **Test Runner** requires [python](https://wiki.python.org/moin/BeginnersGuide/Download) to be installed on the system. Running the **Test Runner** will execute all the tests and compile a report with the results. The report indicates the execution status of all tests along with the execution time. To find error details of any failed test, hover over the test. - -#### Running Tests using the Test Runner on Windows - -Open the project's root directory in a command line interface of your choice. Execute ->**.\run_test_runner.bat** - -The **Test Runner** has been tried and tested with [Python3.8](https://www.python.org/downloads/release/python-380/) on **Windows systems**. Other versions of Python may work, but are untested. - -#### Running Tests using the Test Runner on Mac - -Open the project's root directory in a command line interface of your choice. Execute ->**./run_test_runner.sh** - -The **Test Runner** has been tried and tested with [Python3.7.6](https://www.python.org/downloads/release/python-376/) on **Mac systems**. Other versions of Python may work, but are untested. - -### Running Tests with Coverage (Mac only) - -(using a CMake script provided by George Cave (StableCoder) under the Apache 2.0 license, found [here](https://github.com/StableCoder/cmake-scripts/blob/master/code-coverage.cmake)) - -> **NOTE**: Before building with coverage, make sure the following directory is in your PATH environment variable: -> `/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin` - -To build the tests with code coverage enabled, set the `CODE_COVERAGE` variable to `ON` when preparing your CMake build. -```bash -cmake ... -DBUILD_WITH_TESTS=ON -DCODE_COVERAGE=ON -``` - -To get coverage for the driver library, you must use the `ccov-all` target, which runs all test suites and components with coverage. -```bash -make ccov-all -``` - -This will generate an HTML report at `/ccov/all-merged/index.html`, which can be opened in a web browser to view a summary of the overall code coverage, as well as line-by-line coverage for individual files. - -For more information interpreting this report, see https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#interpreting-reports. - -## Setting up a DSN - -A **D**ata **S**ouce **N**ame is used to store driver information in the system. By storing the information in the system, the information does not need to be specified each time the driver connects. - -### Windows - -> To setup DSN, add following keys in the Registry -> - >* **HKEY_LOCAL_MACHINE/SOFTWARE/ ODBC/ODBC.INI** : Contains a key for each Data Source Name (DSN) - >* **HKEY_LOCAL_MACHINE/SOFTWARE/ ODBC/ODBC.INI/ODBC Data Sources** : Lists the data sources - >* **HKEY_LOCAL_MACHINE/SOFTWARE/ODBC/ODBCINST.INI** : Define each driver's name and setup location - >* **HKEY_LOCAL_MACHINE/SOFTWARE/ODBC/ODBCINST.INI/ODBC Drivers** : Lists the installed drivers. -> ->These keys can be added manually in the Registry Editor (Start > Run > Regedit) one by one. Alternatively, keys can be added together as follows: -> ->1. Modify the appropriate values for these keys in `src/IntegrationTests/ITODBCConnection/test_dsn.reg` ->2. Double click on the `test_dsn.reg` file. ->3. Click `Yes` on the confirmation window to add keys in the registry. - -### Mac - -**iODBC Administrator** can be used to setup a **DSN** on Mac. - -> 1. Open **iODBC Administrator** - > * **iODBC Administrator** is installed with **iODBC Driver Manager** and can be found by searching the **Spotlight** (or found in **/Applications**) -> 2. Go to the **ODBC Drivers** tab -> 3. Click **Add a Driver** - > * **Description of the Driver**: The driver name used for the **ODBC connections** (ex. *ElasticsearchODBC*) - > * **Driver File Name**: The path to the **driver file** (*< Project Directory >/lib64/libelasticodbc.dylib*) - > * **Setup File Name**: The path to the **driver file** (*< Project Directory >/lib64/libelasticodbc.dylib*) - > * Set as a **User** driver - > * Select **OK** to save the options -> 4. Go to the **User DSN** tab -> 5. Select **Add** - > * Choose the driver that was added in **Step 3** - > * **Data Source Name (DSN)**: The name of the DSN used to store connection options (ex. *ElasticsearchODBC*) - > * **Comment**: Not required - > * Add the following **key-value pairs** using the **'+'** button - > * **Host** | **localhost** // Or a different server endpoint - > * **Port** | **9200** // Or whatever your endpoints port is - > * **Username** | **admin** // Or whatever your endpoints username is - > * **Password** | **admin** // Or whatever your endpoints password is - > * Select **OK** to **save options** -> 6. Select **OK** to exit the **Administrator** - -If “General installer error” is encountered when saving the ODBC Driver, see Troubleshooting, below. - -## Working with Tableau - -[Tableau Desktop](https://www.tableau.com/products/desktop) must be installed on the target machine. - - 1. Open **Tableau Desktop** - 2. Select **More…** - 3. Select **Other Databases (ODBC)** - 4. In the **DSN drop-down**, select the *Elasticsearch DSN* you set up in the previous set of steps - 5. The options you added will *automatically* be filled into the **Connection Attributes** - 6. Select **Sign In** - 7. After a few seconds, Tableau will connect to your Elasticsearch server - -## Troubleshooting - -### iODBC Administrator: “General installer error” when saving new ODBC Driver - -Try the following: - -1. Create the folder ~/Library/ODBC, then try again -2. Create two files in ~/Library/ODBC, then open iODBC Administrator and verify the contents of **odbcinst.ini** and **odbc.ini** align with the format below. - * **odbcinst.ini** (will be found in **ODBC Drivers**) - >[ODBC Drivers] - \ = Installed - > - >[\] - Driver = \/lib64/libelasticodbc.dylib - Setup = \/lib64/libelasticodbc.dylib - - * **odbc.ini** (will be found in **User DSNs**) - >[ODBC Data Sources] - \ = \ - > - >[\] - Driver = \/lib64/libelasticodbc.dylib - Description = - Host = localhost - Port = 9200 - Username = admin - Password = admin diff --git a/sql-odbc/README.md b/sql-odbc/README.md index 8245ecf3dd..1b7a3b6c55 100644 --- a/sql-odbc/README.md +++ b/sql-odbc/README.md @@ -65,7 +65,7 @@ To setup a connection, the driver uses an ODBC connection string. Connection str ### Building -Please refer to the [build instructions](./BUILD_INSTRUCTIONS.md) for detailed build instructions on your platform. +Please refer to the [build instructions](./docs/dev/BUILD_INSTRUCTIONS.md) for detailed build instructions on your platform. If your PC is already setup to build the library, you can simply invoke cmake using > cmake ./src diff --git a/sql-odbc/aws_sdk_cpp_setup.ps1 b/sql-odbc/aws_sdk_cpp_setup.ps1 deleted file mode 100644 index ecc4aa6b65..0000000000 --- a/sql-odbc/aws_sdk_cpp_setup.ps1 +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# - -git clone "https://github.com/aws/aws-sdk-cpp.git" - -$prefix_path = (pwd).path - -mkdir sdk-build - -cd sdk-build - -cmake ..\\aws-sdk-cpp\\ -D CMAKE_INSTALL_PREFIX=$prefix_path\AWSSDK\ -D CMAKE_BUILD_TYPE=Release -D BUILD_ONLY="core" -D CUSTOM_MEMORY_MANAGEMENT="OFF" -D ENABLE_RTTI="OFF" -D ENABLE_TESTING="OFF" - -msbuild ALL_BUILD.vcxproj /p:Configuration=Release - -msbuild INSTALL.vcxproj /p:Configuration=Release - -cd .. \ No newline at end of file diff --git a/sql-odbc/build_win_debug32.ps1 b/sql-odbc/build_win_debug32.ps1 index add9a03dfb..7e23ada173 100644 --- a/sql-odbc/build_win_debug32.ps1 +++ b/sql-odbc/build_win_debug32.ps1 @@ -1,27 +1,2 @@ -# Build AWS SDK -$BITNESS=32 - -# Compare Bitness for 32 -# $ARCH="Win32" - -mkdir sdk-build${BITNESS} -cd sdk-build${BITNESS} - -git clone "https://github.com/aws/aws-sdk-cpp.git" - -$prefix_path = (pwd).path -cmake .\aws-sdk-cpp -A Win32 -D CMAKE_INSTALL_PREFIX=${prefix_path}\AWSSDK\ -D CMAKE_BUILD_TYPE=Debug -D BUILD_ONLY="core" -D CUSTOM_MEMORY_MANAGEMENT="OFF" -D ENABLE_RTTI="OFF" -D ENABLE_TESTING="OFF" - -msbuild ALL_BUILD.vcxproj /m /p:Configuration=Debug -msbuild INSTALL.vcxproj /m /p:Configuration=Debug - -cd .. - -# # Configure Project -cmake -S src -B cmake-build${BITNESS} -A Win32 -D CMAKE_INSTALL_PREFIX=sdk-build${BITNESS}\AWSSDK\ -D BUILD_WITH_TESTS=ON - -# # Build Project -cmake --build .\cmake-build${BITNESS} --config Debug --parallel 4 - -cp .\sdk-build32\bin\Debug\* .\bin32\Debug -cp .\cmake-build32\bin\Debug\* .\bin32\Debug +$WORKING_DIR = (Get-Location).Path +.\scripts\build_windows.ps1 $WORKING_DIR Debug 32 diff --git a/sql-odbc/build_win_debug64.ps1 b/sql-odbc/build_win_debug64.ps1 index 998dffedca..ea7084bada 100644 --- a/sql-odbc/build_win_debug64.ps1 +++ b/sql-odbc/build_win_debug64.ps1 @@ -1,27 +1,2 @@ -# Build AWS SDK -$BITNESS = 64 - -# Compare Bitness for 32 -# $ARCH="x64" - -mkdir sdk-build${BITNESS} -cd sdk-build${BITNESS} - -git clone "https://github.com/aws/aws-sdk-cpp.git" - -$prefix_path = (pwd).path -cmake .\aws-sdk-cpp -A x64 -D CMAKE_INSTALL_PREFIX=${prefix_path}\AWSSDK\ -D CMAKE_BUILD_TYPE=Debug -D BUILD_ONLY="core" -D CUSTOM_MEMORY_MANAGEMENT="OFF" -D ENABLE_RTTI="OFF" -D ENABLE_TESTING="OFF" - -msbuild ALL_BUILD.vcxproj /m /p:Configuration=Debug -msbuild INSTALL.vcxproj /m /p:Configuration=Debug - -cd .. - -# # Configure Project -cmake -S src -B cmake-build${BITNESS} -A x64 -D CMAKE_INSTALL_PREFIX=sdk-build${BITNESS}\AWSSDK\ -D BUILD_WITH_TESTS=ON - -# # Build Project -cmake --build .\cmake-build${BITNESS} --config Debug --parallel 4 - -cp .\sdk-build64\bin\Debug\* .\bin64\Debug -cp .\cmake-build64\bin\Debug\* .\bin64\Debug +$WORKING_DIR = (Get-Location).Path +.\scripts\build_windows.ps1 $WORKING_DIR Debug 64 diff --git a/sql-odbc/build_win_release32.ps1 b/sql-odbc/build_win_release32.ps1 index b693778c94..4bcf4bd48e 100644 --- a/sql-odbc/build_win_release32.ps1 +++ b/sql-odbc/build_win_release32.ps1 @@ -1,27 +1,2 @@ -# Build AWS SDK -$BITNESS=32 - -# Compare Bitness for 32 -# $ARCH="Win32" - -mkdir sdk-build${BITNESS} -cd sdk-build${BITNESS} - -git clone "https://github.com/aws/aws-sdk-cpp.git" - -$prefix_path = (pwd).path -cmake .\aws-sdk-cpp -A Win32 -D CMAKE_INSTALL_PREFIX=${prefix_path}\AWSSDK\ -D CMAKE_BUILD_TYPE=Release -D BUILD_ONLY="core" -D CUSTOM_MEMORY_MANAGEMENT="OFF" -D ENABLE_RTTI="OFF" -D ENABLE_TESTING="OFF" - -msbuild ALL_BUILD.vcxproj /m /p:Configuration=Release -msbuild INSTALL.vcxproj /m /p:Configuration=Release - -cd .. - -# # Configure Project -cmake -S src -B cmake-build${BITNESS} -A Win32 -D CMAKE_INSTALL_PREFIX=sdk-build${BITNESS}\AWSSDK\ -D BUILD_WITH_TESTS=ON - -# # Build Project -cmake --build .\cmake-build${BITNESS} --config Release --parallel 4 - -cp .\sdk-build32\bin\Release\* .\bin32\Release -cp .\cmake-build32\bin\Release\* .\bin32\Release +$WORKING_DIR = (Get-Location).Path +.\scripts\build_windows.ps1 $WORKING_DIR Release 32 diff --git a/sql-odbc/build_win_release64.ps1 b/sql-odbc/build_win_release64.ps1 index 15a917219f..82b1199b33 100644 --- a/sql-odbc/build_win_release64.ps1 +++ b/sql-odbc/build_win_release64.ps1 @@ -1,27 +1,2 @@ -# Build AWS SDK -$BITNESS = 64 - -# Compare Bitness for 32 -# $ARCH="x64" - -mkdir sdk-build64 -cd .\sdk-build64 - -git clone "https://github.com/aws/aws-sdk-cpp.git" - -$prefix_path = (pwd).path -cmake .\aws-sdk-cpp -A x64 -D CMAKE_INSTALL_PREFIX=${prefix_path}\AWSSDK\ -D CMAKE_BUILD_TYPE=Release -D BUILD_ONLY="core" -D CUSTOM_MEMORY_MANAGEMENT="OFF" -D ENABLE_RTTI="OFF" -D ENABLE_TESTING="OFF" - -msbuild ALL_BUILD.vcxproj /m /p:Configuration=Release -msbuild INSTALL.vcxproj /m /p:Configuration=Release - -cd .. - -# # Configure Project -cmake -S src -B cmake-build64 -A x64 -D CMAKE_INSTALL_PREFIX=sdk-build64\AWSSDK\ -D BUILD_WITH_TESTS=ON - -# # Build Project -cmake --build .\cmake-build64 --config Release --parallel 4 - -cp .\sdk-build64\bin\Release\* .\bin64\Release -cp .\cmake-build64\bin\Release\* .\bin64\Release +$WORKING_DIR = (Get-Location).Path +.\scripts\build_windows.ps1 $WORKING_DIR Release 64 diff --git a/sql-odbc/docs/dev/BUILD_INSTRUCTIONS.md b/sql-odbc/docs/dev/BUILD_INSTRUCTIONS.md new file mode 100644 index 0000000000..f7d60cc949 --- /dev/null +++ b/sql-odbc/docs/dev/BUILD_INSTRUCTIONS.md @@ -0,0 +1,118 @@ +# ODFE SQL ODBC Driver Build Instructions + +## Windows + +### Dependencies + +* [cmake](https://cmake.org/install/) +* [Visual Studio 2019](https://visualstudio.microsoft.com/vs/) (Other versions may work, but only 2019 has been tested) +* [ODBC Driver source code](https://github.com/opendistro-for-elasticsearch/sql/tree/master/sql-odbc) + +### Build + +#### with Visual Studio + +Run `./build_win_.ps1` to generate a VS2019 project for building/testing the driver. (the build may fail, but should still generate a `.sln` file) + +The solution can be found at `\build\odbc\build\global_make_list.sln`. + +#### with Developer Powershell + +Use `./build_win_.ps1` to build the driver from a Developer Powershell prompt. + +> A shortcut is installed on your system with Visual Studio (search for **"Developer Powershell for VS 2019"**) + +> Programs launched with this prompt (ex: VS Code) also have access to the Developer shell. + +### Build Output + +``` +build +└-- + └-- odbc + └-- bin + └-- + └-- build + └-- lib + └-- aws-sdk + └-- build + └-- install +``` + +* Driver DLL: `.\build\\odbc\bin\\odfesqlodbc.dll` +* Test binaries folder: `.\build\\odbc\bin\` + +### Packaging + +From a Developer Powershell, run: +``` +msbuild .\build\Release\odbc\PACKAGE.vcxproj -p:Configuration=Release +``` + +An installer named as `Open Distro for Elasticsearch SQL ODBC Driver--Windows--bit.msi` will be generated in the build directory. + + +## Mac +(TODO: upgrade build scripts & documentation for Mac) + +### Dependencies + +Homebrew must be installed to manage packages, to install homebrew see the [homebrew homepage](https://brew.sh/). +Using homebrew, install the following packages using the command provided: +>brew install [package] +> +>* curl +>* cmake +>* libiodbc + +### Building the Driver + +From a Bash shell: + +`./build_mac_.sh` + +### Output Location on Mac + +Compiling on Mac will output the tests to **bin64** and the driver to **lib64**. There are also some additional test infrastructure files which output to the **lib64** directory. + +### Packaging + +Run below command from the project's build directory. +>cpack . + +Installer named as `Open Distro for Elasticsearch SQL ODBC Driver--Darwin.pkg` will be generated in the build directory. + +## General Build Info + +### ODBC Driver CMake Options + +**BUILD_WITH_TESTS** + +(Defaults to ON) If disabled, all tests and and test dependencies will be excluded from build which will optimize the installer package size. This option can set with the command line (using `-D`). + +### Working With SSL/TLS + +To disable SSL/TLS in the tests, the main CMakeLists.txt file must be edited. This can be found in the project 'src' directory. In the 'General compiler definitions' in the CMakeLists.txt file, USE_SSL is set. Remove this from the add_compile_definitions function to stop SSL/TLS from being used in the tests. + +To enable SSL/TLS on Elasticsearch, you must edit the Elasticsearch.yml file, found in the config directory of Elasticsearch. An example Elasticsearch yml file can be found in the dev folder of this project. The certificates specified MUST be in the config directory of the Elasticsearch instance. For more information, please refer to the [Elasticsearch security settings documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html). + +If you plan to use Kibana, as suggested for this project, you must also edit the Kibana settings. Notice, when specifying a certificate for Kibana, you do not need to place it in the Kibana config directory, but instead must provide the absolute path to it. An example Kibana.yml file can be found in the dev folder of this project. For more information, please refer to the [Kibana settings documentation](https://www.elastic.co/guide/en/kibana/current/settings.html). + +### Setting up a DSN + +A **D**ata **S**ouce **N**ame is used to store driver information in the system. By storing the information in the system, the information does not need to be specified each time the driver connects. + +#### Windows + +> To setup DSN, add following keys in the Registry +> + >* **HKEY_LOCAL_MACHINE/SOFTWARE/ODBC/ODBC.INI** : Contains a key for each Data Source Name (DSN) + >* **HKEY_LOCAL_MACHINE/SOFTWARE/ODBC/ODBC.INI/ODBC Data Sources** : Lists the data sources + >* **HKEY_LOCAL_MACHINE/SOFTWARE/ODBC/ODBCINST.INI** : Define each driver's name and setup location + >* **HKEY_LOCAL_MACHINE/SOFTWARE/ODBC/ODBCINST.INI/ODBC Drivers** : Lists the installed drivers. +> +>These keys can be added manually in the Registry Editor (Start > Run > Regedit) one by one. Alternatively, keys can be added together as follows: +> +>1. Modify the appropriate values for these keys in `src/IntegrationTests/ITODBCConnection/test_dsn.reg` +>2. Double click on the `test_dsn.reg` file. +>3. Click `Yes` on the confirmation window to add keys in the registry. diff --git a/sql-odbc/docs/dev/datasets/kibana_sample_data_types.md b/sql-odbc/docs/dev/datasets/kibana_sample_data_types.md new file mode 100644 index 0000000000..5f2e0fc185 --- /dev/null +++ b/sql-odbc/docs/dev/datasets/kibana_sample_data_types.md @@ -0,0 +1,72 @@ +# kibana_sample_data_types + +## Mapping + +```json +PUT /kibana_sample_data_types +{ + "mappings": { + "properties": { + "type_boolean" : { "type": "boolean"}, + "type_byte" : { "type": "byte"}, + "type_short" : { "type": "short"}, + "type_integer" : { "type": "integer"}, + "type_long" : { "type": "long"}, + "type_half_float" : { "type": "half_float"}, + "type_float" : { "type": "float"}, + "type_double" : { "type": "double"}, + "type_scaled_float" : { + "type": "scaled_float", + "scaling_factor": 100 + }, + "type_keyword" : { "type": "keyword"}, + "type_text" : { "type": "text"}, + "type_date" : { "type": "date"}, + "type_object" : { "type": "object"}, + "type_nested" : { "type": "nested"} + } + } +} +``` + +## Data + +```json +POST /kibana_sample_data_types/_doc +{ + "type_boolean": true, + "type_byte" : -120, + "type_short" : -2000, + "type_integer" :-350000000, + "type_long" : -8010000000, + "type_half_float" : -2.115, + "type_float" : -3.1512, + "type_double" : -5335.2215, + "type_scaled_float" : -100.1, + "type_keyword" : "goodbye", + "type_text" : "planet", + "type_date" : "2016-02-21T12:23:52.803Z", + "type_object" : { "foo" : "bar" }, + "type_nested" : {"foo":"bar"} +} +``` + +```json +POST /kibana_sample_data_types/_doc +{ + "type_boolean": false, + "type_byte" : 100, + "type_short" : 1000, + "type_integer" : 250000000, + "type_long" : 8000000000, + "type_half_float" : 1.115, + "type_float" : 2.1512, + "type_double" : 25235.2215, + "type_scaled_float" : 100, + "type_keyword" : "hello", + "type_text" : "world", + "type_date" : "2018-07-22T12:23:52.803Z", + "type_object" : { "foo" : "bar" }, + "type_nested" : {"foo":"bar"} +} +``` \ No newline at end of file diff --git a/sql-odbc/docs/dev/run_tests.md b/sql-odbc/docs/dev/run_tests.md new file mode 100644 index 0000000000..903e5f8f0a --- /dev/null +++ b/sql-odbc/docs/dev/run_tests.md @@ -0,0 +1,106 @@ +# ODFE SQL ODBC Driver Testing + +## Preparation + +* Latest version of [Open Distro for Elasticsearch](https://opendistro.github.io/for-elasticsearch-docs/docs/install/) + +### Loading Test Datasets + +Loading a dataset requires an [elasticsearch](https://opendistro.github.io/for-elasticsearch-docs/docs/install/) service running with [kibana](https://opendistro.github.io/for-elasticsearch-docs/docs/kibana/). If either of these are missing, please refer to the documentation on how to set them up. + +Note, if you wish to work with SSL/TLS, you need to configure Elasticsearch and Kibana to support it. See Working With SSL/TLS below. + +First load the sample datasets provided by kibana. + +1. Select home (top left corner) +2. Select 'Load a data set and a Kibana dashboard' +3. Select 'Add data' under 'Sample flight data' +4. Select 'Add data' under 'Sample eCommerce orders' +5. Select 'Add data' under 'Sample web logs' + +Then load the following custom data sets using the kibana console. +Select the wrench on the left control panel. Enter the following commands into the console and hit the play button after each one. + +* [kibana_sample_data_types](./datasets/kibana_sample_data_types.md) + +## Running Tests + +Tests can be **executed directly**, or by using the **Test Runner**. + +**NOTES:** + +* A test DSN named `test_dsn` must be set up in order for certain tests in ITODBCConnection to pass. To configure the DSN, see the instructions, below. +* Datasets must be loaded into Elasticsearch using [kibana](https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html). See the section on loading datasets below. + +### Windows Test DSN Setup + +1. Open `src/IntegrationTests/ITODBCConnection/test_dsn.reg`. + * This contains the registry entries needed for setting up `test_dsn`. +2. Do one of the following: + * As an Administrator, run a command prompt or Powershell and run `reg import <.reg-file>` to add the entries to your registry. + * Manually add the entries to your registry using Registry Editor. + +### Mac Test DSN Setup + +1. Open `src/IntegrationTests/ITODBCConnection/test_odbc.ini` and `src/IntegrationTests/ITODBCConnection/test_odbcinst.ini` + * These contain the minimal configuration necessary for setting up `test_dsn`. +2. Do one of the following: + * Add the following lines to your .bash_profile to point the driver to these files. + * `export ODBCINI=/src/IntegrationTests/ITODBCConnection/test_odbc.ini` + * `export ODBCINSTINI=/src/IntegrationTests/ITODBCConnection/test_odbcinst.ini` + * Manually add the entries to your existing `odbc.ini` and `odbcinst.ini` entries. (normally found at `~/.odbc.ini` and `~/.odbcinst.ini`) + +### Running Tests directly on Windows + +Tests can be executed directly using **Visual Studio** by setting the desired test as a **Start up Project** + +>* **Right click** the desired test project in the **Solution Explorer** +>* Select **Set as Startup Project** +>* Run the test by selecting **Local Windows Debugger** in the toolbar at the top of the application + +For more information, see the [Visual Studio Console Application documentation](https://docs.microsoft.com/en-us/cpp/build/vscpp-step-2-build?view=vs-2019). + +### Running Tests directly on Mac + +Tests can be executed using a command line interface. From the project root directory, enter: +> **bin64/** + +### Running Tests using the Test Runner + +The **Test Runner** requires [python](https://wiki.python.org/moin/BeginnersGuide/Download) to be installed on the system. Running the **Test Runner** will execute all the tests and compile a report with the results. The report indicates the execution status of all tests along with the execution time. To find error details of any failed test, hover over the test. + +#### Running Tests using the Test Runner on Windows + +Open the project's root directory in a command line interface of your choice. Execute +>**.\run_test_runner.bat** + +The **Test Runner** has been tried and tested with [Python3.8](https://www.python.org/downloads/release/python-380/) on **Windows systems**. Other versions of Python may work, but are untested. + +#### Running Tests using the Test Runner on Mac + +Open the project's root directory in a command line interface of your choice. Execute +>**./run_test_runner.sh** + +The **Test Runner** has been tried and tested with [Python3.7.6](https://www.python.org/downloads/release/python-376/) on **Mac systems**. Other versions of Python may work, but are untested. + +### Run with Coverage (Mac only) + +(using a CMake script provided by George Cave (StableCoder) under the Apache 2.0 license, found [here](https://github.com/StableCoder/cmake-scripts/blob/master/code-coverage.cmake)) + +> **NOTE**: Before building with coverage, make sure the following directory is in your PATH environment variable: +> `/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin` + +To build the tests with code coverage enabled, set the `CODE_COVERAGE` variable to `ON` when preparing your CMake build. +```bash +cmake ... -DBUILD_WITH_TESTS=ON -DCODE_COVERAGE=ON +``` + +To get coverage for the driver library, you must use the `ccov-all` target, which runs all test suites and components with coverage. +```bash +make ccov-all +``` + +This will generate an HTML report at `/ccov/all-merged/index.html`, which can be opened in a web browser to view a summary of the overall code coverage, as well as line-by-line coverage for individual files. + +For more information interpreting this report, see https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#interpreting-reports. + diff --git a/sql-odbc/docs/user/mac_configure_dsn.md b/sql-odbc/docs/user/mac_configure_dsn.md index 303ad23640..c4dd6b3faa 100644 --- a/sql-odbc/docs/user/mac_configure_dsn.md +++ b/sql-odbc/docs/user/mac_configure_dsn.md @@ -36,4 +36,36 @@ This is not required if you are using the Tableau Connector, but will help with

-

\ No newline at end of file +

+ + +## Troubleshooting + +### iODBC Administrator: “General installer error” when saving new ODBC Driver + +Try the following: + +1. Create the folder `~/Library/ODBC`, then try again +2. Create two files in `~/Library/ODBC`, then open iODBC Administrator and verify the contents of **odbcinst.ini** and **odbc.ini** align with the format below. + * **odbcinst.ini** (will be found in **ODBC Drivers**) + ```ini + [ODBC Drivers] + = Installed + + [] + Driver = /lib64/libodfesqlodbc.dylib + Setup = /lib64/libodfesqlodbc.dylib + ``` + * **odbc.ini** (will be found in **User DSNs**) + ```ini + [ODBC Data Sources] + = + + [] + Driver = /lib64/libodfesqlodbc.dylib + Description = + Host = localhost + Port = 9200 + Username = admin + Password = admin + ``` \ No newline at end of file diff --git a/sql-odbc/run_test_runner.bat b/sql-odbc/run_test_runner.bat index 3f08b46ddd..a413b3ab81 100644 --- a/sql-odbc/run_test_runner.bat +++ b/sql-odbc/run_test_runner.bat @@ -15,7 +15,7 @@ set PROJECT_DIR=%CD% set TEST_RUNNER_DIR=%PROJECT_DIR%\src\TestRunner -set WORKING_DIR=%PROJECT_DIR%\bin64\Release +set WORKING_DIR=%PROJECT_DIR%\build\Debug64\odbc\bin\Debug cd %WORKING_DIR% diff --git a/sql-odbc/scripts/build_aws-sdk-cpp.ps1 b/sql-odbc/scripts/build_aws-sdk-cpp.ps1 new file mode 100644 index 0000000000..f3835e77a7 --- /dev/null +++ b/sql-odbc/scripts/build_aws-sdk-cpp.ps1 @@ -0,0 +1,34 @@ +$CONFIGURATION = $args[0] +$WIN_ARCH = $args[1] +$SRC_DIR = $args[2] +$BUILD_DIR = $args[3] +$INSTALL_DIR = $args[4] + +Write-Host $args + +# Clone the AWS SDK CPP repo +# $SDK_VER = "1.7.29" +# -b "$SDK_VER" ` +git clone ` + --single-branch ` + "https://github.com/aws/aws-sdk-cpp.git" ` + $SRC_DIR + +# Make and move to build directory +New-Item -Path $BUILD_DIR -ItemType Directory -Force | Out-Null +Set-Location $BUILD_DIR + +# Configure and build +cmake $SRC_DIR ` + -A $WIN_ARCH ` + -D CMAKE_INSTALL_PREFIX=$INSTALL_DIR ` + -D CMAKE_BUILD_TYPE=$CONFIGURATION ` + -D BUILD_ONLY="core" ` + -D ENABLE_UNITY_BUILD="ON" ` + -D CUSTOM_MEMORY_MANAGEMENT="OFF" ` + -D ENABLE_RTTI="OFF" ` + -D ENABLE_TESTING="OFF" + +# Build AWS SDK and install to $INSTALL_DIR +msbuild ALL_BUILD.vcxproj /m /p:Configuration=$CONFIGURATION +msbuild INSTALL.vcxproj /m /p:Configuration=$CONFIGURATION diff --git a/sql-odbc/scripts/build_driver.ps1 b/sql-odbc/scripts/build_driver.ps1 new file mode 100644 index 0000000000..85d9de5731 --- /dev/null +++ b/sql-odbc/scripts/build_driver.ps1 @@ -0,0 +1,15 @@ +$CONFIGURATION = $args[0] +$WIN_ARCH = $args[1] +$SRC_DIR = $args[2] +$BUILD_DIR = $args[3] +$INSTALL_DIR = $args[4] + +cmake -S $SRC_DIR ` + -B $BUILD_DIR ` + -A $WIN_ARCH ` + -D CMAKE_BUILD_TYPE=$CONFIGURATION ` + -D CMAKE_INSTALL_PREFIX=$INSTALL_DIR ` + -D BUILD_WITH_TESTS=ON + +# # Build Project +cmake --build $BUILD_DIR --config $CONFIGURATION --parallel 4 diff --git a/sql-odbc/scripts/build_installer.ps1 b/sql-odbc/scripts/build_installer.ps1 new file mode 100644 index 0000000000..71b0f54e1b --- /dev/null +++ b/sql-odbc/scripts/build_installer.ps1 @@ -0,0 +1,17 @@ +$CONFIGURATION = $args[0] +$WIN_ARCH = $args[1] +$SRC_DIR = $args[2] +$BUILD_DIR = $args[3] +$INSTALL_DIR = $args[4] + +Write-Host $args + +cmake -S $SRC_DIR ` + -B $BUILD_DIR ` + -A $WIN_ARCH ` + -D CMAKE_BUILD_TYPE=$CONFIGURATION ` + -D CMAKE_INSTALL_PREFIX=$INSTALL_DIR ` + -D BUILD_WITH_TESTS=OFF + +# # Build Project +msbuild $BUILD_DIR\PACKAGE.vcxproj -m -p:Configuration=Release diff --git a/sql-odbc/scripts/build_windows.ps1 b/sql-odbc/scripts/build_windows.ps1 new file mode 100644 index 0000000000..41ef150424 --- /dev/null +++ b/sql-odbc/scripts/build_windows.ps1 @@ -0,0 +1,48 @@ +# Build AWS SDK +$CURRENT_DIR = Get-Location +$WORKING_DIR = $args[0] +$CONFIGURATION = $args[1] +$BITNESS = $args[2] +if ($BITNESS -eq "64") { + $WIN_ARCH = "x64" +} +else { + $WIN_ARCH = "Win32" +} + +# Create build directory; remove if exists +$BUILD_DIR = "${WORKING_DIR}\build" +# $BUILD_DIR = "${WORKING_DIR}\build\${CONFIGURATION}${BITNESS}" +New-Item -Path $BUILD_DIR -ItemType Directory -Force | Out-Null + +# Build AWS SDK CPP +$SDK_SOURCE_DIR = "${WORKING_DIR}\src\aws-sdk-cpp" +$SDK_BUILD_DIR = "${BUILD_DIR}\aws-sdk\build" +$SDK_INSTALL_DIR = "${BUILD_DIR}\aws-sdk\install" + +.\scripts\build_aws-sdk-cpp.ps1 ` + $CONFIGURATION $WIN_ARCH ` + $SDK_SOURCE_DIR $SDK_BUILD_DIR $SDK_INSTALL_DIR +Set-Location $CURRENT_DIR + +# Build driver +$DRIVER_SOURCE_DIR = "${WORKING_DIR}\src" +$DRIVER_BUILD_DIR = "${BUILD_DIR}\odbc\cmake" + +.\scripts\build_driver.ps1 ` + $CONFIGURATION $WIN_ARCH ` + $DRIVER_SOURCE_DIR $DRIVER_BUILD_DIR $SDK_INSTALL_DIR +Set-Location $CURRENT_DIR + +# Move driver dependencies to bin directory for testing +$DRIVER_BIN_DIR = "$DRIVER_BUILD_DIR\..\bin\$CONFIGURATION" +New-Item -Path $DRIVER_BIN_DIR -ItemType Directory -Force | Out-Null + +Copy-Item $SDK_BUILD_DIR\bin\$CONFIGURATION\* $DRIVER_BIN_DIR +Copy-Item $DRIVER_BUILD_DIR\bin\$CONFIGURATION\* $DRIVER_BIN_DIR +if ($BITNESS -eq "32") { + # Strip bitness from 32bit VLD DLL dir name + $BITNESS = $null + $WIN_ARCH = "x86" +} +Copy-Item .\libraries\VisualLeakDetector\bin$BITNESS\vld_$WIN_ARCH.dll $DRIVER_BIN_DIR diff --git a/sql-odbc/scripts/prepare_ci_output.ps1 b/sql-odbc/scripts/prepare_ci_output.ps1 new file mode 100644 index 0000000000..27e0728a6b --- /dev/null +++ b/sql-odbc/scripts/prepare_ci_output.ps1 @@ -0,0 +1,20 @@ +$ODBC_BIN_PATH = $args[0] +$ODBC_LIB_PATH = $args[1] +$ODBC_BUILD_PATH = $args[2] + +Write-Host $args + +# Create staging directories for CI artifacts +$CI_OUTPUT_PATH = ".\ci-output" +New-Item -Path $CI_OUTPUT_PATH -ItemType Directory -Force | Out-Null +New-Item -Path $CI_OUTPUT_PATH\build -ItemType Directory -Force | Out-Null +New-Item -Path $CI_OUTPUT_PATH\installer -ItemType Directory -Force | Out-Null + +# Copy CI artifacts to respective directories +Copy-Item $ODBC_BIN_PATH\*.dll $CI_OUTPUT_PATH\build +Copy-Item $ODBC_BIN_PATH\*.exe $CI_OUTPUT_PATH\build +Copy-Item $ODBC_LIB_PATH\*.lib $CI_OUTPUT_PATH\build +Copy-Item $ODBC_BUILD_PATH\*.msi $CI_OUTPUT_PATH\installer +# mkdir $CI_OUTPUT_PATH\test +# Copy-Item $ODBC_BIN_PATH\*.log $CI_OUTPUT_PATH\test +# Copy-Item $ODBC_BIN_PATH\*.html $CI_OUTPUT_PATH\test \ No newline at end of file diff --git a/sql-odbc/src/CMakeLists.txt b/sql-odbc/src/CMakeLists.txt index b8b663f016..cde619bcd3 100644 --- a/sql-odbc/src/CMakeLists.txt +++ b/sql-odbc/src/CMakeLists.txt @@ -25,6 +25,20 @@ project(global_make_list) include("${CMAKE_CURRENT_SOURCE_DIR}/modules/code-coverage.cmake") add_code_coverage_all_targets(EXCLUDE libraries aws-cpp-sdk googletest IntegrationTests) +if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(BITNESS 64) + set(BITNESS_NAME "x64") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../build/odbc/lib") + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../build/odbc/lib") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../build/odbc/bin") +elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) + set(BITNESS 32) + set(BITNESS_NAME "x86") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../build/odbc/lib") + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../build/odbc/lib") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../build/odbc/bin") +endif() + # This wasn't being picked up on mac, causes some symbol errors if(APPLE) set(CMAKE_CXX_STANDARD 20) @@ -47,20 +61,6 @@ else() add_compile_options(-Wall -Wextra -pedantic -Werror) endif() -if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(BITNESS 64) - set(BITNESS_NAME "x64") - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../lib64") - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../lib64") - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../bin64") -elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) - set(BITNESS 32) - set(BITNESS_NAME "x86") - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../lib32") - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../lib32") - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../bin32") -endif() - if(NOT WIN32) # Unix builds require autoconf option(AUTOCONF_ENABLE "Enable autoconf" ON) diff --git a/sql-odbc/src/installer/CMakeLists.txt b/sql-odbc/src/installer/CMakeLists.txt index 712653058c..0b47894b64 100644 --- a/sql-odbc/src/installer/CMakeLists.txt +++ b/sql-odbc/src/installer/CMakeLists.txt @@ -109,10 +109,11 @@ install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/Resources/odfe_sql_odbc.tdc" DESTINAT # Install AWS dependencies if(WIN32) - install(FILES "${PROJECT_ROOT}/sdk-build${BITNESS}/bin/Release/aws-c-common.dll" DESTINATION bin COMPONENT "Driver") - install(FILES "${PROJECT_ROOT}/sdk-build${BITNESS}/bin/Release/aws-c-event-stream.dll" DESTINATION bin COMPONENT "Driver") - install(FILES "${PROJECT_ROOT}/sdk-build${BITNESS}/bin/Release/aws-checksums.dll" DESTINATION bin COMPONENT "Driver") - install(FILES "${PROJECT_ROOT}/sdk-build${BITNESS}/bin/Release/aws-cpp-sdk-core.dll" DESTINATION bin COMPONENT "Driver") + set(AWS_SDK_BIN_DIR "${PROJECT_ROOT}/build/aws-sdk/install/bin") + install(FILES "${AWS_SDK_BIN_DIR}/aws-c-common.dll" DESTINATION bin COMPONENT "Driver") + install(FILES "${AWS_SDK_BIN_DIR}/aws-c-event-stream.dll" DESTINATION bin COMPONENT "Driver") + install(FILES "${AWS_SDK_BIN_DIR}/aws-checksums.dll" DESTINATION bin COMPONENT "Driver") + install(FILES "${AWS_SDK_BIN_DIR}/aws-cpp-sdk-core.dll" DESTINATION bin COMPONENT "Driver") endif() include(CPack)