From d279464b6c6acda5b7cf7606447d7f8fc892c88a Mon Sep 17 00:00:00 2001 From: Muhammet Orazov Date: Thu, 21 Oct 2021 10:49:56 +0200 Subject: [PATCH] #67: Fixed compilation issues (#68) --- .github/workflows/ci-build.yml | 41 ++++ .travis.yml | 25 --- README.md | 7 +- doc/changes/changes_1.0.0.md | 2 +- exa-parquet-api/pom.xml | 32 ++- hadoop-etl-common/pom.xml | 69 ++++-- hadoop-etl-dist/pom.xml | 42 ++-- parquet-pre-1.7/pom.xml | 350 ++++++++++++++--------------- parquet/pom.xml | 387 +++++++++++++++++---------------- pom.xml | 41 ++-- 10 files changed, 538 insertions(+), 458 deletions(-) create mode 100644 .github/workflows/ci-build.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml new file mode 100644 index 0000000..ae75eaf --- /dev/null +++ b/.github/workflows/ci-build.yml @@ -0,0 +1,41 @@ +name: CI Build + +on: + - push + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Run tests and build with maven + run: mvn --batch-mode --update-snapshots clean verify + env: + GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + - name: Build Apache artifact + run: mvn clean -DskipTests package -P apache -Dhadoop.version=3.2.2 -Dhive.version=2.3.9 + - name: Build Cloudera artifact + run: mvn clean -DskipTests package -P cloudera -Dhadoop.version=2.6.0-cdh5.16.2 -Dhive.version=1.1.0-cdh5.16.2 + - name: Build Cloudera 6.x version artifact + run: mvn clean -DskipTests package -P cloudera6x -Dhadoop.version=3.0.0-cdh6.2.0 -Dhive.version=2.1.1-cdh6.2.0 + - name: Build Hortonworks artifact + run: mvn clean -DskipTests package -P hortonworks -Dhadoop.version=2.7.3.2.6.5.3004-13 -Dhive.version=2.1.0.2.6.5.3004-13 + - name: Build Mapr artifact + run: mvn clean -DskipTests package -P mapr -Dhadoop.version=2.7.0-mapr-1602 -Dhive.version=2.0.0-mapr-1605 + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6ead9aa..0000000 --- a/.travis.yml +++ /dev/null @@ -1,25 +0,0 @@ -language: java -dist: trusty - -jdk: - - oraclejdk8 - - openjdk8 - -env: - - MVN_ARGS= - - MVN_ARGS="-P apache -Dhadoop.version=2.8.5 -Dhive.version=2.3.4" - - MVN_ARGS="-P cloudera -Dhadoop.version=2.6.0-cdh5.16.0 -Dhive.version=1.1.0-cdh5.16.0" - - MVN_ARGS="-P hortonworks -Dhadoop.version=2.7.3.2.6.5.3004-13 -Dhive.version=2.1.0.2.6.5.3004-13" - - MVN_ARGS="-P mapr -Dhadoop.version=2.7.0-mapr-1602 -Dhive.version=2.0.0-mapr-1605 -DskipTests=true" - -# Builds an uber-jar with all dependencies. -script: "mvn $MVN_ARGS clean install" - -# Removes install outputs so that the cache is stable. -before_cache: - - rm -rf $HOME/.m2/repository/com/exasol - - find $HOME/.m2 -name resolver-status.properties -exec rm {} \; - -cache: - directories: - - $HOME/.m2 diff --git a/README.md b/README.md index f273783..d80eac1 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ # Hadoop ETL UDFs -[![Build Status](https://travis-ci.com/exasol/hadoop-etl-udfs.svg?branch=main)](https://travis-ci.com/exasol/hadoop-etl-udfs) - - -###### Please note that this is an open source project which is officially supported by Exasol. For any questions, you can contact our support team. Please note, however, that the EXPORT functionality is still in BETA mode. +[![Build Status](https://github.com/exasol/hadoop-etl-udfs/actions/workflows/ci-build.yml/badge.svg)](https://github.com/exasol/hadoop-etl-udfs/actions/workflows/ci-build.yml) +[![GitHub Release](https://img.shields.io/github/release/exasol/hadoop-etl-udfs.svg?logo=github)](https://github.com/exasol/hadoop-etl-udfs/releases/latest) ## Overview + Hadoop ETL UDFs are the main way to transfer data between Exasol and Hadoop (HCatalog tables on HDFS). The SQL syntax for calling the UDFs is similar to that of Exasol's native IMPORT and EXPORT commands, but with added UDF paramters for specifying the various necessary and optional Hadoop properties. A brief overview of features includes support for: diff --git a/doc/changes/changes_1.0.0.md b/doc/changes/changes_1.0.0.md index f7aeba5..e653e65 100644 --- a/doc/changes/changes_1.0.0.md +++ b/doc/changes/changes_1.0.0.md @@ -1,4 +1,4 @@ -# Hadoo ETL UDFs 1.0.0, released 2020-12-DD +# Hadoo ETL UDFs 1.0.0, released 2021-10-21 ## Bugfixes diff --git a/exa-parquet-api/pom.xml b/exa-parquet-api/pom.xml index 662d28d..6b9037a 100644 --- a/exa-parquet-api/pom.xml +++ b/exa-parquet-api/pom.xml @@ -1,16 +1,28 @@ - - 4.0.0 - - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + com.exasol exa-hadoop-etl-udfs - 1.0.0-SNAPSHOT + 1.0.0 + exa-parquet-api + + UTF-8 + 1.8 + - exa-parquet-api - - - + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + ${java.version} + ${java.version} + + + + diff --git a/hadoop-etl-common/pom.xml b/hadoop-etl-common/pom.xml index fbc8a4c..34c8a87 100644 --- a/hadoop-etl-common/pom.xml +++ b/hadoop-etl-common/pom.xml @@ -1,16 +1,12 @@ - - 4.0.0 - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 com.exasol exa-hadoop-etl-udfs - 1.0.0-SNAPSHOT + 1.0.0 - - hadoop-etl-common - + hadoop-etl-common UTF-8 1.8 @@ -45,7 +41,7 @@ com.exasol parquet - 1.0.0-SNAPSHOT + 1.0.0 @@ -58,7 +54,26 @@ com.exasol parquet-pre-1.7 - 1.0.0-SNAPSHOT + 1.0.0 + + + + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + + + cloudera6x + + false + + + + com.exasol + parquet + 1.0.0 @@ -77,18 +92,18 @@ com.exasol parquet - 1.0.0-SNAPSHOT + 1.0.0 hortonworks releases - http://repo.hortonworks.com/content/repositories/releases/ + https://repo.hortonworks.com/content/repositories/releases/ hortonworks jetty - http://repo.hortonworks.com/content/repositories/jetty-hadoop/ + https://repo.hortonworks.com/content/repositories/jetty-hadoop/ @@ -101,13 +116,13 @@ com.exasol parquet - 1.0.0-SNAPSHOT + 1.0.0 mapr-releases - http://repository.mapr.com/maven/ + https://repository.mapr.com/maven/ @@ -117,7 +132,7 @@ com.exasol exa-parquet-api - 1.0.0-SNAPSHOT + 1.0.0 com.exasol @@ -191,6 +206,16 @@ org.apache.hive hive-exec ${hive.version} + + + eigenbase + eigenbase-properties + + + org.pentaho + pentaho-aggdesigner-algorithm + + org.apache.hive @@ -201,15 +226,12 @@ org.apache.slider slider-core + + org.glassfish + javax.el + - @@ -225,5 +247,4 @@ - diff --git a/hadoop-etl-dist/pom.xml b/hadoop-etl-dist/pom.xml index 746f435..6334034 100644 --- a/hadoop-etl-dist/pom.xml +++ b/hadoop-etl-dist/pom.xml @@ -1,16 +1,12 @@ - - 4.0.0 - - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + com.exasol exa-hadoop-etl-udfs - 1.0.0-SNAPSHOT + 1.0.0 - - hadoop-etl-dist - + hadoop-etl-dist true @@ -25,7 +21,7 @@ com.exasol parquet - 1.0.0-SNAPSHOT + 1.0.0 @@ -38,7 +34,20 @@ com.exasol parquet-pre-1.7 - 1.0.0-SNAPSHOT + 1.0.0 + + + + + cloudera6x + + false + + + + com.exasol + parquet + 1.0.0 @@ -51,7 +60,7 @@ com.exasol parquet - 1.0.0-SNAPSHOT + 1.0.0 @@ -64,7 +73,7 @@ com.exasol parquet - 1.0.0-SNAPSHOT + 1.0.0 @@ -74,12 +83,12 @@ com.exasol hadoop-etl-common - 1.0.0-SNAPSHOT + 1.0.0 com.exasol exa-parquet-api - 1.0.0-SNAPSHOT + 1.0.0 @@ -131,7 +140,7 @@ - + @@ -139,5 +148,4 @@ - diff --git a/parquet-pre-1.7/pom.xml b/parquet-pre-1.7/pom.xml index 563603b..a17f510 100644 --- a/parquet-pre-1.7/pom.xml +++ b/parquet-pre-1.7/pom.xml @@ -1,183 +1,189 @@ - - 4.0.0 - - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + com.exasol exa-hadoop-etl-udfs - 1.0.0-SNAPSHOT + 1.0.0 + parquet-pre-1.7 + + UTF-8 + 1.8 + 2.8.5 + 2.3.4 + - parquet-pre-1.7 - - - UTF-8 - 1.8 - 2.8.5 - 2.3.4 - + + + maven.exasol.com + https://maven.exasol.com/artifactory/exasol-releases + + false + + + + maven.exasol.com-snapshots + https://maven.exasol.com/artifactory/exasol-snapshots + + true + + + - - - maven.exasol.com - https://maven.exasol.com/artifactory/exasol-releases - - false - - - - maven.exasol.com-snapshots - https://maven.exasol.com/artifactory/exasol-snapshots - - true - - - + + + apache + + true + + + + cloudera + + false + + + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + + + hortonworks + + false + + + + hortonworks releases + https://repo.hortonworks.com/content/repositories/releases/ + + + + hortonworks jetty + https://repo.hortonworks.com/content/repositories/jetty-hadoop/ + + + + - - - apache - - true - - - - cloudera - - false - - - - cloudera - https://repository.cloudera.com/artifactory/cloudera-repos/ - - - - - hortonworks - - false - - - - hortonworks releases - http://repo.hortonworks.com/content/repositories/releases/ - - - - hortonworks jetty - http://repo.hortonworks.com/content/repositories/jetty-hadoop/ - - - - - - - - com.exasol - exa-parquet-api - 1.0.0-SNAPSHOT - - - com.exasol - exasol-script-api - 6.0-SNAPSHOT - - - com.exasol - virtualschema-common - 0.0.1-SNAPSHOT - - - - org.glassfish - javax.json - 1.0.4 - - - junit - junit - 4.13.1 - test - - - org.mockito - mockito-core - 2.0.31-beta - - - com.google.guava - guava - [24.1.1,) - - - commons-io - commons-io - 2.7 - - - org.apache.hadoop - hadoop-annotations - ${hadoop.version} - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - - - - org.apache.hadoop - hadoop-mapreduce-client-common - ${hadoop.version} - - - org.apache.hive - hive-serde - ${hive.version} - - - org.apache.hive - hive-exec - ${hive.version} - - - org.apache.hive - hive-jdbc - ${hive.version} - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.2 - - ${java.version} - ${java.version} - - - - + + + org.glassfish + javax.json + 1.0.4 + + + junit + junit + 4.13.1 + test + + + org.mockito + mockito-core + 2.0.31-beta + + + com.google.guava + guava + [24.1.1,) + + + commons-io + commons-io + 2.7 + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + + org.apache.hadoop + hadoop-mapreduce-client-common + ${hadoop.version} + + + org.apache.hive + hive-serde + ${hive.version} + + + org.apache.hive + hive-exec + ${hive.version} + + + eigenbase + eigenbase-properties + + + org.pentaho + pentaho-aggdesigner-algorithm + + + + + org.apache.hive + hive-jdbc + ${hive.version} + + + org.glassfish + javax.el + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + ${java.version} + ${java.version} + + + + diff --git a/parquet/pom.xml b/parquet/pom.xml index 4eb5c59..458e6f4 100644 --- a/parquet/pom.xml +++ b/parquet/pom.xml @@ -1,201 +1,210 @@ - - 4.0.0 - - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + com.exasol exa-hadoop-etl-udfs - 1.0.0-SNAPSHOT + 1.0.0 + parquet + + UTF-8 + 1.8 + 2.8.5 + 2.3.4 + - parquet - - - UTF-8 - 1.8 - 2.8.5 - 2.3.4 - + + + maven.exasol.com + https://maven.exasol.com/artifactory/exasol-releases + + false + + + + maven.exasol.com-snapshots + https://maven.exasol.com/artifactory/exasol-snapshots + + true + + + - - - maven.exasol.com - https://maven.exasol.com/artifactory/exasol-releases - - false - - - - maven.exasol.com-snapshots - https://maven.exasol.com/artifactory/exasol-snapshots - - true - - - + + + apache + + true + + + + cloudera + + false + + + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + + + cloudera6x + + false + + + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + + + hortonworks + + false + + + + hortonworks releases + https://repo.hortonworks.com/content/repositories/releases/ + + + + hortonworks jetty + https://repo.hortonworks.com/content/repositories/jetty-hadoop/ + + + + + mapr + + false + + + + mapr-releases + https://repository.mapr.com/maven/ + + + + - - - apache - - true - - - - cloudera - - false - - - - cloudera - https://repository.cloudera.com/artifactory/cloudera-repos/ - - - - - hortonworks - - false - - - - hortonworks releases - http://repo.hortonworks.com/content/repositories/releases/ - - - - hortonworks jetty - http://repo.hortonworks.com/content/repositories/jetty-hadoop/ - - - - - mapr - - false - - - - mapr-releases - http://repository.mapr.com/maven/ - - - - - - - - com.exasol - exa-parquet-api - 1.0.0-SNAPSHOT - - - com.exasol - exasol-script-api - 6.0-SNAPSHOT - - - com.exasol - virtualschema-common - 0.0.1-SNAPSHOT - - - - org.glassfish - javax.json - 1.0.4 - - - junit - junit - 4.13.1 - test - - - org.mockito - mockito-core - 2.0.31-beta - - - com.google.guava - guava - [24.1.1,) - - - commons-io - commons-io - 2.7 - - - org.apache.hadoop - hadoop-annotations - ${hadoop.version} - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - - - - org.apache.hadoop - hadoop-mapreduce-client-common - ${hadoop.version} - - - org.apache.hive - hive-serde - ${hive.version} - - - org.apache.hive - hive-exec - ${hive.version} - - - org.apache.hive - hive-jdbc - ${hive.version} - - - org.apache.slider - slider-core - - - - + + org.glassfish + javax.json 1.0.4 - --> - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.2 - - ${java.version} - ${java.version} - - - - + + junit + junit + 4.13.1 + test + + + org.mockito + mockito-core + 2.0.31-beta + + + com.google.guava + guava + [24.1.1,) + + + commons-io + commons-io + 2.7 + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + + org.apache.hadoop + hadoop-mapreduce-client-common + ${hadoop.version} + + + org.apache.hive + hive-serde + ${hive.version} + + + org.apache.hive + hive-exec + ${hive.version} + + + eigenbase + eigenbase-properties + + + org.pentaho + pentaho-aggdesigner-algorithm + + + + + org.apache.hive + hive-jdbc + ${hive.version} + + + org.apache.slider + slider-core + + + org.glassfish + javax.el + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + ${java.version} + ${java.version} + + + + diff --git a/pom.xml b/pom.xml index cad3558..2f0f932 100644 --- a/pom.xml +++ b/pom.xml @@ -1,19 +1,17 @@ + - - 4.0.0 - pom - - com.exasol - exa-hadoop-etl-udfs - 1.0.0-SNAPSHOT - - - UTF-8 - 1.8 - 2.8.5 - 2.3.4 - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + pom + com.exasol + exa-hadoop-etl-udfs + 1.0.0 + + UTF-8 + 1.8 + 2.8.5 + 2.3.4 + @@ -58,6 +56,18 @@ hadoop-etl-dist + + cloudera6x + + false + + + hadoop-etl-common + exa-parquet-api + parquet + hadoop-etl-dist + + hortonworks @@ -83,5 +93,4 @@ -