Updated the build to use OpenSearch 1.0.0. Made a change to an integr…

…ation test to support OpenSearch. The build is currently using a Sonatype release Maven repository for AWS for the OpenSearch artifacts. Signed-off-by: David Venable <[email protected]>
asifsmohammed · Sep 1, 2021 · 17a104c · 17a104c
1 parent 5deaa48
commit 17a104c
Show file tree

Hide file tree

Showing 18 changed files with 48 additions and 97 deletions.
diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml
@@ -22,16 +22,6 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: ${{ matrix.java }}
-    # TODO: replace local built OpenSearch artifact with the public artifact
-    - name: Checkout OpenSearch
-      uses: actions/checkout@v2
-      with:
-        repository: 'opensearch-project/OpenSearch'
-        path: OpenSearch
-        ref: '1.0.0-alpha2'
-    - name: Build OpenSearch
-      working-directory: ./OpenSearch
-      run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=alpha2 -Dbuild.snapshot=false
     - name: Checkout Data-Prepper
       uses: actions/checkout@v2
     - name: Grant execute permission for gradlew

diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml
@@ -22,16 +22,6 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      # TODO: replace local built OpenSearch artifact with the public artifact
-      - name: Checkout OpenSearch
-        uses: actions/checkout@v2
-        with:
-          repository: 'opensearch-project/OpenSearch'
-          path: OpenSearch
-          ref: '1.0.0-alpha2'
-      - name: Build OpenSearch
-        working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=alpha2 -Dbuild.snapshot=false
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
       - name: Grant execute permission for gradlew

diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml
@@ -22,16 +22,6 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      # TODO: replace local built OpenSearch artifact with the public artifact
-      - name: Checkout OpenSearch
-        uses: actions/checkout@v2
-        with:
-          repository: 'opensearch-project/OpenSearch'
-          path: OpenSearch
-          ref: '1.0.0-alpha2'
-      - name: Build OpenSearch
-        working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=alpha2 -Dbuild.snapshot=false
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
       - name: Grant execute permission for gradlew

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -22,16 +22,6 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: ${{ matrix.java }}
-    # TODO: replace local built OpenSearch artifact with the public artifact
-    - name: Checkout OpenSearch
-      uses: actions/checkout@v2
-      with:
-        repository: 'opensearch-project/OpenSearch'
-        path: OpenSearch
-        ref: '1.0.0-alpha2'
-    - name: Build OpenSearch
-      working-directory: ./OpenSearch
-      run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=alpha2 -Dbuild.snapshot=false
     - name: Checkout Data-Prepper
       uses: actions/checkout@v2
     - name: Grant execute permission for gradlew

diff --git a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
@@ -22,16 +22,6 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      # TODO: replace local built OpenSearch artifact with the public artifact
-      - name: Checkout OpenSearch
-        uses: actions/checkout@v2
-        with:
-          repository: 'opensearch-project/OpenSearch'
-          path: OpenSearch
-          ref: '1.0.0-alpha2'
-      - name: Build OpenSearch
-        working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=alpha2 -Dbuild.snapshot=false
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
       - name: Grant execute permission for gradlew

diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
@@ -22,16 +22,6 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      # TODO: replace local built OpenSearch artifact with the public artifact
-      - name: Checkout OpenSearch
-        uses: actions/checkout@v2
-        with:
-          repository: 'opensearch-project/OpenSearch'
-          path: OpenSearch
-          ref: '1.0.0-alpha2'
-      - name: Build OpenSearch
-        working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=alpha2 -Dbuild.snapshot=false
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
       - name: Grant execute permission for gradlew

diff --git a/.github/workflows/opensearch-sink-os-integration-tests.yml b/.github/workflows/opensearch-sink-os-integration-tests.yml
@@ -18,25 +18,15 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      # TODO: replace local built OpenSearch artifact with the public artifact
-      - name: Checkout OpenSearch
-        uses: actions/checkout@v2
-        with:
-          repository: 'opensearch-project/OpenSearch'
-          path: OpenSearch
-          ref: '1.0.0-alpha2'
-      - name: Build OpenSearch
-        working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=alpha2 -Dbuild.snapshot=false
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
       - name: Grant execute permission for gradlew
         run: chmod +x gradlew
       - name: Run OpenSearch docker
         run: |
-          export version=1.0.0-beta1
-          docker pull opensearchstaging/opensearch:$version
-          docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d opensearchstaging/opensearch:$version
+          export version=1.0.0
+          docker pull opensearchproject/opensearch:$version
+          docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d opensearchproject/opensearch:$version
           sleep 90
       - name: Run OpenSearch tests
         run: |

diff --git a/build-resources.gradle b/build-resources.gradle
@@ -14,7 +14,7 @@ ext.versionMap = [
         junitJupiter : '5.7.2',
         mockito : '3.11.2',
         opentelemetryProto : '1.0.1-alpha',
-        opensearchVersion : '1.0.0-alpha2'
+        opensearchVersion : '1.0.0'
 ]
 
 ext.coreProjects = [project(':data-prepper-api'), project(':data-prepper-core'), project('data-prepper-plugins:common')]
diff --git a/build.gradle b/build.gradle
@@ -20,7 +20,7 @@ allprojects {
 
     repositories {
         mavenCentral()
-        maven { url "https://jitpack.io" }
+        maven { url 'https://jitpack.io' }
     }
 
     spotless {

diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle
@@ -43,8 +43,13 @@ configurations {
 }
 
 repositories {
-    // TODO: replace local built OpenSearch artifact with the public artifact
-    mavenLocal()
+    // TODO: Replace with Maven Central once it has synced
+    maven {
+        url 'https://aws.oss.sonatype.org/content/repositories/releases/'
+        content {
+            includeGroupByRegex 'org\\.opensearch.*'
+        }
+    }
 }
 
 dependencies {

diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle
@@ -6,9 +6,14 @@ buildscript {
     }
 
     repositories {
-        maven { url "https://plugins.gradle.org/m2/" }
-        // TODO: replace local built OpenSearch artifact with the public artifact
-        mavenLocal()
+        maven { url 'https://plugins.gradle.org/m2/' }
+        // TODO: Replace with Maven Central once it has synced
+        maven {
+            url 'https://aws.oss.sonatype.org/content/repositories/releases/'
+            content {
+                includeGroupByRegex 'org\\.opensearch.*'
+            }
+        }
     }
 
     dependencies {
@@ -41,8 +46,13 @@ ext {
 }
 
 repositories {
-    // TODO: replace local built OpenSearch artifact with the public artifact
-    mavenLocal()
+    // TODO: Replace with Maven Central once it has synced
+    maven {
+        url 'https://aws.oss.sonatype.org/content/repositories/releases/'
+        content {
+            includeGroupByRegex 'org\\.opensearch.*'
+        }
+    }
 }
 
 dependencies {

diff --git a/...search/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/...search/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java
@@ -474,8 +474,9 @@ private String getIndexPolicyId(final String index) throws IOException {
     final Response response = client().performRequest(request);
     final String responseBody = EntityUtils.toString(response.getEntity());
 
+    // TODO: This is a difference between OpenSearch and ODFE
     @SuppressWarnings("unchecked") final String policyId = (String) ((Map<String, Object>) createParser(XContentType.JSON.xContent(),
-            responseBody).map().get(index)).get("index.opendistro.index_state_management.policy_id");
+            responseBody).map().get(index)).get("index.plugins.index_state_management.policy_id");
     return policyId;
   }
 

diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle
@@ -18,8 +18,13 @@ ext {
 }
 
 repositories {
-    // TODO: replace local built OpenSearch artifact with the public artifact
-    mavenLocal()
+    // TODO: Replace with Maven Central once it has synced
+    maven {
+        url 'https://aws.oss.sonatype.org/content/repositories/releases/'
+        content {
+            includeGroupByRegex 'org\\.opensearch.*'
+        }
+    }
 }
 
 dependencies {

diff --git a/examples/dev/k8s/sample-project-applications.yaml b/examples/dev/k8s/sample-project-applications.yaml
@@ -37,7 +37,7 @@ spec:
           value: https://odfe:9200
         - name: OPENSEARCH_URL
           value: https://odfe:9200
-        image: opensearchstaging/opensearch-dashboards:1.0.0
+        image: opensearchproject/opensearch-dashboards:1.0.0
         imagePullPolicy: ""
         name: opensearch-dashboard
         ports:
@@ -135,7 +135,7 @@ spec:
       - env:
         - name: discovery.type
           value: single-node
-        image: opensearchstaging/opensearch:1.0.0
+        image: opensearchproject/opensearch:1.0.0
         imagePullPolicy: ""
         name: odfe
         ports:

diff --git a/examples/dev/trace-analytics-sample-app/Dockerfile b/examples/dev/trace-analytics-sample-app/Dockerfile
@@ -1,12 +1,7 @@
 FROM gradle:jdk14 AS builder
 COPY . /home/gradle/src
 WORKDIR /home/gradle/src
-# TODO: replace local built OpenSearch artifact with the public artifact
-RUN tar -xzf opensearch-build-1_0_0-alpha2.tar.gz
-RUN mkdir -p ~/.m2/repository/org
-RUN mv opensearch-build-1_0_0-alpha2 ~/.m2/repository/org/opensearch
-WORKDIR /home/gradle/src/data-prepper-core
-RUN gradle clean jar --daemon
+RUN gradle -p data-prepper-core clean jar --no-daemon
 
 FROM amazoncorretto:15-al2-full
 EXPOSE 21890

diff --git a/examples/dev/trace-analytics-sample-app/docker-compose.yml b/examples/dev/trace-analytics-sample-app/docker-compose.yml
@@ -76,7 +76,7 @@ services:
 
   opensearch:
     container_name: node-0.example.com
-    image: opensearchstaging/opensearch:1.0.0
+    image: opensearchproject/opensearch:1.0.0
     ports:
       - "9200:9200"
       - "9600:9600"
@@ -87,7 +87,7 @@ services:
 
   opensearch-dashboard:
     container_name: opensearch-dashboard
-    image: opensearchstaging/opensearch-dashboards:1.0.0
+    image: opensearchproject/opensearch-dashboards:1.0.0
     ports:
       - 5601:5601
     expose:

diff --git a/opensearch-build-1_0_0-alpha2.tar.gz b/opensearch-build-1_0_0-alpha2.tar.gz
diff --git a/research/zipkin-elastic-to-otel/build.gradle b/research/zipkin-elastic-to-otel/build.gradle
@@ -25,8 +25,13 @@ sourceCompatibility = 1.8
 
 repositories {
     mavenCentral()
-    // TODO: replace local built OpenSearch artifact with the public artifact
-    mavenLocal()
+    // TODO: Replace with Maven Central once it has synced
+    maven {
+        url 'https://aws.oss.sonatype.org/content/repositories/releases/'
+        content {
+            includeGroupByRegex 'org\\.opensearch.*'
+        }
+    }
 }
 
 application {