Merge integration & resolve conflicts

.github/workflows/build-accumulo.yml

@@ -0,0 +1,99 @@
+name: Build Accumulo snapshot and update DataWave to use
+
+on:
+  workflow_dispatch:
+    inputs:
+      accumuloBranch:
+        required: true
+        default: "2.1"
+        description: "Branch name to build. Will be used as image tag."
+      accumuloRepo:
+        required: true
+        default: "apache/accumulo"
+        description: "Accumulo Repo to use. Expected to be at Github. Example: apache/accumulo"
+      deployAccumulo:
+        required: true
+        default: "false"
+        description: "Set to false if this accumulo version has already been pushed to Github Packages"
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository.lowercase }}
+  ACCUMULO_JAVA_VERSION: '17'
+  DATAWAVE_JAVA_VERSION: '11'
+  JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8
+  USER_NAME: ${{ secrets.GHCR_WRITE_USER_NAME }}
+  ACCESS_TOKEN: ${{ secrets.GHCR_WRITE_ACCESS_TOKEN }}
+
+jobs:
+  build-and-deploy-accumulo:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      #
+    steps:
+    - name: Checkout DataWave
+      uses: actions/checkout@v4
+      with:
+        path: datawave
+
+    - name: Checkout Accumulo
+      uses: actions/checkout@v4
+      id: accumuloCheckout
+      with:
+        repository: ${{ github.event.inputs.accumuloRepo }}
+        path: accumulo
+        ref: ${{ github.event.inputs.accumuloBranch }}
+
+    - name: Set up JDK ${{env.ACCUMULO_JAVA_VERSION}}
+      uses: actions/setup-java@v4
+      with:
+        distribution: ${{env.JAVA_DISTRIBUTION}}
+        java-version: ${{env.ACCUMULO_JAVA_VERSION}}
+        cache: 'maven'
+    - run: echo "ACCUMULO_JAVA=$JAVA_HOME" >> $GITHUB_ENV
+    - name: Set up JDK ${{env.DATAWAVE_JAVA_VERSION}}
+      uses: actions/setup-java@v4
+      with:
+        distribution: ${{env.JAVA_DISTRIBUTION}}
+        java-version: ${{env.DATAWAVE_JAVA_VERSION}}
+        cache: 'maven'
+    - run: echo "DATAWAVE_JAVA=$JAVA_HOME" >> $GITHUB_ENV
+
+    - name: Get Accumulo Version
+      id: get-accumulo-version
+      run: |
+        export JAVA_HOME="$ACCUMULO_JAVA"
+        cd "$GITHUB_WORKSPACE/accumulo"
+        mvn build-helper:parse-version versions:set -DgenerateBackupPoms=false -DnewVersion=\${parsedVersion.majorVersion}.\${parsedVersion.minorVersion}.\${parsedVersion.incrementalVersion}-dwv-$(git rev-parse --short HEAD)
+        export newVersion=$(mvn -q help:evaluate -DforceStdout -Dexpression=project.version)
+        echo accumuloVersion=$newVersion >> $GITHUB_OUTPUT
+    - name: Deploy Accumulo
+      if: ${{ github.event.inputs.deployAccumulo == 'true'}}
+      run: | 
+        export JAVA_HOME="$ACCUMULO_JAVA"
+        cd "$GITHUB_WORKSPACE/accumulo"
+        mvn -DaltDeploymentRepository=github-datawave::https://maven.pkg.github.com/NationalSecurityAgency/datawave -V -B -e -ntp "-Dstyle.color=always" -DskipTests -T1C clean source:jar deploy -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml"
+    - name: Log in to the Container registry
+      uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ env.USER_NAME }}
+        password: ${{ env.ACCESS_TOKEN }}
+
+    - name: Update DataWave Dependency Version
+      run: |
+
+        export JAVA_HOME="$DATAWAVE_JAVA"
+        cd "$GITHUB_WORKSPACE/datawave"
+        mvn -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml" versions:set-property -Dproperty=version.accumulo -DnewVersion=${{ steps.get-accumulo-version.outputs.accumuloVersion }} -DgenerateBackupPoms=false
+    - name: Build Web and Ingest Docker Images (Maven)
+      run: |
+        export JAVA_HOME="$DATAWAVE_JAVA"
+        cd "$GITHUB_WORKSPACE/datawave"
+        mvn -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml" clean install -Prpm,kubernetes,assemble,deploy-ws -Ddist -Pdocker -DpushImage -Ddocker-release -DskipTests -Ddocker.image.accumulo.tag=${{ steps.get-accumulo-version.outputs.accumuloVersion }}
+
+ 

.github/workflows/build-images.yml

@@ -32,9 +32,9 @@ jobs:
         username: ${{ env.USER_NAME }}
         password: ${{ env.ACCESS_TOKEN }}
     - name: Checkout Code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up JDK ${{env.JAVA_VERSION}}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: ${{env.JAVA_DISTRIBUTION}}
         java-version: ${{env.JAVA_VERSION}}

.github/workflows/microservice-build-image.yaml

@@ -0,0 +1,41 @@
+#
+name: Create and publish a Docker image
+
+on:
+  workflow_call:
+    secrets:
+      USER_NAME:
+        description: "User Name for maven pulls"
+        required: true
+      ACCESS_TOKEN:
+        description: "Access token for maven pulls"
+        required: true
+
+
+jobs:
+  build-and-push-datawave-images:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Log in to the Container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ secrets.USER_NAME }}
+        password: ${{ secrets.ACCESS_TOKEN }}
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        distribution: "zulu"
+        java-version: 11
+        cache: 'maven'
+    - name: Build And Push Docker Image (Maven)
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+      run: |
+        mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e clean install -Pdocker,exec -Ddocker.image.prefix=ghcr.io/nationalsecurityagency/ -DpushImage
+
+ 

.github/workflows/microservice-maven-tests.yaml

@@ -0,0 +1,72 @@
+name: Tests
+
+on:
+  workflow_call:
+    secrets:
+      USER_NAME:
+        description: "User Name for maven pulls"
+        required: true
+      ACCESS_TOKEN:
+        description: "Access token for maven pulls"
+        required: true
+
+env:
+  MAVEN_OPTS: "-Djansi.force=true -Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true -XX:ThreadStackSize=1m"
+
+jobs:
+  # Runs the pom sorter and code formatter to ensure that the code
+  # is formatted and poms are sorted according to project rules. This
+  # will fail if the formatter makes any changes.
+  check-code-formatting:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        java-version: 11
+        distribution: 'zulu'
+    - uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-format-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-format-
+          ${{ runner.os }}-maven-
+    - name: Format code
+      run: |
+        mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e clean formatter:format sortpom:sort -Pautoformat
+        git status
+        git diff-index --quiet HEAD || (echo "Error! There are modified files after formatting." && false)
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+
+  # Build the code and run the unit/integration tests.
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        java-version: 11
+        distribution: 'zulu'
+    - uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-build-
+          ${{ runner.os }}-maven-format-
+          ${{ runner.os }}-maven-
+    - name: Build and Run Unit Tests
+      run: mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Ddist clean verify
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ on:
     - 'integration'
     - 'release/version*'
   pull_request:
-    paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE']
+    paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE', '.github/workflows/microservice*.yaml']
   workflow_dispatch:
 
 env:
@@ -23,9 +23,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up JDK ${{env.JAVA_VERSION}}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: ${{env.JAVA_DISTRIBUTION}}
         java-version: ${{env.JAVA_VERSION}}
@@ -60,9 +60,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up JDK ${{env.JAVA_VERSION}}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: ${{env.JAVA_DISTRIBUTION}}
         java-version: ${{env.JAVA_VERSION}}
@@ -88,15 +88,15 @@ jobs:
   #   runs-on: ubuntu-latest
   #   steps:
   #     - name: Checkout Code
-  #       uses: actions/checkout@v3
+  #       uses: actions/checkout@v4
   #       with:
   #         submodules: 'recursive'
   #     - name: Set up JDK ${{env.JAVA_VERSION}}
-  #       uses: actions/setup-java@v3
+  #       uses: actions/setup-java@v4
   #       with:
   #         distribution: ${{env.JAVA_DISTRIBUTION}}
   #         java-version: ${{env.JAVA_VERSION}}
-  #     - uses: actions/cache@v3
+  #     - uses: actions/cache@v4
   #       with:
   #         path: ~/.m2/repository
   #         key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
@@ -126,9 +126,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up JDK ${{env.JAVA_VERSION}}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: ${{env.JAVA_DISTRIBUTION}}
         java-version: ${{env.JAVA_VERSION}}
@@ -165,11 +165,11 @@ jobs:
           sudo rm -rf /usr/local/share/boost
           sudo rm -rf $AGENT_TOOLSDIRECTORY
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - name: Set up JDK ${{env.JAVA_VERSION}}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: ${{env.JAVA_DISTRIBUTION}}
           java-version: ${{env.JAVA_VERSION}}
@@ -234,15 +234,15 @@ jobs:
           sudo rm -rf /usr/local/share/boost
           sudo rm -rf $AGENT_TOOLSDIRECTORY
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - name: Set up JDK ${{env.JAVA_VERSION}}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: ${{env.JAVA_DISTRIBUTION}}
           java-version: ${{env.JAVA_VERSION}}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
@@ -256,10 +256,17 @@ jobs:
           USER_NAME: ${{ secrets.USER_NAME }}
           ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
         run: |
-          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install
+          # set some bogus URLs to trigger dependency download via maven
+          DIST_URLS="-Durl.zookeeper=https://bogus.apache.org/zookeeper/zookeeper-3.7.2/apache-zookeeper-3.7.2-bin.tar.gz.tar.gz \
+                     -Durl.accumulo=https://bogus.apache.org/accumulo/2.1.3/accumulo-2.1.3-bin.tar.gz \
+                     -Durl.wildfly=https://bogus.jboss.org/wildfly/17.0.1.Final/wildfly-17.0.1.Final.tar.gz \
+                     -Durl.hadoop=https://bogus.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz \
+                     -Durl.maven=https://bogus.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz"
+          
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean install
           # free up some space so that we don't run out
           docker system prune -f
-          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean
           cd docker
           ./bootstrap.sh
           attempt=0

.gitmodules

@@ -82,3 +82,9 @@
 [submodule "microservices/starters/cached-results"]
 	path = microservices/starters/cached-results
 	url = [email protected]:NationalSecurityAgency/datawave-spring-boot-starter-cached-results.git
+[submodule "microservices/services/map"]
+	path = microservices/services/map
+	url = [email protected]:NationalSecurityAgency/datawave-map-service.git
+[submodule "microservices/services/file-provider"]
+	path = microservices/services/file-provider
+	url = [email protected]:NationalSecurityAgency/datawave-file-provider-service.git

.mvn/maven-build-cache-config.xml

@@ -23,7 +23,7 @@
      -->
 
     <configuration>
-        <enabled>true</enabled>
+        <enabled>false</enabled>
         <hashAlgorithm>SHA-256</hashAlgorithm>
         <validateXml>true</validateXml>
         <local>

BUILDME.md

@@ -64,7 +64,7 @@ mvn -Pdev,assemble,rpm -Ddeploy -Dtar -Ddist -DskipTests clean install
 
 Datawave web services utilize several microservices at runtime (currently authorization and auditing, although that
 list will expand soon). Datawave depends on api modules for some of these services, and the dependencies are set in
-the parent pom (see `version.microservice.*` properties) to released versions. If you wish to build the microservices
+the parent pom (see `version.datawave.*` properties) to released versions. If you wish to build the microservices
 for some reason, you can simply add `-Dservices` to your maven build command.
 
 ### Releasing Microservices
@@ -95,7 +95,7 @@ the authorization service API version 1.0 is tagged with `svc_authorization-api_
 
 Note that simply building a new API or service release won't ensure that it is used anywhere. You will need to update
 build properties in either the datawave parent pom or within other service poms (for cross-service dependencies) to
-ensure that the new version is used. Look for properties starting with `version.microservice.` to see what to update.
+ensure that the new version is used. Look for properties starting with `version.datawave.` to see what to update.
 If you are updating an API module, you should be careful. In general, the associated service will need to be updated as
 well to support the API changes. The service should _add_ a new version of the API and continue to support the old
 version until it can be ensured that there are no more consumers of the old API.

common-test/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>7.7.0-SNAPSHOT</version>
+        <version>7.11.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-common-test</artifactId>
     <name>${project.artifactId}</name>