Merge remote-tracking branch 'origin/integration' into 2365-consisten…

…t-translateId-name

.github/CODEOWNERS

@@ -1,7 +1,8 @@
-
 # In order to ensure the query microservices are consistent with the wildfly webservice, we need
 # to ensure that changes made to QueryExecutorBean match QueryManagementService, and changes made
 # to LookupUUIDUtil match LookupService in the Query Service.
-QueryExecutorBean.java @jwomeara
-LookupUUIDUtil.java @jwomeara
-RunningQuery.java @jwomeara
+QueryExecutorBean.java @jwomeara @ivakegg
+LookupUUIDUtil.java @jwomeara @ivakegg
+RunningQuery.java @jwomeara @ivakegg
+/core/ @jwomeara @ivakegg
+/warehouse/query-core/ @jwomeara @ivakegg

.github/workflows/tests.yml

@@ -31,6 +31,10 @@ jobs:
         java-version: ${{env.JAVA_VERSION}}
         maven-version: 3.9.5
         cache: 'maven'
+    - name: Extract branch name
+      shell: bash
+      run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+      id: extract_branch
     - name: Format code
       env:
         USER_NAME: ${{ secrets.USER_NAME }}
@@ -42,6 +46,7 @@ jobs:
     - name: Commit Changes
       run: |
         if [ "$diffs_found" = true ]; then
+          git checkout -b ${{ steps.extract_branch.outputs.branch }}
           git config --global user.name "GitHub Actions"
           git config --global user.email "[email protected]"
           git commit -am "Formatting job fix"
@@ -150,6 +155,75 @@ jobs:
         TAG=$(mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -q -N -Dmaven.build.cache.enabled=false -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec)
         contrib/datawave-quickstart/docker/docker-build.sh ${TAG} --docker-opts "${DOCKER_BUILD_OPTS}"
 
+  compose-build-and-test-latest-snapshots:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free up some space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf $AGENT_TOOLSDIRECTORY
+      - name: Checkout Code
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+      - name: Set up JDK ${{env.JAVA_VERSION}}
+        uses: actions/setup-java@v3
+        with:
+          distribution: ${{env.JAVA_DISTRIBUTION}}
+          java-version: ${{env.JAVA_VERSION}}
+          maven-version: 3.9.5
+          cache: 'maven'
+      # Builds the quickstart and microservice docker images and runs a query test
+      - name: Docker Compose Query Tests
+        env:
+          USER_NAME: ${{ secrets.USER_NAME }}
+          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+        run: |
+          # update datawave dependencies to use the latest snapshots
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e versions:update-properties versions:update-parent -DallowSnapshots=true -Dincludes=gov.nsa.*
+          
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install
+          
+          # free up some space so that we don't run out
+          docker system prune -f
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean
+          
+          cd docker
+          ./bootstrap.sh
+          
+          attempt=0
+          max_attempts=20
+          while [ $attempt -lt $max_attempts ]; do
+            attempt=$((attempt+1))
+          
+            echo "Starting docker compose (Attempt ${attempt}/${max_attempts})"
+            nohup docker compose up -d --no-recreate < /dev/null > compose.out 2>&1 &
+            sleep 60s
+            cat compose.out
+          
+            # check to see if the query service is running
+            QUERY="$(docker compose ps --status running --services | grep query || true)"
+          
+            if [ "$QUERY" == "query" ] ; then
+              echo "Docker compose started successfully"
+              break
+            elif [ $attempt -eq $max_attempts ] ; then
+              echo "Failed to start docker compose"
+              exit 1
+            fi
+          done
+          
+          cd scripts
+          ./testAll.sh
+
+      - name: Dump Logs
+        if: failure()
+        run: |
+          cd docker
+          docker compose logs
+
     # Here's an example of how you'd deploy the image to the github package registry.
     # We don't want to do this by default since packages on github cannot be deleted
     # or overwritten. So this could only be done for tags, however it seems the quickstart

common-test/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-common-test</artifactId>
     <name>${project.artifactId}</name>

contrib/datawave-quickstart/docker/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
         <relativePath>../../../pom.xml</relativePath>
     </parent>
     <artifactId>quickstart</artifactId>

core/cached-results/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-cached-results</artifactId>
     <name>${project.artifactId}</name>

core/common-util/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-common-util</artifactId>
     <name>${project.artifactId}</name>

core/common/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-common</artifactId>
     <name>${project.artifactId}</name>

core/connection-pool/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-connection-pool</artifactId>
     <name>${project.artifactId}</name>

core/map-reduce/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-map-reduce</artifactId>
     <name>${project.artifactId}</name>

core/modification/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-modification</artifactId>
     <name>${project.artifactId}</name>

core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <groupId>gov.nsa.datawave.core</groupId>
     <artifactId>datawave-core-parent</artifactId>

core/query/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-query</artifactId>
     <name>${project.artifactId}</name>

core/query/src/main/java/datawave/core/query/configuration/GenericQueryConfiguration.java

@@ -70,6 +70,8 @@ public class GenericQueryConfiguration implements Serializable {
     // use a value like 'env:PASS' to pull from the environment
     private String accumuloPassword = "";
 
+    private String connPoolName;
+
     // Whether or not this query emits every result or performs some kind of result reduction
     protected boolean reduceResults = false;
 
@@ -100,6 +102,7 @@ public GenericQueryConfiguration(GenericQueryConfiguration genericConfig) {
         this.setBaseIteratorPriority(genericConfig.getBaseIteratorPriority());
         this.setBypassAccumulo(genericConfig.getBypassAccumulo());
         this.setAccumuloPassword(genericConfig.getAccumuloPassword());
+        this.setConnPoolName(genericConfig.getConnPoolName());
         this.setAuthorizations(genericConfig.getAuthorizations());
         this.setBeginDate(genericConfig.getBeginDate());
         this.setClient(genericConfig.getClient());
@@ -278,6 +281,14 @@ public void setAccumuloPassword(String password) {
         this.accumuloPassword = EnvProvider.resolve(password);
     }
 
+    public String getConnPoolName() {
+        return connPoolName;
+    }
+
+    public void setConnPoolName(String connPoolName) {
+        this.connPoolName = connPoolName;
+    }
+
     public Map<String,ScannerBase.ConsistencyLevel> getTableConsistencyLevels() {
         return tableConsistencyLevels;
     }
@@ -336,12 +347,13 @@ && getBypassAccumulo() == that.getBypassAccumulo() && Objects.equals(getAuthoriz
                         && Objects.equals(getBeginDate(), that.getBeginDate()) && Objects.equals(getEndDate(), that.getEndDate())
                         && Objects.equals(getMaxWork(), that.getMaxWork()) && Objects.equals(getTableName(), that.getTableName())
                         && Objects.equals(getQueries(), that.getQueries()) && Objects.equals(getAccumuloPassword(), that.getAccumuloPassword())
-                        && Objects.equals(isReduceResults(), that.isReduceResults());
+                        && Objects.equals(getConnPoolName(), that.getConnPoolName()) && Objects.equals(isReduceResults(), that.isReduceResults());
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(isCheckpointable(), getAuthorizations(), getQuery(), getQueryString(), getBeginDate(), getEndDate(), getMaxWork(),
-                        getBaseIteratorPriority(), getTableName(), getQueries(), getBypassAccumulo(), getAccumuloPassword(), isReduceResults());
+                        getBaseIteratorPriority(), getTableName(), getQueries(), getBypassAccumulo(), getConnPoolName(), getAccumuloPassword(),
+                        isReduceResults());
     }
 }

core/query/src/main/java/datawave/core/query/logic/BaseQueryLogic.java

@@ -41,7 +41,6 @@ public abstract class BaseQueryLogic<T> implements QueryLogic<T> {
     private int maxPageSize = 0;
     private long pageByteTrigger = 0;
     private boolean collectQueryMetrics = true;
-    private String _connPoolName;
     private Set<String> authorizedDNs;
 
     protected ProxiedUserDetails currentUser;
@@ -79,7 +78,6 @@ public BaseQueryLogic(BaseQueryLogic<T> other) {
         setPageByteTrigger(other.getPageByteTrigger());
         setCollectQueryMetrics(other.getCollectQueryMetrics());
         this.authorizedDNs = other.authorizedDNs;
-        setConnPoolName(other.getConnPoolName());
         setRequiredRoles(other.getRequiredRoles());
         setSelectorExtractor(other.getSelectorExtractor());
         setCurrentUser(other.getCurrentUser());
@@ -330,13 +328,13 @@ public void setCollectQueryMetrics(boolean collectQueryMetrics) {
     /** {@inheritDoc} */
     @Override
     public String getConnPoolName() {
-        return _connPoolName;
+        return getConfig().getConnPoolName();
     }
 
     /** {@inheritDoc} */
     @Override
     public void setConnPoolName(final String connPoolName) {
-        _connPoolName = connPoolName;
+        getConfig().setConnPoolName(connPoolName);
     }
 
     /** {@inheritDoc} */

core/utils/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.2.0-SNAPSHOT</version>
+        <version>7.4.0-SNAPSHOT</version>
     </parent>
     <groupId>gov.nsa.datawave.core</groupId>
     <artifactId>datawave-utils-parent</artifactId>

docker/docker-compose.yml

@@ -82,6 +82,8 @@ services:
 
   # When auto.create.topics.enable is true, this causes deleted topics to be recreated at random.  So, leave it disabled.
   kafka:
+    profiles:
+      - kafka
     image: docker.io/bitnami/kafka:3.2
     ports:
       - "9094:9094"
@@ -102,8 +104,7 @@ services:
 
   kafdrop:
     profiles:
-      - management
-      - full
+      - kafka
     image: docker.io/obsidiandynamics/kafdrop
     ports:
       - "8999:9000"
@@ -365,7 +366,7 @@ services:
       - AUDIT_SERVER_URL=http://audit:8080/audit
       - HADOOP_HOST=${DW_HADOOP_HOST}
       - HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
-      - "BACKEND=${BACKEND:-kafka}"
+      - "BACKEND=${BACKEND:-rabbitmq}"
       - CACHED_RESULTS=${CACHED_RESULTS:-false}
     ports:
       - "8080:8080"
@@ -390,8 +391,6 @@ services:
         condition: service_healthy
       metrics:
         condition: service_healthy
-      kafka:
-        condition: service_started
       executor-pool1:
         condition: service_started
 
@@ -430,8 +429,6 @@ services:
         condition: service_healthy
       metrics:
         condition: service_healthy
-      kafka:
-        condition: service_started
       executor-pool1:
         condition: service_started
 
@@ -448,7 +445,7 @@ services:
     environment:
       - ZOOKEEPER_HOST=${DW_ZOOKEEPER_HOST}
       - HADOOP_HOST=${DW_HADOOP_HOST}
-      - BACKEND=${BACKEND:-kafka}
+      - BACKEND=${BACKEND:-rabbitmq}
       - HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
     # This mapping is required to enable the metrics service to communicate
     # with host-deployed services like hadoop, zookeeper, and accumulo.
@@ -479,8 +476,6 @@ services:
         condition: service_healthy
       metrics:
         condition: service_healthy
-      kafka:
-        condition: service_started
 
   executor-pool2:
     profiles:
@@ -497,7 +492,7 @@ services:
     environment:
       - ZOOKEEPER_HOST=${DW_ZOOKEEPER_HOST}
       - HADOOP_HOST=${DW_HADOOP_HOST}
-      - BACKEND=${BACKEND:-kafka}
+      - BACKEND=${BACKEND:-rabbitmq}
       - HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
     # This mapping is required to enable the metrics service to communicate
     # with host-deployed services like hadoop, zookeeper, and accumulo.