Merge pull request #230 from marklogic/develop

Release 1.27.0

.env

@@ -1,4 +1,4 @@
 # Defines environment variables for docker-compose.
 
 # The version of the connector.
-VERSION=1.24.2
+VERSION=1.27.0

CONTRIBUTING.md

@@ -140,7 +140,8 @@ application is deployed via the following steps:
 1. `cd test-app`
 2. If running a native MarkLogic install, verify that the admin password in the `gradle.properties` file is correct,
 overriding it in `gradle-local.properties` as needed.
-3. Run `./gradlew -i mldeploy`
+3. Run `./gradlew hubInit` to initialize the DHF project; a DHF project is needed in order to test the `RunFlowMarkLogic` processor.
+4. Run `./gradlew -i mldeploy`
 
 You can then run all tests in the project by returning to the parent directory and running `verify`:
 

Jenkinsfile

@@ -22,7 +22,7 @@ pipeline{
  export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
  export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
  cd nifi-connector/test-app
- echo mlPassword=admin >> gradle-local.properties
+ ./gradlew hubInit
  ./gradlew mlDeploy
  '''
  sh label:'test', script: '''#!/bin/bash

docker-compose.yml

@@ -4,7 +4,7 @@ name: marklogic_nifi
 services:
 
  marklogic:
- image: "marklogicdb/marklogic-db:11.2.0-centos-1.1.2"
+ image: "marklogicdb/marklogic-db:latest-11"
  container_name: "marklogic"
  platform: linux/amd64
  environment:
@@ -14,7 +14,7 @@ services:
  volumes:
  - ./docker/marklogic/logs:/var/opt/MarkLogic/Logs
  ports:
- - 8000-8006:8000-8006
+ - 8000-8014:8000-8014
 
  nifi:
  image: apache/nifi

docs/components/run-flow.md

@@ -10,8 +10,20 @@ The `RunFlowMarkLogic` processor supports running a
 a new FlowFile. The flow is expected to run non-ingestion steps, where data has already been ingested into MarkLogic 
 and needs to be reprocessed via a Data Hub flow. 
 
-As of the 1.16.3.3 release, this processor uses the Data Hub 5.7.2 Java API to run a flow. This may work with more 
-recent versions of Data Hub, but running flows in a Data Hub 5.7.2 installation is recommended. 
+In order to support running a Data Hub flow, the connector must depend on a particular version of the Data Hub Java 
+API. As of the 6.1.1 release of the Data Hub Java API, there is not a guarantee that the Data Hub Java API will work 
+with any release of Data Hub other than its own. It is therefore recommended to ensure that the version of Data Hub 
+you have installed in MarkLogic is the same as that of the MarkLogic NiFi connector release that you are using. 
+
+The table below lists the MarkLogic NiFi connector releases and the associated Data Hub Java API version:
+
+| Connector version | Data Hub Java API version |
+| --- | --- |
+| 1.27.0 | 6.1.1 |
+| 1.24.2 | 5.8.1 |
+| 1.16.3.1 | 5.7.2 |
+| 1.15.3.2 | 5.4.4 |
+| 1.9.1.7 | 5.4.2 |
 
 Please see the [Data Hub documentation for roles](https://docs.marklogic.com/datahub/5.8/security/users-and-roles.
 html) for which roles a MarkLogic user will need in order to run a flow.

nifi-marklogic-nar/pom.xml

@@ -22,7 +22,7 @@
  <parent>
  <groupId>org.apache.nifi</groupId>
  <artifactId>nifi-marklogic-bundle</artifactId>
- <version>1.24.2</version>
+ <version>1.27.0</version>
  </parent>
 
  <artifactId>nifi-marklogic-nar</artifactId>

nifi-marklogic-processors/flows-for-manual-testing.json

@@ -34,7 +34,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "ORIGINAL",
@@ -143,7 +143,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "FAILURE",
@@ -252,7 +252,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "RESULTS",
@@ -361,7 +361,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-marklogic-nar",
- "version": "1.24.2"
+ "version": "1.27.0"
  },
  "properties": {
  "Payload Source": "None",
@@ -602,7 +602,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "FAILURE!!!!",
@@ -711,7 +711,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "QUERY RESULT",
@@ -820,7 +820,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-marklogic-nar",
- "version": "1.24.2"
+ "version": "1.27.0"
  },
  "properties": {
  "DatabaseClient Service": "a9cc748c-c567-3828-888d-d615d5c9d10d",
@@ -1062,7 +1062,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "character-set": "UTF-8",
@@ -1148,7 +1148,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "AFTER:",
@@ -1259,7 +1259,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-marklogic-nar",
- "version": "1.24.2"
+ "version": "1.27.0"
  },
  "properties": {
  "URI Prefix": "/stuff/",
@@ -1530,7 +1530,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "RESULTS",
@@ -1641,7 +1641,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-marklogic-nar",
- "version": "1.24.2"
+ "version": "1.27.0"
  },
  "properties": {
  "Module Path": null,
@@ -1730,7 +1730,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "LOGERROR!!!",
@@ -1839,7 +1839,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "character-set": "UTF-8",
@@ -1925,7 +1925,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-standard-nar",
- "version": "1.24.0"
+ "version": "1.27.0"
  },
  "properties": {
  "Log prefix": "ORIGINAL",
@@ -2238,7 +2238,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-marklogic-nar",
- "version": "1.24.2"
+ "version": "1.27.0"
  },
  "properties": {
  "Base Path": null,
@@ -2332,7 +2332,7 @@
  "bundle": {
  "group": "org.apache.nifi",
  "artifact": "nifi-marklogic-services-api-nar",
- "version": "1.24.2"
+ "version": "1.27.0"
  }
  }
  ],

nifi-marklogic-processors/pom.xml

@@ -22,7 +22,7 @@
  <parent>
  <groupId>org.apache.nifi</groupId>
  <artifactId>nifi-marklogic-bundle</artifactId>
- <version>1.24.2</version>
+ <version>1.27.0</version>
  </parent>
 
  <artifactId>nifi-marklogic-processors</artifactId>
@@ -40,13 +40,6 @@
  <artifactId>nifi-utils</artifactId>
  <version>${nifi.version}</version>
  </dependency>
-
- <!-- Force usage of this version until DHF 5.8 is available -->
- <dependency>
- <groupId>com.marklogic</groupId>
- <artifactId>ml-app-deployer</artifactId>
- <version>4.8.0</version>
- </dependency>
  <dependency>
  <groupId>com.marklogic</groupId>
  <artifactId>marklogic-client-api</artifactId>
@@ -55,7 +48,7 @@
  <dependency>
  <groupId>com.marklogic</groupId>
  <artifactId>marklogic-data-hub</artifactId>
- <version>5.8.1</version>
+ <version>6.1.1</version>
  </dependency>
  <dependency>
  <groupId>org.apache.nifi</groupId>
@@ -68,7 +61,7 @@
  <dependency>
  <groupId>com.google.code.gson</groupId>
  <artifactId>gson</artifactId>
- <version>2.10.1</version>
+ <version>2.11.0</version>
  </dependency>
 
  <dependency>

nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/RunFlowMarkLogic.java

@@ -43,10 +43,10 @@
 
 @Tags({"MarkLogic", "Data Hub Framework"})
 @InputRequirement(InputRequirement.Requirement.INPUT_ALLOWED)
-@CapabilityDescription("Run a MarkLogic Data Hub 5 flow via the Data Hub 5.7.2 Java API. This is expected to be run on non-ingestion steps, where data " +
+@CapabilityDescription("Run a MarkLogic Data Hub 6.1.x flow via the Data Hub 6.1.1 Java API. This is expected to be run on non-ingestion steps, where data " +
  "has already been ingested into MarkLogic. Ingestion steps depend on access to local files, which isn't a common " +
  "use case for NiFi in production. Note that a given version of the Data Hub Java API is not yet guaranteed to work " +
- "with any other version of Data Hub. It is thus recommended to only use this against a Data Hub 5.7.2 installation, " +
+ "with any other version of Data Hub. It is thus recommended to only use this against a Data Hub 6.1.0 or 6.1.1 installation, " +
  "though it may work with other versions of Data Hub. Requires a MarkLogic user that is able to run the flow; " +
  "consult your Data Hub Framework documentation for guidelines on what Data Hub roles are required.")
 public class RunFlowMarkLogic extends AbstractMarkLogicProcessor {

nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java

@@ -72,6 +72,19 @@ public abstract class AbstractMarkLogicIT extends AbstractSpringMarkLogicTest {
 
  protected DataMovementManager dataMovementManager;
 
+ @Override
+ protected String getJavascriptForDeletingDocumentsBeforeTestRuns() {
+ return "declareUpdate(); " +
+ "cts.uris('', [], cts.notQuery(cts.collectionQuery([" +
+ "'hub-core-artifact', " +
+ "'http://marklogic.com/data-hub/flow', " +
+ "'http://marklogic.com/data-hub/step-definition', " +
+ "'http://marklogic.com/data-hub/steps'" +
+ "])))" +
+ ".toArray()" +
+ ".forEach(item => xdmp.documentDelete(item))";
+ }
+
  class IngestDoc {
  private Map<String, String> attributes;
  private String content;

nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/RunFlowMarkLogicIT.java

@@ -1,5 +1,9 @@
 package org.apache.nifi.marklogic.processor;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.JacksonHandle;
+import com.marklogic.client.io.StringHandle;
 import org.apache.nifi.util.MockFlowFile;
 import org.apache.nifi.util.TestRunner;
 import org.junit.jupiter.api.Test;
@@ -9,12 +13,30 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-/**
- * The test app does not yet have a DHF application available within it - it would really need to be converted into a
- * DHF application to make that feasible. For now, just verifying error handling in the processor.
- */
 public class RunFlowMarkLogicIT extends AbstractMarkLogicIT {
 
+ @Test
+ void realFlow() {
+ final String uri = "/a1.json";
+
+ getDatabaseClient().newJSONDocumentManager().write(uri,
+ new DocumentMetadataHandle()
+ .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE)
+ .withCollections("staging-data"),
+ new StringHandle("{}"));
+
+ TestRunner runner = getNewTestRunner(RunFlowMarkLogic.class);
+ runner.setProperty(RunFlowMarkLogic.FLOW_NAME, "testFlow");
+ runner.run();
+
+ List<MockFlowFile> files = runner.getFlowFilesForRelationship(RunFlowMarkLogic.FINISHED);
+ assertEquals(1, files.size());
+
+ JsonNode doc = getDatabaseClient().newJSONDocumentManager().read(uri, new JacksonHandle()).get();
+ assertTrue(doc.has("envelope"), "The custom step in the testFlow flow should have wrapped the document " +
+ "in an envelope. Flow response: " + files.get(0).getContent());
+ }
+
  @Test
  void flowDoesntExist() {
  TestRunner runner = newReaderTestRunner(RunFlowMarkLogic.class);
@@ -30,4 +52,5 @@ void flowDoesntExist() {
  "Expecting the processor to fail because the flow can't be found, since the test doesn't even run " +
  "against a DHF application; error message: " + errorMessage);
  }
+
 }

nifi-marklogic-services-api-nar/pom.xml

@@ -22,7 +22,7 @@
  <parent>
  <groupId>org.apache.nifi</groupId>
  <artifactId>nifi-marklogic-bundle</artifactId>
- <version>1.24.2</version>
+ <version>1.27.0</version>
  </parent>
 
  <artifactId>nifi-marklogic-services-api-nar</artifactId>

nifi-marklogic-services-api/pom.xml

@@ -22,7 +22,7 @@
  <parent>
  <groupId>org.apache.nifi</groupId>
  <artifactId>nifi-marklogic-bundle</artifactId>
- <version>1.24.2</version>
+ <version>1.27.0</version>
  </parent>
 
  <artifactId>nifi-marklogic-services-api</artifactId>

nifi-marklogic-services-nar/pom.xml

@@ -22,7 +22,7 @@
  <parent>
  <groupId>org.apache.nifi</groupId>
  <artifactId>nifi-marklogic-bundle</artifactId>
- <version>1.24.2</version>
+ <version>1.27.0</version>
  </parent>
 
  <artifactId>nifi-marklogic-services-nar</artifactId>

nifi-marklogic-services/pom.xml

@@ -19,7 +19,7 @@
  <parent>
  <groupId>org.apache.nifi</groupId>
  <artifactId>nifi-marklogic-bundle</artifactId>
- <version>1.24.2</version>
+ <version>1.27.0</version>
  </parent>
 
  <artifactId>nifi-marklogic-services</artifactId>