From 253ad43b273075bf91165f320bac3794a2446d4f Mon Sep 17 00:00:00 2001 From: Tan Luu Date: Tue, 25 Apr 2023 22:16:14 +0700 Subject: [PATCH 1/7] Disable zkClientEnsembleTracker --- Chart.yaml | 4 ++-- README.md | 2 +- configs/nifi.properties | 1 + doc/INSTALLATION.md | 2 +- templates/statefulset.yaml | 1 + tests/06-alpha.flow.xml | 6 +++--- values.yaml | 3 ++- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index 22cf8abc..ffea66f0 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -1,8 +1,8 @@ --- apiVersion: v2 name: nifi -version: 1.1.4 -appVersion: 1.16.3 +version: 1.1.5 +appVersion: 1.20.0 description: Apache NiFi is a software project from the Apache Software Foundation designed to automate the flow of data between software systems. keywords: - nifi diff --git a/README.md b/README.md index d5c51035..5023a77e 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ The following table lists the configurable parameters of the nifi chart and the | `replicaCount` | Number of nifi nodes | `1` | | **Image** | | `image.repository` | nifi Image name | `apache/nifi` | -| `image.tag` | nifi Image tag | `1.16.3` | +| `image.tag` | nifi Image tag | `1.20.0` | | `image.pullPolicy` | nifi Image pull policy | `IfNotPresent` | | `image.pullSecret` | nifi Image pull secret | `nil` | | **SecurityContext** | diff --git a/configs/nifi.properties b/configs/nifi.properties index d4cfd899..25d53e85 100644 --- a/configs/nifi.properties +++ b/configs/nifi.properties @@ -238,6 +238,7 @@ nifi.cluster.flow.election.max.wait.time=1 mins nifi.cluster.flow.election.max.candidates= # zookeeper properties, used for cluster management # +nifi.zookeeper.client.ensembleTracker={{.Values.properties.zkClientEnsembleTraker}} nifi.zookeeper.connect.string= nifi.zookeeper.connect.timeout=3 secs nifi.zookeeper.session.timeout=3 secs diff --git a/doc/INSTALLATION.md b/doc/INSTALLATION.md index 821ba2d4..2b03284e 100644 --- a/doc/INSTALLATION.md +++ b/doc/INSTALLATION.md @@ -16,7 +16,7 @@ helm dep up ``` 2. **Set a sensitiveKey** -In 1.16.3 version, Nifi needs a sensitiveKey to encrypt sensitive information. This key can be setted in the `values.yaml` file: +In 1.20.0 version, Nifi needs a sensitiveKey to encrypt sensitive information. This key can be setted in the `values.yaml` file: ```` properties: diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml index 7a732024..019ebcb6 100644 --- a/templates/statefulset.yaml +++ b/templates/statefulset.yaml @@ -191,6 +191,7 @@ spec: fi {{ end }} + prop_replace nifi.ui.banner.text $(hostname -s) prop_replace nifi.remote.input.host ${FQDN} prop_replace nifi.cluster.node.address ${FQDN} prop_replace nifi.zookeeper.connect.string ${NIFI_ZOOKEEPER_CONNECT_STRING} diff --git a/tests/06-alpha.flow.xml b/tests/06-alpha.flow.xml index f8b781f8..2f8c677b 100644 --- a/tests/06-alpha.flow.xml +++ b/tests/06-alpha.flow.xml @@ -24,7 +24,7 @@ org.apache.nifi nifi-standard-nar - 1.16.3 + 1.20.0 1 1 sec @@ -117,7 +117,7 @@ org.apache.nifi nifi-ssl-context-service-nar - 1.16.3 + 1.20.0 true @@ -162,7 +162,7 @@ org.apache.nifi nifi-site-to-site-reporting-nar - 1.16.3 + 1.20.0 5 sec RUNNING diff --git a/values.yaml b/values.yaml index 158bdfed..6ebace4f 100644 --- a/values.yaml +++ b/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 ## image: repository: apache/nifi - tag: "1.16.3" + tag: "1.20.0" pullPolicy: "IfNotPresent" ## Optionally specify an imagePullSecret. @@ -94,6 +94,7 @@ properties: httpsPort: 8443 webProxyHost: # : (If Nifi service is NodePort or LoadBalancer) clusterPort: 6007 + zkClientEnsembleTraker: false # https://issues.apache.org/jira/browse/NIFI-10481 provenanceStorage: "8 GB" provenanceMaxStorageTime: "10 days" siteToSite: From 9bf4698fc16ca6193d6056cb8409be24c19cfb6b Mon Sep 17 00:00:00 2001 From: Tan Luu Date: Mon, 1 May 2023 08:30:12 +0700 Subject: [PATCH 2/7] Update s2s bravo flow.xml --- tests/06-bravo.flow.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/06-bravo.flow.xml b/tests/06-bravo.flow.xml index 5e5a3ec3..832aa2d6 100644 --- a/tests/06-bravo.flow.xml +++ b/tests/06-bravo.flow.xml @@ -24,7 +24,7 @@ org.apache.nifi nifi-update-attribute-nar - 1.15.2 + 1.20.0 1 0 sec @@ -62,7 +62,7 @@ org.apache.nifi nifi-update-attribute-nar - 1.15.2 + 1.20.0 1 0 sec @@ -159,7 +159,7 @@ org.apache.nifi nifi-prometheus-nar - 1.15.2 + 1.20.0 5 sec RUNNING From 2a58c3dacb28357bb8c01fcdf816b5aec4636953 Mon Sep 17 00:00:00 2001 From: Tan Luu Date: Mon, 1 May 2023 10:05:00 +0700 Subject: [PATCH 3/7] Update s2s test to use cmctl from cert-manager instead of jetstack Signed-off-by: Tan Luu --- .github/workflows/test-site-to-site.yml | 2 +- tests/05-install-cert-manager.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-site-to-site.yml b/.github/workflows/test-site-to-site.yml index 31f082bf..0f36148b 100644 --- a/.github/workflows/test-site-to-site.yml +++ b/.github/workflows/test-site-to-site.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Minikube uses: manusa/actions-setup-minikube@v2.7.2 with: - minikube version: 'v1.28.0' + minikube version: 'v1.30.1' kubernetes version: 'v1.25.4' github token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout code diff --git a/tests/05-install-cert-manager.bash b/tests/05-install-cert-manager.bash index d30add6f..ddc2ef96 100755 --- a/tests/05-install-cert-manager.bash +++ b/tests/05-install-cert-manager.bash @@ -11,7 +11,7 @@ ARCH=$(go env GOARCH) /bin/rm -rf /tmp/cmctl-install mkdir -p /tmp/cmctl-install -curl -L -o /tmp/cmctl-install/cmctl.tar.gz https://github.com/jetstack/cert-manager/releases/latest/download/cmctl-$OS-$ARCH.tar.gz +curl -fsSL -o /tmp/cmctl-install/cmctl.tar.gz https://github.com/cert-manager/cert-manager/releases/latest/download/cmctl-$OS-$ARCH.tar.gz (cd /tmp/cmctl-install ; tar xvzf cmctl.tar.gz ; sudo mv cmctl /usr/local/bin) cmctl experimental install From f6a84e713700bf6d4aec5a98e9fc2862dc94c017 Mon Sep 17 00:00:00 2001 From: Tan Luu Date: Mon, 1 May 2023 10:18:10 +0700 Subject: [PATCH 4/7] Change cert-manager installation Signed-off-by: Tan Luu --- tests/05-install-cert-manager.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/05-install-cert-manager.bash b/tests/05-install-cert-manager.bash index ddc2ef96..24608098 100755 --- a/tests/05-install-cert-manager.bash +++ b/tests/05-install-cert-manager.bash @@ -14,4 +14,5 @@ mkdir -p /tmp/cmctl-install curl -fsSL -o /tmp/cmctl-install/cmctl.tar.gz https://github.com/cert-manager/cert-manager/releases/latest/download/cmctl-$OS-$ARCH.tar.gz (cd /tmp/cmctl-install ; tar xvzf cmctl.tar.gz ; sudo mv cmctl /usr/local/bin) -cmctl experimental install +kubectl create namespace cert-manager +cmctl experimental install -n cert-manager From 9d34bed9709dfab703538713a6aafa46a8d1f059 Mon Sep 17 00:00:00 2001 From: nathluu Date: Fri, 5 May 2023 00:02:41 +0700 Subject: [PATCH 5/7] Fix s2s test (# * Fix s2s test Signed-off-by: Tan Luu --- .github/workflows/test-site-to-site.yml | 9 ++++++--- tests/05-install-cert-manager.bash | 5 ++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-site-to-site.yml b/.github/workflows/test-site-to-site.yml index 0f36148b..041c1952 100644 --- a/.github/workflows/test-site-to-site.yml +++ b/.github/workflows/test-site-to-site.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Minikube uses: manusa/actions-setup-minikube@v2.7.2 with: - minikube version: 'v1.30.1' + minikube version: 'v1.28.0' kubernetes version: 'v1.25.4' github token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout code @@ -87,12 +87,15 @@ jobs: - name: Confirm certificate rotation run: | OLDCERTSTART=$(kubectl -n alpha exec nifi-0 -c server -- curl -kv https://localhost:8443 2>&1 | /bin/grep start) - echo Certificate "$OLDCERTSTART" + echo Old Certificate "$OLDCERTSTART" cmctl -n alpha renew nifi-0 + kubectl -n alpha rollout restart statefulset/nifi + kubectl -n alpha rollout status --watch statefulset/nifi --timeout=5m + sleep 5m for n in [ 0 1 2 3 4 5 6 7 8 9 ] do NEWCERTSTART=$(kubectl -n alpha exec nifi-0 -c server -- curl -kv https://localhost:8443 2>&1 | /bin/grep start) - echo Certificate "$NEWCERTSTART" + echo New Certificate "$NEWCERTSTART" if [ \""$OLDCERTSTART"\" != \""$NEWCERTSTART"\" ] then exit 0 diff --git a/tests/05-install-cert-manager.bash b/tests/05-install-cert-manager.bash index 24608098..d30add6f 100755 --- a/tests/05-install-cert-manager.bash +++ b/tests/05-install-cert-manager.bash @@ -11,8 +11,7 @@ ARCH=$(go env GOARCH) /bin/rm -rf /tmp/cmctl-install mkdir -p /tmp/cmctl-install -curl -fsSL -o /tmp/cmctl-install/cmctl.tar.gz https://github.com/cert-manager/cert-manager/releases/latest/download/cmctl-$OS-$ARCH.tar.gz +curl -L -o /tmp/cmctl-install/cmctl.tar.gz https://github.com/jetstack/cert-manager/releases/latest/download/cmctl-$OS-$ARCH.tar.gz (cd /tmp/cmctl-install ; tar xvzf cmctl.tar.gz ; sudo mv cmctl /usr/local/bin) -kubectl create namespace cert-manager -cmctl experimental install -n cert-manager +cmctl experimental install From eaed349a9d845f5aa1b229d135ae5008a22cc9b9 Mon Sep 17 00:00:00 2001 From: Tan Luu Date: Fri, 19 May 2023 18:59:13 +0700 Subject: [PATCH 6/7] Update s2s test Signed-off-by: Tan Luu --- .github/workflows/test-site-to-site.yml | 3 +++ tests/05-install-cert-manager.bash | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-site-to-site.yml b/.github/workflows/test-site-to-site.yml index 041c1952..bd8d8e21 100644 --- a/.github/workflows/test-site-to-site.yml +++ b/.github/workflows/test-site-to-site.yml @@ -89,9 +89,12 @@ jobs: OLDCERTSTART=$(kubectl -n alpha exec nifi-0 -c server -- curl -kv https://localhost:8443 2>&1 | /bin/grep start) echo Old Certificate "$OLDCERTSTART" cmctl -n alpha renew nifi-0 + # Remove below block of code when issue NIFI-11536 is fixed + ######################################################################### kubectl -n alpha rollout restart statefulset/nifi kubectl -n alpha rollout status --watch statefulset/nifi --timeout=5m sleep 5m + ######################################################################### for n in [ 0 1 2 3 4 5 6 7 8 9 ] do NEWCERTSTART=$(kubectl -n alpha exec nifi-0 -c server -- curl -kv https://localhost:8443 2>&1 | /bin/grep start) diff --git a/tests/05-install-cert-manager.bash b/tests/05-install-cert-manager.bash index d30add6f..d8e38d80 100755 --- a/tests/05-install-cert-manager.bash +++ b/tests/05-install-cert-manager.bash @@ -11,7 +11,7 @@ ARCH=$(go env GOARCH) /bin/rm -rf /tmp/cmctl-install mkdir -p /tmp/cmctl-install -curl -L -o /tmp/cmctl-install/cmctl.tar.gz https://github.com/jetstack/cert-manager/releases/latest/download/cmctl-$OS-$ARCH.tar.gz +curl -L -o /tmp/cmctl-install/cmctl.tar.gz https://github.com/cert-manager/cert-manager/releases/latest/download/cmctl-$OS-$ARCH.tar.gz (cd /tmp/cmctl-install ; tar xvzf cmctl.tar.gz ; sudo mv cmctl /usr/local/bin) cmctl experimental install From 219b17e883d0bfe2b768f2f66bc88fd7efac0f82 Mon Sep 17 00:00:00 2001 From: Sebastien Dupont Date: Mon, 2 Oct 2023 10:54:01 +0200 Subject: [PATCH 7/7] relaunch tests --- .github/workflows/test-site-to-site.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-site-to-site.yml b/.github/workflows/test-site-to-site.yml index bd8d8e21..dc0336f3 100644 --- a/.github/workflows/test-site-to-site.yml +++ b/.github/workflows/test-site-to-site.yml @@ -90,6 +90,7 @@ jobs: echo Old Certificate "$OLDCERTSTART" cmctl -n alpha renew nifi-0 # Remove below block of code when issue NIFI-11536 is fixed + # i.e. in version 1.23 https://issues.apache.org/jira/browse/NIFI-11536 ######################################################################### kubectl -n alpha rollout restart statefulset/nifi kubectl -n alpha rollout status --watch statefulset/nifi --timeout=5m