From b8b1fa01792b30ea52d9d2bb79952b984925f298 Mon Sep 17 00:00:00 2001 From: rhkp <87712456+rhkp@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:56:29 -0400 Subject: [PATCH] Add sonataflow postgres db migrator tool for data index and jobs service (#517) --- .github/workflows/db-migrator-tests.yml | 55 ++++ .gitignore | 3 +- Makefile | 4 +- hack/bump-version.sh | 14 + .../sonataflow-db-migrator/.dockerignore | 5 + .../tools/sonataflow-db-migrator/.gitignore | 44 +++ images/tools/sonataflow-db-migrator/README.md | 56 ++++ .../build-container-image.sh | 78 +++++ images/tools/sonataflow-db-migrator/mvnw | 308 ++++++++++++++++++ images/tools/sonataflow-db-migrator/mvnw.cmd | 205 ++++++++++++ images/tools/sonataflow-db-migrator/pom.xml | 186 +++++++++++ .../src/main/cekit/image.yaml | 45 +++ .../kogito-postgres-db-migration-deps/install | 32 ++ .../migration.sh | 36 ++ .../module.yaml | 30 ++ .../migrator/DBConnectionChecker.java | 66 ++++ .../postgresql/migrator/DBMigrator.java | 73 +++++ .../postgresql/migrator/MigrationService.java | 66 ++++ .../src/main/resources/application.properties | 40 +++ .../migrator/DBConnectionCheckerTest.java | 52 +++ .../postgresql/migrator/DBMigratorTest.java | 78 +++++ .../migrator/MigrationServiceTest.java | 53 +++ test/e2e/clusterplatform_test.go | 4 +- test/e2e/helpers.go | 10 + test/e2e/platform_test.go | 12 +- test/e2e/workflow_test.go | 8 +- 26 files changed, 1545 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/db-migrator-tests.yml create mode 100644 images/tools/sonataflow-db-migrator/.dockerignore create mode 100644 images/tools/sonataflow-db-migrator/.gitignore create mode 100644 images/tools/sonataflow-db-migrator/README.md create mode 100755 images/tools/sonataflow-db-migrator/build-container-image.sh create mode 100755 images/tools/sonataflow-db-migrator/mvnw create mode 100644 images/tools/sonataflow-db-migrator/mvnw.cmd create mode 100644 images/tools/sonataflow-db-migrator/pom.xml create mode 100644 images/tools/sonataflow-db-migrator/src/main/cekit/image.yaml create mode 100644 images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/install create mode 100644 images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh create mode 100644 images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml create mode 100644 images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/DBConnectionChecker.java create mode 100644 images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/DBMigrator.java create mode 100644 images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java create mode 100644 images/tools/sonataflow-db-migrator/src/main/resources/application.properties create mode 100644 images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/DBConnectionCheckerTest.java create mode 100644 images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/DBMigratorTest.java create mode 100644 images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java diff --git a/.github/workflows/db-migrator-tests.yml b/.github/workflows/db-migrator-tests.yml new file mode 100644 index 000000000..a60b3c44f --- /dev/null +++ b/.github/workflows/db-migrator-tests.yml @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Sonataflow DB Migrator Unit Tests + +on: + pull_request: + types: [opened, reopened, edited, synchronize] + branches: + - main + paths: + - 'images/tools/sonataflow-db-migrator/**' + - '!images/tools/sonataflow-db-migrator/README.md' + - '.github/workflows/db-migrator-tests.yml' + +jobs: + unit-tests: + strategy: + matrix: + OS: [ubuntu-latest] + JAVA_VERSION: [17] + MAVEN_VERSION: [3.9.8] + timeout-minutes: 10 + name: Unit Tests + runs-on: ${{ matrix.OS }} + steps: + - name: Java and Maven Setup + uses: apache/incubator-kie-kogito-pipelines/.ci/actions/maven@main + with: + java-version: ${{ matrix.JAVA_VERSION }} + maven-version: ${{ matrix.MAVEN_VERSION }} + cache-key-prefix: ${{ matrix.OS }}-${{ matrix.JAVA_VERSION }}-maven${{ matrix.MAVEN_VERSION }} + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: | + cd images/tools/sonataflow-db-migrator + mvn test \ No newline at end of file diff --git a/.gitignore b/.gitignore index d56122269..d92a4294b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ Dockerfile /target/ database/index.db -e2e-test-report.xml +e2e-test-report*.xml +*.tar \ No newline at end of file diff --git a/Makefile b/Makefile index ddb2ff8b3..1401f1a50 100644 --- a/Makefile +++ b/Makefile @@ -338,7 +338,9 @@ generate-all: generate generate-deploy bundle addheaders vet fmt .PHONY: test-e2e # You will need to have a Minikube/Kind cluster up in running to run this target, and run container-builder before the test test-e2e: - go test ./test/e2e/* -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report.xml -timeout 120m + go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/clusterplatform_test.go -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report-clusterplatform_test.xml -timeout 60m; + go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/platform_test.go -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report-platform_test.xml -timeout 60m; + go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/workflow_test.go -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report-workflow_test.xml -timeout 60m; .PHONY: before-pr before-pr: test generate-all diff --git a/hack/bump-version.sh b/hack/bump-version.sh index 432a1872b..a884e88a8 100755 --- a/hack/bump-version.sh +++ b/hack/bump-version.sh @@ -23,13 +23,20 @@ source "${script_dir_path}"/env.sh imageTag='docker.io/apache/incubator-kie-sonataflow-operator' # shellcheck disable=SC2034 old_version=$(getImageTagVersion) +old_operator_version=$(getOperatorVersion) new_version=$1 +quarkus_version=$2 if [ -z "${new_version}" ]; then echo "Please inform the new version" exit 1 fi +if [ -z "${quarkus_version}" ]; then + quarkus_version="3.13.0" + echo "Warning: The Quarkus version is not supplied, defaulting to version: $quarkus_version" +fi + oldMajorMinorVersion=${old_version%.*} newMajorMinorVersion=${new_version%.*} @@ -48,6 +55,13 @@ sed -i -r "s|operatorVersion =.*|operatorVersion = \"${new_version}\"|g" version sed -i "s|containerImage:.*|containerImage: ${imageTag}:${newMajorMinorVersion}|g" $(getCsvFile) +# Begin: Sonataflow DB Migrator tool +sed -i "s|OPERATOR_VERSION=${old_operator_version}|OPERATOR_VERSION=${new_version}|g" images/tools/sonataflow-db-migrator/build-container-image.sh +sed -i "s|OPERATOR_VERSION=${old_operator_version}|OPERATOR_VERSION=${new_version}|g" images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh +sed -i "s|${old_operator_version}|${new_version}|g" images/tools/sonataflow-db-migrator/pom.xml +sed -i "s|.*|${quarkus_version}|g" images/tools/sonataflow-db-migrator/pom.xml +# End: Sonataflow DB Migrator tool + make generate-all make vet diff --git a/images/tools/sonataflow-db-migrator/.dockerignore b/images/tools/sonataflow-db-migrator/.dockerignore new file mode 100644 index 000000000..94810d006 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/.dockerignore @@ -0,0 +1,5 @@ +* +!target/*-runner +!target/*-runner.jar +!target/lib/* +!target/quarkus-app/* \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/.gitignore b/images/tools/sonataflow-db-migrator/.gitignore new file mode 100644 index 000000000..33b277154 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/.gitignore @@ -0,0 +1,44 @@ +#Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +release.properties +.flattened-pom.xml + +# Eclipse +.project +.classpath +.settings/ +bin/ + +# IntelliJ +.idea +*.ipr +*.iml +*.iws + +# NetBeans +nb-configuration.xml + +# Visual Studio Code +.vscode +.factorypath + +# OSX +.DS_Store + +# Vim +*.swp +*.swo + +# patch +*.orig +*.rej + +# Local environment +.env + +# Plugin directory +/.quarkus/cli/plugins/ +*.iml \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/README.md b/images/tools/sonataflow-db-migrator/README.md new file mode 100644 index 000000000..8fdaa0697 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/README.md @@ -0,0 +1,56 @@ +# sonataflow-db-migrator + +This is a quarkus postgres database migrator application for Sonataflow Data Index and Jobs Service applications. + +## Running the application in dev mode +Though you can run the application locally in dev mode but it is advisable to run this application as a container image as described in the next section. +The primary reason not to run as standalone application in dev mode, is that by default there are no DDL migration files included in the source. +However the DDL files are dynamically included from its respective sources when a container image is created. + +You can run your application in dev mode that enables live coding using: + +```shell script +./mvnw compile quarkus:dev +``` + +## Build and Run container image locally +You can build the cekit container image by using the provided image builder shell script +```shell +./build-container-image.sh +``` +Ensure the script completes without errors. +If you may have a cekit specific Python virtual environment, be sure to activate it, so the script can find the cekit command. +```shell +virtualenv ~/cekit +source ~/cekit/bin/activate; +``` + +Assuming you have a Postgres database running locally, e.g., a `di` database for data index and a `js` database for jobs service, you can run the image with the following command. Substitute appropriate values: +```shell +podman run \ +--env MIGRATE_DB_DATAINDEX=true \ +--env QUARKUS_DATASOURCE_DATAINDEX_JDBC_URL= \ +--env QUARKUS_DATASOURCE_DATAINDEX_USERNAME= \ +--env QUARKUS_DATASOURCE_DATAINDEX_PASSWORD= \ +--env QUARKUS_FLYWAY_DATAINDEX_SCHEMAS=dataindex \ +--env MIGRATE_DB_JOBSSERVICE=true \ +--env QUARKUS_DATASOURCE_JOBSSERVICE_JDBC_URL= \ +--env QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME= \ +--env QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD= \ +--env QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS=jobsservice \ +docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql:999-SNAPSHOT +``` + +### Environment variables +| NAME | DESCRIPTION | DEFAULT | +|---|---|---| +| MIGRATE_DB_DATAINDEX | Set to true if you want to migrate data index database, set to false otherwise | false | +| QUARKUS_DATASOURCE_DATAINDEX_JDBC_URL | Data index database url e.g. jdbc:postgresql://host.docker.internal:5432/di| jdbc:postgresql://localhost:5432/postgres | +| QUARKUS_DATASOURCE_DATAINDEX_USERNAME | Data index database username| postgres | +| QUARKUS_DATASOURCE_DATAINDEX_PASSWORD | Data index database password| postgres | +| QUARKUS_FLYWAY_DATAINDEX_SCHEMAS | Data index database schema| dataindex | +| MIGRATE_DB_JOBSSERVICE | Set to true if you want to migrate jobs service database, set to false otherwise | false | +| QUARKUS_DATASOURCE_JOBSSERVICE_JDBC_URL | Jobs service database url e.g. jdbc:postgresql://host.docker.internal:5432/js| jdbc:postgresql://localhost:5432/postgres | +| QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME | Jobs service database username| postgres | +| QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD | Jobs service database password| postgres | +| QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS | Jobs service database schema| jobsservice | \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/build-container-image.sh b/images/tools/sonataflow-db-migrator/build-container-image.sh new file mode 100755 index 000000000..53a2fbdd9 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/build-container-image.sh @@ -0,0 +1,78 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +# cleanup temporary files +cleanup () { + echo "Cleaning up" + rm -rf target + rm -rf src/main/resources/postgresql + rm -rf tmp + rm -f src/main/cekit/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar +} + +# Script variables with default values. These values will be updated by hack/bump-version.sh, don't change it. +OPERATOR_VERSION=999.0.0 # comes from version.go +DDL_VERSION=10.0.999-SNAPSHOT +DDL_FILE=kogito-ddl-10.0.999-20240806.011718-23-db-scripts.zip +DDL_BASE_URL=https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl +CEKIT_BUILDER=podman + +# Use specific variable values, if passed +for arg in "$@"; do + case "$arg" in + OPERATOR_VERSION=*) OPERATOR_VERSION="${arg#*=}" ;; + DDL_VERSION=*) DDL_VERSION="${arg#*=}" ;; + DDL_FILE=*) DDL_FILE="${arg#*=}" ;; + DDL_BASE_URL=*) DDL_BASE_URL="${arg#*=}" ;; + CEKIT_BUILDER=*) CEKIT_BUILDER="${arg#*=}" ;; + esac +done + +DDL_URL=$DDL_BASE_URL/$DDL_VERSION/$DDL_FILE + +printf "Variables being used for the image: \n OPERATOR_VERSION: %s, \n DDL_VERSION: %s, \n DDL_FILE: %s, \n DDL_BASE_URL: %s, \n CEKIT_BUILDER=%s\n\n" "$OPERATOR_VERSION" "$DDL_VERSION" "$DDL_FILE" "$DDL_BASE_URL" "$CEKIT_BUILDER" + +# Start with cleanup +cleanup "$OPERATOR_VERSION" + +# Get Data Index/ Jobs Service DDL Files +mkdir -p tmp +# Change the variables below, as needed +wget "$DDL_URL" +mv "$DDL_FILE" tmp +cd tmp || exit +unzip "$DDL_FILE" +mv ./postgresql ../src/main/resources +cd .. || exit + +# Create an Uber jar +mvn package -Dquarkus.package.jar.type=uber-jar +cp target/sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar src/main/cekit/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar + +# Build the container image +cd src/main/cekit || exit +cekit -v build "$CEKIT_BUILDER" + +# Cleanup +cd ../../.. || exit +cleanup "$OPERATOR_VERSION" \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/mvnw b/images/tools/sonataflow-db-migrator/mvnw new file mode 100755 index 000000000..8d937f4c1 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/mvnw @@ -0,0 +1,308 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.2.0 +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "$(uname)" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME + else + JAVA_HOME="/Library/Java/Home"; export JAVA_HOME + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --unix "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --unix "$CLASSPATH") +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && + JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="$(which javac)" + if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=$(which readlink) + if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then + if $darwin ; then + javaHome="$(dirname "\"$javaExecutable\"")" + javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" + else + javaExecutable="$(readlink -f "\"$javaExecutable\"")" + fi + javaHome="$(dirname "\"$javaExecutable\"")" + javaHome=$(expr "$javaHome" : '\(.*\)/bin') + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=$(cd "$wdir/.." || exit 1; pwd) + fi + # end of workaround + done + printf '%s' "$(cd "$basedir" || exit 1; pwd)" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + # Remove \r in case we run on Windows within Git Bash + # and check out the repository with auto CRLF management + # enabled. Otherwise, we may read lines that are delimited with + # \r\n and produce $'-Xarg\r' rather than -Xarg due to word + # splitting rules. + tr -s '\r\n' ' ' < "$1" + fi +} + +log() { + if [ "$MVNW_VERBOSE" = true ]; then + printf '%s\n' "$1" + fi +} + +BASE_DIR=$(find_maven_basedir "$(dirname "$0")") +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR +log "$MAVEN_PROJECTBASEDIR" + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" +if [ -r "$wrapperJarPath" ]; then + log "Found $wrapperJarPath" +else + log "Couldn't find $wrapperJarPath, downloading it ..." + + if [ -n "$MVNW_REPOURL" ]; then + wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + else + wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + fi + while IFS="=" read -r key value; do + # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) + safeValue=$(echo "$value" | tr -d '\r') + case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; + esac + done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" + log "Downloading from: $wrapperUrl" + + if $cygwin; then + wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") + fi + + if command -v wget > /dev/null; then + log "Found wget ... using wget" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + log "Found curl ... using curl" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + else + curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + fi + else + log "Falling back to using Java to download" + javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" + javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaSource=$(cygpath --path --windows "$javaSource") + javaClass=$(cygpath --path --windows "$javaClass") + fi + if [ -e "$javaSource" ]; then + if [ ! -e "$javaClass" ]; then + log " - Compiling MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/javac" "$javaSource") + fi + if [ -e "$javaClass" ]; then + log " - Running MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +# If specified, validate the SHA-256 sum of the Maven wrapper jar file +wrapperSha256Sum="" +while IFS="=" read -r key value; do + case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; + esac +done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" +if [ -n "$wrapperSha256Sum" ]; then + wrapperSha256Result=false + if command -v sha256sum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + elif command -v shasum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." + echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + exit 1 + fi + if [ $wrapperSha256Result = false ]; then + echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 + echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 + echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + exit 1 + fi +fi + +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --windows "$CLASSPATH") + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# shellcheck disable=SC2086 # safe args +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/images/tools/sonataflow-db-migrator/mvnw.cmd b/images/tools/sonataflow-db-migrator/mvnw.cmd new file mode 100644 index 000000000..c4586b564 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/mvnw.cmd @@ -0,0 +1,205 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.2.0 +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* +if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %WRAPPER_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file +SET WRAPPER_SHA_256_SUM="" +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B +) +IF NOT %WRAPPER_SHA_256_SUM%=="" ( + powershell -Command "&{"^ + "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ + "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ + " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ + " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ + " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ + " exit 1;"^ + "}"^ + "}" + if ERRORLEVEL 1 goto error +) + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% ^ + %JVM_CONFIG_MAVEN_PROPS% ^ + %MAVEN_OPTS% ^ + %MAVEN_DEBUG_OPTS% ^ + -classpath %WRAPPER_JAR% ^ + "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ + %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" +if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%"=="on" pause + +if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% + +cmd /C exit /B %ERROR_CODE% diff --git a/images/tools/sonataflow-db-migrator/pom.xml b/images/tools/sonataflow-db-migrator/pom.xml new file mode 100644 index 000000000..2afc373d7 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/pom.xml @@ -0,0 +1,186 @@ + + + 4.0.0 + com.redhat.sonataflow.db.postgres + sonataflow-db-migrator + 999.0.0 + + + 3.13.0 + UTF-8 + UTF-8 + quarkus-bom + io.quarkus.platform + 3.13.0 + true + 3.2.5 + 0.8.12 + 1.19.0 + + + + + + ${quarkus.platform.group-id} + ${quarkus.platform.artifact-id} + ${quarkus.platform.version} + pom + import + + + + + + + io.quarkus + quarkus-flyway + + + io.quarkus + quarkus-jdbc-postgresql + + + org.flywaydb + flyway-database-postgresql + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-junit5 + test + + + org.mockito + mockito-junit-jupiter + test + + + org.mockito + mockito-core + test + + + com.github.stefanbirkner + system-rules + ${systemrules.version} + test + + + org.junit.jupiter + junit-jupiter + test + + + + + src/test/java + + + ${quarkus.platform.group-id} + quarkus-maven-plugin + ${quarkus.platform.version} + true + + + + build + generate-code + generate-code-tests + native-image-agent + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${compiler-plugin.version} + + + -parameters + + + + + maven-surefire-plugin + ${surefire-plugin.version} + + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + maven-failsafe-plugin + ${surefire-plugin.version} + + + + integration-test + verify + + + + + + ${project.build.directory}/${project.build.finalName}-runner + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + + prepare-agent + + + + report + prepare-package + + report + + + + + + + + + + native + + + native + + + + false + true + + + + + + + apache + Apache Maven + default + https://repo.maven.apache.org/maven2/ + + false + + + + diff --git a/images/tools/sonataflow-db-migrator/src/main/cekit/image.yaml b/images/tools/sonataflow-db-migrator/src/main/cekit/image.yaml new file mode 100644 index 000000000..4f4ae8a46 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/main/cekit/image.yaml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: "docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql" +version: "999-SNAPSHOT" +from: registry.access.redhat.com/ubi8/openjdk-17-runtime:1.19 +description: Flyway image for DI/JS database migration + +labels: + - name: "org.kie.kogito.version" + value: "999-SNAPSHOT" + - name: "maintainer" + value: "Apache KIE " + - name: "io.k8s.description" + value: "Kogito DB Migration creates schemas and tables for Data Index and Jobs Service for PostgreSQL database" + - name: "io.k8s.display-name" + value: "Kogito DB Migration for Data Index and Jobs Service - PostgreSQL" + - name: "io.openshift.tags" + value: "kogito,db-migration" + +modules: + repositories: + - path: modules + install: + - name: kogito-postgres-db-migration-deps + +run: + workdir: "/home/kogito" + entrypoint: + - "/home/kogito/migration.sh" \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/install b/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/install new file mode 100644 index 000000000..5fa1a3e0e --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/install @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +useradd -m kogito + +cd /home/kogito || exit + +chgrp -R 0 /home/kogito +chown -R 0 /home/kogito +chmod -R g=u /home/kogito + +chmod +x /home/kogito/migration.sh \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh b/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh new file mode 100644 index 000000000..fe41d336d --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +OPERATOR_VERSION=999.0.0 + +for arg in "$@"; do + case "$arg" in + OPERATOR_VERSION=*) OPERATOR_VERSION="${arg#*=}" ;; + esac +done + +cd /home/kogito/ + +mv sonataflow-db-migrator-runner.jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar + +java -jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml b/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml new file mode 100644 index 000000000..0605dacc6 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +schema_version: 1 +name: kogito-postgres-db-migration-deps +version: "1.0" +artifacts: + - name: migration.sh + path: migration.sh + dest: /home/kogito + - name: sonataflow-db-migrator-runner.jar + path: sonataflow-db-migrator-runner.jar + dest: /home/kogito +execute: + - script: install diff --git a/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/DBConnectionChecker.java b/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/DBConnectionChecker.java new file mode 100644 index 000000000..c9143f850 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/DBConnectionChecker.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.kie.kogito.postgresql.migrator; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import io.quarkus.logging.Log; + +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import jakarta.enterprise.context.ApplicationScoped; + +@ApplicationScoped +public class DBConnectionChecker { + @ConfigProperty(name = "quarkus.datasource.dataindex.jdbc.url") + String dataIndexDBURL; + + @ConfigProperty(name = "quarkus.datasource.dataindex.username") + String dataIndexDBUserName; + + @ConfigProperty(name = "quarkus.datasource.dataindex.password") + String dataIndexDBPassword; + + @ConfigProperty(name = "quarkus.datasource.jobsservice.jdbc.url") + String jobsServiceDBURL; + + @ConfigProperty(name = "quarkus.datasource.jobsservice.username") + String jobsServiceDBUserName; + + @ConfigProperty(name = "quarkus.datasource.jobsservice.password") + String jobsServiceDBPassword; + + private void checkDBConnection(String dbURL, String dbUser, String dbPassword) throws SQLException { + try (Connection db = DriverManager.getConnection(dbURL, dbUser, dbPassword)) { + Log.infof("Checking DB connection: %s - success", dbURL); + } catch (SQLException sqe) { + Log.infof("Checking DB connection %s- failed", dbURL); + throw sqe; + } + } + + public void checkDataIndexDBConnection() throws SQLException { + checkDBConnection(dataIndexDBURL, dataIndexDBUserName, dataIndexDBPassword); + } + + public void checkJobsServiceDBConnection() throws SQLException { + checkDBConnection(jobsServiceDBURL, jobsServiceDBUserName, jobsServiceDBPassword); + } +} diff --git a/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/DBMigrator.java b/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/DBMigrator.java new file mode 100644 index 000000000..bb69266f4 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/DBMigrator.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.kie.kogito.postgresql.migrator; + +import io.quarkus.runtime.Quarkus; +import io.quarkus.runtime.QuarkusApplication; +import io.quarkus.runtime.annotations.QuarkusMain; +import jakarta.inject.Inject; +import io.quarkus.logging.Log; + +import java.sql.SQLException; + +import org.eclipse.microprofile.config.inject.ConfigProperty; + +@QuarkusMain +public class DBMigrator implements QuarkusApplication { + + @Inject + MigrationService service; + + @Inject + DBConnectionChecker dbConnectionChecker; + + @ConfigProperty(name = "migrate.db.dataindex") + Boolean migrateDataIndex; + + @ConfigProperty(name = "migrate.db.jobsservice") + Boolean migrateJobsService; + + @Override + public int run(String... args) { + if (migrateDataIndex) { + try { + dbConnectionChecker.checkDataIndexDBConnection(); + } catch (SQLException e) { + Log.error( "Error obtaining data index database connection. Cannot proceed, exiting."); + Quarkus.asyncExit(-1); + return -1; + } + service.migrateDataIndex(); + } + + if (migrateJobsService) { + try { + dbConnectionChecker.checkJobsServiceDBConnection(); + } catch (SQLException e) { + Log.error( "Error obtaining jobs service database connection. Cannot proceed, exiting."); + Quarkus.asyncExit(-2); + return -2; + } + service.migrateJobsService(); + } + + Quarkus.asyncExit(0); + return 0; + } +} \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java b/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java new file mode 100644 index 000000000..233539cc3 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.kie.kogito.postgresql.migrator; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import io.quarkus.logging.Log; + +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.flywaydb.core.Flyway; + +import io.quarkus.flyway.FlywayDataSource; + +@ApplicationScoped +public class MigrationService { + @Inject + @FlywayDataSource("dataindex") + Flyway flywayDataIndex; + + @Inject + @FlywayDataSource("jobsservice") + Flyway flywayJobsService; + + @ConfigProperty(name = "quarkus.flyway.dataindex.clean-at-start") + Boolean cleanDataIndex; + + @ConfigProperty(name = "quarkus.flyway.jobsservice.clean-at-start") + Boolean cleanJobsService; + + private void migrateDB(Flyway flywayService, Boolean clean, String serviceName) { + Log.info("Migrating " + serviceName); + if (clean) { + Log.info("Cleaned the " + serviceName); + flywayService.clean(); + } + flywayService.migrate(); + if (flywayService.info() != null) { + Log.info(flywayService.info().current().getVersion().toString()); + } + } + + public void migrateDataIndex() { + migrateDB(flywayDataIndex, cleanDataIndex, "data-index"); + } + + public void migrateJobsService() { + migrateDB(flywayJobsService, cleanJobsService, "jobs-service"); + } +} \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/src/main/resources/application.properties b/images/tools/sonataflow-db-migrator/src/main/resources/application.properties new file mode 100644 index 000000000..59b8c2382 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/main/resources/application.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Data Index data source +migrate.db.dataindex=false +quarkus.datasource.dataindex.db-kind=postgresql +quarkus.datasource.dataindex.username=postgres +quarkus.datasource.dataindex.password=postgres +quarkus.datasource.dataindex.jdbc.url=jdbc:postgresql://localhost:5432/postgres +quarkus.flyway.dataindex.locations=classpath:postgresql/data-index +quarkus.flyway.dataindex.schemas=dataindex +quarkus.flyway.dataindex.migrate-at-start=false +quarkus.flyway.dataindex.clean-at-start=false + +# Jobs Service data source +migrate.db.jobsservice=false +quarkus.datasource.jobsservice.db-kind=postgresql +quarkus.datasource.jobsservice.username=postgres +quarkus.datasource.jobsservice.password=postgres +quarkus.datasource.jobsservice.jdbc.url=jdbc:postgresql://localhost:5432/postgres +quarkus.flyway.jobsservice.locations=classpath:postgresql/jobs-service +quarkus.flyway.jobsservice.schemas=jobsservice +quarkus.flyway.jobsservice.migrate-at-start=false +quarkus.flyway.jobsservice.clean-at-start=false \ No newline at end of file diff --git a/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/DBConnectionCheckerTest.java b/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/DBConnectionCheckerTest.java new file mode 100644 index 000000000..0b5032a34 --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/DBConnectionCheckerTest.java @@ -0,0 +1,52 @@ +package org.kie.kogito.postgresql.migrator; + +import io.quarkus.test.Mock; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mockStatic; + +public class DBConnectionCheckerTest { + DBConnectionChecker dbConnectionChecker = new DBConnectionChecker(); + + @Mock + static DriverManager driverManager; + + @BeforeAll + public static void init() { + mockStatic(DriverManager.class); + } + + @BeforeEach + public void setupEach() { + dbConnectionChecker.dataIndexDBURL = "jdbc:postgresql://db-service:5432/di"; + dbConnectionChecker.dataIndexDBUserName = "postgres"; + dbConnectionChecker.dataIndexDBPassword = "postgres"; + + dbConnectionChecker.jobsServiceDBURL = "jdbc:postgresql://db-service:5432/js"; + dbConnectionChecker.jobsServiceDBUserName = "postgres"; + dbConnectionChecker.jobsServiceDBPassword = "postgres"; + } + + @Test + public void testCheckDBConnections() throws SQLException { + Mockito.when(driverManager.getConnection(anyString(), anyString(), anyString())).thenReturn(Mockito.mock(Connection.class)); + assertDoesNotThrow(() -> dbConnectionChecker.checkDataIndexDBConnection()); + assertDoesNotThrow(() -> dbConnectionChecker.checkJobsServiceDBConnection()); + } + + @Test + public void testCheckDBConnectionsThrowSQLException() throws SQLException { + Mockito.when(driverManager.getConnection(anyString(), anyString(), anyString())).thenThrow(SQLException.class); + assertThrows(SQLException.class, () -> dbConnectionChecker.checkDataIndexDBConnection()); + assertThrows(SQLException.class, () -> dbConnectionChecker.checkJobsServiceDBConnection()); + } +} diff --git a/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/DBMigratorTest.java b/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/DBMigratorTest.java new file mode 100644 index 000000000..f88f960af --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/DBMigratorTest.java @@ -0,0 +1,78 @@ +package org.kie.kogito.postgresql.migrator; + +import io.quarkus.test.Mock; +import org.junit.Rule; +import org.junit.contrib.java.lang.system.ExpectedSystemExit; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.sql.SQLException; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +public class DBMigratorTest { + @Rule + public final ExpectedSystemExit exitRule = ExpectedSystemExit.none(); + + @Mock + MigrationService migrationService; + + @Mock + DBConnectionChecker dbConnectionChecker; + + DBMigrator dbMigrator = new DBMigrator(); + + @BeforeEach + public void setupEach() { + migrationService = mock(MigrationService.class); + dbConnectionChecker = mock(DBConnectionChecker.class); + } + + @Test + public void testMigratorWithNoMigrations() throws Exception { + dbMigrator.migrateDataIndex = false; + dbMigrator.migrateJobsService = false; + + exitRule.expectSystemExitWithStatus(0); + dbMigrator.run(); + } + + @Test + public void testMigratorWithAllMigrations() throws Exception { + dbMigrator.migrateDataIndex = true; + dbMigrator.migrateJobsService = true; + dbMigrator.dbConnectionChecker = dbConnectionChecker; + dbMigrator.service = migrationService; + + exitRule.expectSystemExitWithStatus(0); + dbMigrator.run(); + } + + @Test + public void testDataIndexMigrationWithException() throws Exception { + dbMigrator.migrateDataIndex = true; + dbMigrator.migrateJobsService = false; + dbMigrator.dbConnectionChecker = dbConnectionChecker; + dbMigrator.service = migrationService; + + doThrow(new SQLException()).when(dbConnectionChecker).checkDataIndexDBConnection(); + + exitRule.expectSystemExitWithStatus(-1); + dbMigrator.run(); + } + + @Test + public void testJobsServiceWithException() throws Exception { + dbMigrator.migrateDataIndex = false; + dbMigrator.migrateJobsService = true; + dbMigrator.dbConnectionChecker = dbConnectionChecker; + dbMigrator.service = migrationService; + + doThrow(new SQLException()).when(dbConnectionChecker).checkJobsServiceDBConnection(); + + exitRule.expectSystemExitWithStatus(-2); + dbMigrator.run(); + } +} diff --git a/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java b/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java new file mode 100644 index 000000000..71a81681b --- /dev/null +++ b/images/tools/sonataflow-db-migrator/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java @@ -0,0 +1,53 @@ +package org.kie.kogito.postgresql.migrator; + +import io.quarkus.test.Mock; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.output.CleanResult; +import org.flywaydb.core.api.output.MigrateResult; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.mockito.Mockito.*; + +public class MigrationServiceTest { + @Mock + Flyway flyway; + + MigrationService migrationService = new MigrationService(); + + @BeforeEach + public void setupEach() { + flyway = mock(Flyway.class); + when(flyway.migrate()).thenReturn(new MigrateResult("flywayVersion", "db", "schema", "postgres")); + when(flyway.clean()).thenReturn(new CleanResult("flywayVersion", "db")); + } + + @Test + public void testMigrateDataIndexWithNoClean() { + migrationService.cleanDataIndex = false; + migrationService.flywayDataIndex = flyway; + migrationService.migrateDataIndex(); + } + + @Test + public void testMigrateDataIndexWithClean() { + migrationService.cleanDataIndex = true; + migrationService.flywayDataIndex = flyway; + migrationService.migrateDataIndex(); + } + + @Test + public void testMigrateJobsServiceWithNoClean() { + migrationService.cleanJobsService = false; + migrationService.flywayJobsService = flyway; + migrationService.migrateJobsService(); + } + + @Test + public void testMigrateJobsServiceWithClean() { + migrationService.cleanJobsService = true; + migrationService.flywayJobsService = flyway; + migrationService.migrateJobsService(); + } +} diff --git a/test/e2e/clusterplatform_test.go b/test/e2e/clusterplatform_test.go index a2e316eb4..507662eb3 100644 --- a/test/e2e/clusterplatform_test.go +++ b/test/e2e/clusterplatform_test.go @@ -49,12 +49,12 @@ var _ = Describe("Validate a clusterplatform", Ordered, func() { ) BeforeEach(func() { - targetNamespace = fmt.Sprintf("test-%d", rand.Intn(1024)+1) + targetNamespace = fmt.Sprintf("test-%d", rand.Intn(randomIntRange)+1) cmd := exec.Command("kubectl", "create", "namespace", targetNamespace) _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) - targetNamespace2 = fmt.Sprintf("test-%d", rand.Intn(1024)+1) + targetNamespace2 = fmt.Sprintf("test-%d", rand.Intn(randomIntRange)+1) cmd = exec.Command("kubectl", "create", "namespace", targetNamespace2) _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 104afa315..1afd9d5af 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -44,6 +44,16 @@ type check struct { Data map[string]string `json:"data"` } +const ( + ephemeral = "ephemeral" + postgreSQL = "postgreSQL" + clusterWideEphemeral = "cluster-wide-ephemeral" + ephemeralDataIndex = "ephemeral-data-index" + ephemeralJobService = "ephemeral-job-service" +) + +const randomIntRange = 16384 //Set to large number to avoid cluster namespace name collisions + var ( upStatus string = "UP" ) diff --git a/test/e2e/platform_test.go b/test/e2e/platform_test.go index 511503be5..bd400cc60 100644 --- a/test/e2e/platform_test.go +++ b/test/e2e/platform_test.go @@ -36,14 +36,6 @@ import ( . "github.com/onsi/gomega" ) -const ( - ephemeral = "ephemeral" - postgreSQL = "postgreSQL" - clusterWideEphemeral = "cluster-wide-ephemeral" - ephemeralDataIndex = "ephemeral-data-index" - ephemeralJobService = "ephemeral-job-service" -) - var _ = Describe("Validate the persistence", Ordered, func() { var ( @@ -52,7 +44,7 @@ var _ = Describe("Validate the persistence", Ordered, func() { ) BeforeEach(func() { - targetNamespace = fmt.Sprintf("test-%d", rand.Intn(1024)+1) + targetNamespace = fmt.Sprintf("test-%d", rand.Intn(randomIntRange)+1) cmd := exec.Command("kubectl", "create", "namespace", targetNamespace) _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) @@ -113,7 +105,7 @@ var _ = Describe("Validate the persistence", Ordered, func() { Expect(sf).NotTo(BeEmpty(), "sonataflow name is empty") EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningStateInNamespace(sf, targetNamespace) - }, 10*time.Minute, 5).Should(BeTrue()) + }, 15*time.Minute, 1*time.Minute).Should(BeTrue()) } }, Entry("with both Job Service and Data Index and ephemeral persistence and the workflow in a dev profile", test.GetSonataFlowE2EPlatformServicesDirectory(), metadata.DevProfile.String(), ephemeral), diff --git a/test/e2e/workflow_test.go b/test/e2e/workflow_test.go index ed8f95c4e..8b6e1c99c 100644 --- a/test/e2e/workflow_test.go +++ b/test/e2e/workflow_test.go @@ -44,7 +44,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { var targetNamespace string BeforeEach(func() { - targetNamespace = fmt.Sprintf("test-%d", rand.Intn(1024)+1) + targetNamespace = fmt.Sprintf("test-%d", rand.Intn(randomIntRange)+1) cmd := exec.Command("kubectl", "create", "namespace", targetNamespace) _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) @@ -153,7 +153,7 @@ var _ = Describe("Validate the persistence ", Ordered, func() { ) BeforeEach(func() { - ns = fmt.Sprintf("test-%d", rand.Intn(1024)+1) + ns = fmt.Sprintf("test-%d", rand.Intn(randomIntRange)+1) cmd := exec.Command("kubectl", "create", "namespace", ns) _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) @@ -188,7 +188,7 @@ var _ = Describe("Validate the persistence ", Ordered, func() { out, err := utils.Run(cmd) GinkgoWriter.Printf("%s\n", string(out)) return err - }, 12*time.Minute, 5).Should(Succeed()) + }, 15*time.Minute, 1*time.Minute).Should(Succeed()) By("Evaluate status of the workflow's pod database connection health endpoint") cmd = exec.Command("kubectl", "get", "pod", "-l", "sonataflow.org/workflow-app", "-n", ns, "-ojsonpath={.items[*].metadata.name}") @@ -224,7 +224,7 @@ var _ = Describe("Validate the persistence ", Ordered, func() { } } return false - }, 1*time.Minute).Should(BeTrue()) + }, 4*time.Minute).Should(BeTrue()) // Persistence initialization checks cmd = exec.Command("kubectl", "get", "pod", "-l", "sonataflow.org/workflow-app", "-n", ns, "-ojsonpath={.items[*].metadata.name}") output, err = utils.Run(cmd)