diff --git a/.github/workflows/java-publish-docker.yml b/.github/workflows/build-dev-release.yml similarity index 66% rename from .github/workflows/java-publish-docker.yml rename to .github/workflows/build-dev-release.yml index 196ca4ee..83c821e2 100644 --- a/.github/workflows/java-publish-docker.yml +++ b/.github/workflows/build-dev-release.yml @@ -1,20 +1,14 @@ -name: Build and publish Docker images to github registry +name: Build and publish dev release Docker image to Github Container Registry ghcr.io -on: - push: - branches: - - master - - version-* - paths: - - gradle.properties +on: workflow_dispatch jobs: build: uses: th2-net/.github/.github/workflows/compound-java.yml@main with: build-target: 'Docker' - runsOn: ubuntu-latest - gradleVersion: '7' + devRelease: true + createTag: true docker-username: ${{ github.actor }} # FIXME: strict scanner was disabled for 4.6.4 hotfix publishing and must be removed after that strict-scanner: false diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml new file mode 100644 index 00000000..dcf70be4 --- /dev/null +++ b/.github/workflows/build-release.yml @@ -0,0 +1,15 @@ +name: Build and publish release Docker image to Github Container Registry ghcr.io + +on: workflow_dispatch + +jobs: + build: + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Docker' + devRelease: false + createTag: true + docker-username: ${{ github.actor }} + secrets: + docker-password: ${{ secrets.GITHUB_TOKEN }} + nvd-api-key: ${{ secrets.NVD_APIKEY }} \ No newline at end of file diff --git a/.github/workflows/dev-java-publish-docker.yml b/.github/workflows/build-sanpshot.yml similarity index 76% rename from .github/workflows/dev-java-publish-docker.yml rename to .github/workflows/build-sanpshot.yml index 7f3c716f..9454329f 100644 --- a/.github/workflows/dev-java-publish-docker.yml +++ b/.github/workflows/build-sanpshot.yml @@ -1,12 +1,11 @@ -name: Dev build and publish Docker images to github registry +name: Build and publish Docker image to Github Container Registry ghcr.io on: push: branches-ignore: - master - version-* - - dev-version-* - - dependabot* + - dependabot** paths-ignore: - README.md @@ -15,8 +14,6 @@ jobs: uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main with: build-target: 'Docker' - runsOn: ubuntu-latest - gradleVersion: '7' docker-username: ${{ github.actor }} # FIXME: strict scanner was disabled for 4.6.4 hotfix publishing and must be removed after that strict-scanner: false diff --git a/.github/workflows/ci-unwelcome-words.yml b/.github/workflows/ci-unwelcome-words.yml index cd7adcf3..add8e7fd 100644 --- a/.github/workflows/ci-unwelcome-words.yml +++ b/.github/workflows/ci-unwelcome-words.yml @@ -5,19 +5,19 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.sha }} - - name: Checkout tool - uses: actions/checkout@v2 - with: - repository: exactpro-th2/ci-github-action - ref: master - token: ${{ secrets.PAT_CI_ACTION }} - path: ci-github-action - - name: Run CI action - uses: ./ci-github-action - with: - ref: ${{ github.sha }} + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + - name: Checkout tool + uses: actions/checkout@v4 + with: + repository: exactpro-th2/ci-github-action + ref: master + token: ${{ secrets.PAT_CI_ACTION }} + path: ci-github-action + - name: Run CI action + uses: ./ci-github-action + with: + ref: ${{ github.sha }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000..23e327e0 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,26 @@ +name: "Run integration tests for infra-operator" + +on: + push: + branches: + - '*' + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 'zulu' '11' + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '11' + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + - name: Build with Gradle + run: ./gradlew --info clean integrationTest + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: integration-test-results + path: build/reports/tests/integrationTest/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d7a0206c..666e6cd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,9 @@ -FROM gradle:7.6-jdk11 AS build +FROM gradle:8.7-jdk11 AS build ARG app_version=0.0.0 COPY ./ . -RUN gradle build -Prelease_version=${app_version} +RUN gradle --no-daemon clean build dockerPrepare -Prelease_version=${release_version} -RUN mkdir /home/app -RUN cp ./build/libs/*.jar /home/app/application.jar - -FROM eclipse-temurin:11-alpine -COPY --from=build /home/app /home/app - -WORKDIR /home/app/ -ENTRYPOINT ["java", "-Dlog4j2.configurationFile=file:/var/th2/config/log4j2.properties", "-jar", "/home/app/application.jar"] \ No newline at end of file +FROM adoptopenjdk/openjdk11:alpine +WORKDIR /home +COPY --from=build /home/gradle/build/docker . +ENTRYPOINT ["/home/service/bin/service", "/var/th2/config/log4j2.properties"] \ No newline at end of file diff --git a/README.md b/README.md index 3cdc7c7e..e9daf809 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,6 @@ rabbitMQManagement: persistence: true # determines if the RabbitMQ resources are persistent or not - cleanUpOnStart: false - # if option is true, operator removes all queues and exchanges from RabbitMQ on start - schemaPermissions: # this section describes what permissions schema RabbitMQ user will have on its own resources # see RabbitMQ documentation to find out how permissions are described @@ -147,6 +144,20 @@ openshift: ## Release notes +### 4.7.0 ++ Improved clean rubbish from RabbitMQ on start to delete only redundant resources. + The `cleanUpOnStart` option has been removed, the clean rubbish function is enabled. ++ Migrated to th2 plugin `0.1.2` + ++ Updated: + + bom: `4.6.1` + + kubernetes-client: `6.13.1` + + force okhttp: `4.12.0` + + force logging-interceptor: `4.12.0` + + http-client: `5.2.0` + + java-uuid-generator: `5.1.0` + + kotlin-logging: `5.1.4` + ### 4.6.5 + Fix issue when changing `desabled` flag to `true` does not remove the resource from k8s diff --git a/build.gradle b/build.gradle index a4d1eafd..9e1f2008 100644 --- a/build.gradle +++ b/build.gradle @@ -1,141 +1,117 @@ -/* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - plugins { - id 'java' + id 'application' id 'checkstyle' - id "io.gitlab.arturbosch.detekt" version "${detekt_version}" - id "org.jetbrains.kotlin.jvm" version "${kotlin_version}" - id "org.owasp.dependencycheck" version "${owaspVersion}" + alias(libs.plugins.detekt) + alias(libs.plugins.kotlin) + alias(libs.plugins.th2.component) + alias(libs.plugins.download) } group = 'com.exactpro.th2' version = release_version +kotlin { + jvmToolchain(11) +} + repositories { mavenCentral() } checkstyle { - toolVersion = "10.12.0" + toolVersion = "10.12.4" } detekt { buildUponDefaultConfig = true autoCorrect = true - config = files("$rootDir/config/detekt/detekt.yml") -} - -ext { - uuid_generator_version = '4.2.0' - okhttp_version = '4.10.0' - fabric8_version = '6.6.2' - rabbit_amqp_version = '5.16.0' - rabbit_http_version = '5.0.0' - kotlin_logging_version = '3.0.0' // 3.0.0 the las version supported 1.6.21 - jetbrains_annotations_version = '24.0.1' - mockito_version = '3.11.2' - jupiter_version = '5.9.2' - guava_version = '32.0.1-jre' - snakeyaml_version = '2.0' -} - -configurations.configureEach() { - resolutionStrategy { - force "com.google.guava:guava:$guava_version" - force "org.yaml:snakeyaml:$snakeyaml_version" - } + config.setFrom("$rootDir/config/detekt/detekt.yml") } dependencies { - api platform('com.exactpro.th2:bom:4.3.0') implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" - implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin' - implementation "com.fasterxml.uuid:java-uuid-generator:${uuid_generator_version}" - - implementation "com.squareup.okhttp3:okhttp:${okhttp_version}" + implementation "com.fasterxml.jackson.module:jackson-module-kotlin" + implementation libs.java.uuid.generator implementation "org.apache.commons:commons-text" - implementation "io.fabric8:kubernetes-client:${fabric8_version}" - implementation "com.rabbitmq:amqp-client:${rabbit_amqp_version}" - implementation "com.rabbitmq:http-client:${rabbit_http_version}" + implementation libs.kubernetes.client + implementation(libs.okhttp) { + because "okhttp:3.12.12 has vulnerabilities" + } + implementation(libs.logging.interceptor) { + because "logging-interceptor:3.12.12 has vulnerabilities" + } - implementation "org.slf4j:slf4j-api" - implementation "org.apache.logging.log4j:log4j-slf4j2-impl" - implementation "org.apache.logging.log4j:log4j-core" - implementation group: 'io.github.microutils', name: 'kotlin-logging', version: kotlin_logging_version + implementation "com.rabbitmq:amqp-client" + implementation libs.http.client - implementation "org.jetbrains:annotations:${jetbrains_annotations_version}" + implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' + implementation 'org.apache.logging.log4j:log4j-core' + implementation libs.kotlin.logging + implementation "org.jetbrains:annotations" implementation "io.prometheus:simpleclient" implementation "io.prometheus:simpleclient_httpserver" implementation "io.prometheus:simpleclient_hotspot" - testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" + implementation "com.google.guava:guava" - testImplementation "org.junit.jupiter:junit-jupiter-api:${jupiter_version}" - testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${jupiter_version}" + testImplementation(platform(libs.testcontainers.bom)) + testImplementation 'org.testcontainers:rabbitmq' + testImplementation 'org.testcontainers:k3s' - detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:${detekt_version}") -} -dependencyCheck { - formats=['SARIF', 'JSON', 'HTML'] - failBuildOnCVSS=5 + testImplementation libs.mockito.core + testImplementation libs.mockito.kotlin + testImplementation libs.junit.jupiter + testImplementation libs.junit.jupiter.params + testImplementation "org.jetbrains.kotlin:kotlin-test-junit5" + testImplementation libs.bcpkix.jdk18on + testImplementation libs.awaitility + testImplementation libs.strikt.core - analyzers { - assemblyEnabled = false - nugetconfEnabled = false - nodeEnabled = false - } + detektPlugins libs.detekt.formatting } -jar { - duplicatesStrategy = DuplicatesStrategy.INCLUDE - manifest { - attributes( - 'Created-By': "${System.getProperty('java.version')} (${System.getProperty('java.vendor')})", - 'Specification-Title': '', - 'Specification-Vendor': 'Exactpro Systems LLC', - 'Main-Class': 'com.exactpro.th2.infraoperator.Th2CrdController', - 'Implementation-Title': project.archivesBaseName, - 'Implementation-Vendor': 'Exactpro Systems LLC', - 'Implementation-Vendor-Id': 'com.exactpro', - 'Implementation-Version': project.version - ) - } - - from { - configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } - } +wrapper { + version '8.7' + distributionType Wrapper.DistributionType.BIN } -test { - useJUnitPlatform() +tasks.register("downloadCRDs", Download) { + group = "verification" + src([ + 'https://raw.githubusercontent.com/th2-net/th2-infra/master/chart/crds/th2-box-crd.yaml', + 'https://raw.githubusercontent.com/th2-net/th2-infra/master/chart/crds/th2-core-box-crd.yaml', + 'https://raw.githubusercontent.com/th2-net/th2-infra/master/chart/crds/th2-dictionary-crd.yaml', + 'https://raw.githubusercontent.com/th2-net/th2-infra/master/chart/crds/th2-estore-crd.yaml', + 'https://raw.githubusercontent.com/th2-net/th2-infra/master/chart/crds/th2-job-crd.yaml', + 'https://raw.githubusercontent.com/th2-net/th2-infra/master/chart/crds/th2-mstore-crd.yaml', + ]) + dest layout.buildDirectory.dir('resources/test/crds').get() } -compileKotlin { - kotlinOptions { - jvmTarget = "11" +checkstyleTest.dependsOn("downloadCRDs") + +test { + dependsOn("downloadCRDs") + useJUnitPlatform { + excludeTags("integration-test") } } -compileTestKotlin { - kotlinOptions { - jvmTarget = "11" +tasks.register("integrationTest", Test) { + group = "verification" + dependsOn("downloadCRDs") + useJUnitPlatform { + includeTags("integration-test") + } + testLogging { + showStandardStreams = true } } + +dependencyCheck { + skipConfigurations += "checkstyle" +} \ No newline at end of file diff --git a/config/detekt/detekt.yml b/config/detekt/detekt.yml index c6447f59..70661f98 100644 --- a/config/detekt/detekt.yml +++ b/config/detekt/detekt.yml @@ -119,7 +119,7 @@ complexity: active: true functionThreshold: 6 constructorThreshold: 15 - ignoreDefaultParameters: false + ignoreDefaultParameters: true ignoreDataClasses: false ignoreAnnotatedParameter: [] MethodOverloading: diff --git a/gradle.properties b/gradle.properties index 621c0cda..72df96a7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1 @@ -release_version = 4.6.5 -kotlin_version = 1.6.21 -detekt_version = 1.22.0 -owaspVersion = 8.2.1 +release_version = 4.7.0 \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 00000000..cd746e8e --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,34 @@ +[versions] +kotlin = "1.8.22" +th2-plugin = "0.1.2" +jupiter = "5.10.3" +okhttp3 = "4.12.0" +detekt = "1.23.6" +strikt = "0.34.1" + +[libraries] +kotlin-logging = { group = "io.github.oshai", name = "kotlin-logging", version = "5.1.4" } +okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp3" } +logging-interceptor = { group = "com.squareup.okhttp3", name = "logging-interceptor", version.ref = "okhttp3" } +kubernetes-client = { group = "io.fabric8", name = "kubernetes-client", version = "6.13.1" } +http-client = { group = "com.rabbitmq", name = "http-client", version = "5.2.0" } +java-uuid-generator = { group = "com.fasterxml.uuid", name = "java-uuid-generator", version = "5.1.0" } + +mockito-core = { group = "org.mockito", name = "mockito-core", version = "5.12.0" } +mockito-kotlin = { group = "org.mockito.kotlin", name = "mockito-kotlin", version = "5.4.0" } +junit-jupiter = { group = "org.junit.jupiter", name = "junit-jupiter", version.ref = "jupiter" } +junit-jupiter-params = { group = "org.junit.jupiter", name = "junit-jupiter-params", version.ref = "jupiter" } + +strikt-core = { group = "io.strikt", name = "strikt-core", version.ref = "strikt" } +testcontainers-bom = { group = "org.testcontainers", name = "testcontainers-bom", version = "1.20.1" } +bcpkix-jdk18on = { group = "org.bouncycastle", name = "bcpkix-jdk18on", version = "1.78.1" } +awaitility = { group = "org.awaitility", name = "awaitility", version = "4.2.2" } + +detekt-formatting = { group = "io.gitlab.arturbosch.detekt", name = "detekt-formatting", version.ref = "detekt" } + +[plugins] +kotlin = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } +detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" } +download = { id = "de.undercouch.download", version = "5.6.0" } + +th2-component = { id = "com.exactpro.th2.gradle.component", version.ref = "th2-plugin" } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 6ba21842..b82aa23a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,22 +1,7 @@ -# -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#Mon May 04 17:09:53 MSK 2020 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-all.zip diff --git a/gradlew b/gradlew index 54125e4d..1aa94a42 100755 --- a/gradlew +++ b/gradlew @@ -1,13 +1,13 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -17,78 +17,111 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -97,92 +130,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=$((i+1)) + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" fi +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index f9553162..25da30db 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,4 +1,20 @@ -@if "%DEBUG%" == "" @echo off +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -9,25 +25,29 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -35,48 +55,36 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/src/main/java/com/exactpro/th2/infraoperator/OperatorState.java b/src/main/java/com/exactpro/th2/infraoperator/OperatorState.java index c0e2155b..4c7de721 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/OperatorState.java +++ b/src/main/java/com/exactpro/th2/infraoperator/OperatorState.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import static java.util.Collections.unmodifiableSet; + public enum OperatorState { INSTANCE; @@ -104,6 +106,10 @@ public Collection getAllBoxResources() { return resources; } + public Collection getNamespaces() { + return unmodifiableSet(namespaceStates.keySet()); + } + private NamespaceState computeIfAbsent(String namespace) { return namespaceStates.computeIfAbsent(namespace, s -> new NamespaceState()); } diff --git a/src/main/java/com/exactpro/th2/infraoperator/Th2CrdController.java b/src/main/java/com/exactpro/th2/infraoperator/Th2CrdController.java index 4518890b..82a1d92e 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/Th2CrdController.java +++ b/src/main/java/com/exactpro/th2/infraoperator/Th2CrdController.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package com.exactpro.th2.infraoperator; +import com.exactpro.th2.infraoperator.configuration.ConfigLoader; +import com.exactpro.th2.infraoperator.configuration.OperatorConfig; import com.exactpro.th2.infraoperator.metrics.OperatorMetrics; import com.exactpro.th2.infraoperator.metrics.PrometheusServer; import com.exactpro.th2.infraoperator.operator.impl.BoxHelmTh2Op; @@ -27,36 +29,133 @@ import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import com.exactpro.th2.infraoperator.spec.strategy.redeploy.ContinuousTaskWorker; import com.exactpro.th2.infraoperator.spec.strategy.redeploy.tasks.CheckResourceCacheTask; +import com.exactpro.th2.infraoperator.util.RabbitMQUtils; +import com.exactpro.th2.infraoperator.util.Utils; +import io.fabric8.kubernetes.client.KubernetesClient; +import org.apache.logging.log4j.core.LoggerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class Th2CrdController { +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Deque; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; - private static final Logger logger = LoggerFactory.getLogger(Th2CrdController.class); +import static com.exactpro.th2.infraoperator.util.KubernetesUtils.createKubernetesClient; - public static void main(String[] args) { +public class Th2CrdController implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(Th2CrdController.class); + + private final PrometheusServer prometheusServer; + + private final KubernetesClient kubClient; + + private final DefaultWatchManager watchManager; + + private final RabbitMQContext rabbitMQContext; + + private final ContinuousTaskWorker continuousTaskWorker; + + public Th2CrdController() throws IOException, URISyntaxException { + OperatorConfig config = ConfigLoader.loadConfiguration(); + prometheusServer = new PrometheusServer(config.getPrometheusConfiguration()); + kubClient = createKubernetesClient(); + rabbitMQContext = new RabbitMQContext(config.getRabbitMQManagement()); + watchManager = new DefaultWatchManager(kubClient, rabbitMQContext); + continuousTaskWorker = new ContinuousTaskWorker(); - var watchManager = DefaultWatchManager.getInstance(); - PrometheusServer.start(); OperatorMetrics.resetCacheErrors(); - try { - RabbitMQContext.declareTopicExchange(); - RabbitMQContext.cleanUpRabbitBeforeStart(); + RabbitMQUtils.deleteRabbitMQRubbish(kubClient, rabbitMQContext); + // TODO: topic exchange should be removed when all namespaces are removed / disabled + rabbitMQContext.declareTopicExchange(); + + watchManager.addTarget(MstoreHelmTh2Op::new); + watchManager.addTarget(EstoreHelmTh2Op::new); + watchManager.addTarget(BoxHelmTh2Op::new); + watchManager.addTarget(CoreBoxHelmTh2Op::new); + watchManager.addTarget(JobHelmTh2Op::new); + + watchManager.startInformers(); + continuousTaskWorker.add(new CheckResourceCacheTask(300)); + } + + public static void main(String[] args) { + Deque resources = new ConcurrentLinkedDeque<>(); + Lock lock = new ReentrantLock(); + Condition condition = lock.newCondition(); - watchManager.addTarget(MstoreHelmTh2Op::new); - watchManager.addTarget(EstoreHelmTh2Op::new); - watchManager.addTarget(BoxHelmTh2Op::new); - watchManager.addTarget(CoreBoxHelmTh2Op::new); - watchManager.addTarget(JobHelmTh2Op::new); + configureShutdownHook(resources, lock, condition); - watchManager.startInformers(); + try { + if (args.length > 0) { + configureLogger(args[0]); + } + Th2CrdController controller = new Th2CrdController(); + resources.add(controller); - new ContinuousTaskWorker().add(new CheckResourceCacheTask(300)); + awaitShutdown(lock, condition); } catch (Exception e) { - logger.error("Exception in main thread", e); - watchManager.stopInformers(); - watchManager.close(); - throw e; + LOGGER.error("Exception in main thread", e); + System.exit(1); + } + } + + @Override + public void close() { + Utils.close(prometheusServer, "Prometheus server"); + Utils.close(kubClient, "Kubernetes client"); + Utils.close(watchManager, "Watch manager"); + Utils.close(rabbitMQContext, "RabbitMQ context"); + Utils.close(continuousTaskWorker, "Continuous task worker"); + } + + private static void configureLogger(String filePath) { + Path path = Path.of(filePath); + if (Files.exists(path)) { + LoggerContext loggerContext = LoggerContext.getContext(false); + loggerContext.setConfigLocation(path.toUri()); + loggerContext.reconfigure(); + LOGGER.info("Logger configuration from {} file is applied", path); + } + } + + private static void configureShutdownHook(Deque resources, Lock lock, Condition condition) { + Runtime.getRuntime().addShutdownHook(new Thread( + () -> { + LOGGER.info("Shutdown start"); + lock.lock(); + try { + condition.signalAll(); + } finally { + lock.unlock(); + } + resources.descendingIterator().forEachRemaining((resource) -> { + try { + resource.close(); + } catch (Exception e) { + LOGGER.error("Cannot close resource {}", resource.getClass(), e); + } + }); + LOGGER.info("Shutdown end"); + }, + "Shutdown hook" + )); + } + + private static void awaitShutdown(Lock lock, Condition condition) throws InterruptedException { + lock.lock(); + try { + LOGGER.info("Wait shutdown"); + condition.await(); + LOGGER.info("App shutdown"); + } finally { + lock.unlock(); } } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/metrics/OperatorMetrics.java b/src/main/java/com/exactpro/th2/infraoperator/metrics/OperatorMetrics.java index ffcd66ca..3a9c5556 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/metrics/OperatorMetrics.java +++ b/src/main/java/com/exactpro/th2/infraoperator/metrics/OperatorMetrics.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,35 +25,35 @@ public class OperatorMetrics { private static final double[] TOTAL_PROCESSING_TIME_BUCKETS = {5, 10, 20, 30, 40, 50, 60, 70, 80, 120}; - private static final String KEY_DETECTION_TIME = "th2.exactpro.com/detection-time"; + public static final String KEY_DETECTION_TIME = "th2.exactpro.com/detection-time"; private static final double MILLIS_PER_SECOND = 1000; //local metrics - private static Gauge eventCounter = Gauge + private static final Gauge EVENT_COUNTER = Gauge .build("th2_infra_operator_event_queue", "Amount of events to be processed") .labelNames("exported_namespace", "category") .register(); - private static Gauge resourceCacheErrors = Gauge + private static final Gauge RESOURCE_CACHE_ERRORS = Gauge .build("th2_infra_operator_resource_cache_errors", "Amount of errors in operator resource cache") .register(); - private static Histogram crEventProcessingTime = Histogram + private static final Histogram CR_EVENT_PROCESSING_TIME = Histogram .build("th2_infra_operator_custom_resource_event_processing_time", "Time it took to process specific event by operator") .buckets(LOCAL_PROCESSING_TIME_BUCKETS) .labelNames("exported_namespace", "kind", "resName") .register(); - private static Histogram cmEventProcessingTime = Histogram + private static final Histogram CM_EVENT_PROCESSING_TIME = Histogram .build("th2_infra_operator_config_map_event_processing_time", "Time it took to process specific config map") .buckets(LOCAL_PROCESSING_TIME_BUCKETS) .labelNames("exported_namespace", "resName") .register(); - private static Histogram dictionaryEventProcessingTime = Histogram + private static final Histogram DICTIONARY_EVENT_PROCESSING_TIME = Histogram .build("th2_infra_operator_dictionary_event_processing_time", "Time it took to process dictionary") .buckets(LOCAL_PROCESSING_TIME_BUCKETS) @@ -61,7 +61,7 @@ public class OperatorMetrics { .register(); //total processing time metric - private static Histogram eventProcessingTimeTotal = Histogram + private static final Histogram EVENT_PROCESSING_TIME_TOTAL = Histogram .build("th2_infra_event_processing_total_time", "Time it took to process specific event by both manager and operator") .buckets(TOTAL_PROCESSING_TIME_BUCKETS) @@ -69,38 +69,38 @@ public class OperatorMetrics { .register(); public static void setPriorityEventCount(int value, String exportedNamespace) { - eventCounter.labels(exportedNamespace, "priority").set(value); + EVENT_COUNTER.labels(exportedNamespace, "priority").set(value); } public static void setRegularEventCount(int value, String exportedNamespace) { - eventCounter.labels(exportedNamespace, "regular").set(value); + EVENT_COUNTER.labels(exportedNamespace, "regular").set(value); } public static Histogram.Timer getCustomResourceEventTimer(HasMetadata resource) { String exportedNamespace = resource.getMetadata().getNamespace(); String resName = resource.getMetadata().getName(); String kind = resource.getKind(); - return crEventProcessingTime.labels(exportedNamespace, kind, resName).startTimer(); + return CR_EVENT_PROCESSING_TIME.labels(exportedNamespace, kind, resName).startTimer(); } public static Histogram.Timer getConfigMapEventTimer(HasMetadata resource) { String exportedNamespace = resource.getMetadata().getNamespace(); String resName = resource.getMetadata().getName(); - return cmEventProcessingTime.labels(exportedNamespace, resName).startTimer(); + return CM_EVENT_PROCESSING_TIME.labels(exportedNamespace, resName).startTimer(); } public static Histogram.Timer getDictionaryEventTimer(HasMetadata resource) { String exportedNamespace = resource.getMetadata().getNamespace(); String resName = resource.getMetadata().getName(); - return dictionaryEventProcessingTime.labels(exportedNamespace, resName).startTimer(); + return DICTIONARY_EVENT_PROCESSING_TIME.labels(exportedNamespace, resName).startTimer(); } public static void resetCacheErrors() { - resourceCacheErrors.set(0); + RESOURCE_CACHE_ERRORS.set(0); } public static void incrementCacheErrors() { - resourceCacheErrors.inc(); + RESOURCE_CACHE_ERRORS.inc(); } public static void observeTotal(HasMetadata resource) { @@ -113,6 +113,6 @@ public static void observeTotal(HasMetadata resource) { String resName = resource.getMetadata().getName(); long detectionTime = Long.parseLong(detectionTimeStr); double duration = (System.currentTimeMillis() - detectionTime) / MILLIS_PER_SECOND; - eventProcessingTimeTotal.labels(exportedNamespace, kind, resName).observe(duration); + EVENT_PROCESSING_TIME_TOTAL.labels(exportedNamespace, kind, resName).observe(duration); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/metrics/PrometheusServer.java b/src/main/java/com/exactpro/th2/infraoperator/metrics/PrometheusServer.java index ab8e91e5..8cecc6a7 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/metrics/PrometheusServer.java +++ b/src/main/java/com/exactpro/th2/infraoperator/metrics/PrometheusServer.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,40 +16,40 @@ package com.exactpro.th2.infraoperator.metrics; -import com.exactpro.th2.infraoperator.configuration.ConfigLoader; import com.exactpro.th2.infraoperator.spec.shared.PrometheusConfiguration; import io.prometheus.client.exporter.HTTPServer; import io.prometheus.client.hotspot.DefaultExports; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.concurrent.atomic.AtomicReference; -public class PrometheusServer { - private static final Logger logger = LoggerFactory.getLogger(PrometheusServer.class); +public class PrometheusServer implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusServer.class); - private static final AtomicReference prometheusExporter = new AtomicReference<>(); + @Nullable + private final HTTPServer server; - public static void start() { + static { DefaultExports.initialize(); - PrometheusConfiguration prometheusConfiguration = ConfigLoader.getConfig().getPrometheusConfiguration(); - - String host = prometheusConfiguration.getHost(); - int port = Integer.parseInt(prometheusConfiguration.getPort()); - boolean enabled = Boolean.parseBoolean(prometheusConfiguration.getEnabled()); - - prometheusExporter.updateAndGet(server -> { - if (server == null && enabled) { - try { - server = new HTTPServer(host, port); - logger.info("Started prometheus server on: \"{}:{}\"", host, port); - return server; - } catch (IOException e) { - throw new RuntimeException("Failed to create Prometheus exporter", e); - } - } - return server; - }); + } + + public PrometheusServer(PrometheusConfiguration configuration) throws IOException { + if (Boolean.parseBoolean(configuration.getEnabled())) { + String host = configuration.getHost(); + int port = Integer.parseInt(configuration.getPort()); + server = new HTTPServer(host, port); + LOGGER.info("Started prometheus server on: \"{}:{}\"", host, port); + } else { + server = null; + } + } + + @Override + public void close() { + if (server != null) { + server.close(); + } } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/model/kubernetes/client/DefaultResourceClient.java b/src/main/java/com/exactpro/th2/infraoperator/model/kubernetes/client/DefaultResourceClient.java index b3d8d979..4b961217 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/model/kubernetes/client/DefaultResourceClient.java +++ b/src/main/java/com/exactpro/th2/infraoperator/model/kubernetes/client/DefaultResourceClient.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,14 +21,10 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.Resource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public abstract class DefaultResourceClient implements ResourceClient { - private static final Logger logger = LoggerFactory.getLogger(DefaultResourceClient.class); - - private final KubernetesClient client; + private final KubernetesClient kubClient; private final Class resourceType; @@ -37,15 +33,15 @@ public abstract class DefaultResourceClient implement private final String crdName; public DefaultResourceClient( - KubernetesClient client, + KubernetesClient kubClient, Class resourceType, String crdName ) { - this.client = client; + this.kubClient = kubClient; this.resourceType = resourceType; this.crdName = crdName; - instance = client.resources(resourceType); + instance = kubClient.resources(resourceType); } @Override @@ -54,7 +50,7 @@ public Class getResourceType() { } public KubernetesClient getClient() { - return this.client; + return this.kubClient; } public MixedOperation, ? extends Resource> getInstance() { diff --git a/src/main/java/com/exactpro/th2/infraoperator/model/kubernetes/client/impl/DictionaryClient.java b/src/main/java/com/exactpro/th2/infraoperator/model/kubernetes/client/impl/DictionaryClient.java deleted file mode 100644 index 399e13e2..00000000 --- a/src/main/java/com/exactpro/th2/infraoperator/model/kubernetes/client/impl/DictionaryClient.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.exactpro.th2.infraoperator.model.kubernetes.client.impl; - -import com.exactpro.th2.infraoperator.model.kubernetes.client.DefaultResourceClient; -import com.exactpro.th2.infraoperator.spec.dictionary.Th2Dictionary; -import io.fabric8.kubernetes.client.KubernetesClient; - -public class DictionaryClient extends DefaultResourceClient { - - public DictionaryClient(KubernetesClient client) { - super( - client, - Th2Dictionary.class, - "th2dictionaries.th2.exactpro.com" - ); - } - -} diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/AbstractTh2Operator.java b/src/main/java/com/exactpro/th2/infraoperator/operator/AbstractTh2Operator.java index f354e72d..d62d04f5 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/AbstractTh2Operator.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/AbstractTh2Operator.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,16 +27,14 @@ import com.exactpro.th2.infraoperator.spec.strategy.redeploy.NonTerminalException; import com.exactpro.th2.infraoperator.spec.strategy.redeploy.RetryableTaskQueue; import com.exactpro.th2.infraoperator.spec.strategy.redeploy.tasks.TriggerRedeployTask; -import com.exactpro.th2.infraoperator.util.CustomResourceUtils; - import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.api.model.Namespace; import io.fabric8.kubernetes.api.model.OwnerReference; import io.fabric8.kubernetes.api.model.OwnerReferenceBuilder; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; +import io.fabric8.kubernetes.client.dsl.Resource; import io.prometheus.client.Histogram; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,13 +47,16 @@ import java.util.concurrent.ConcurrentHashMap; import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.ANTECEDENT_LABEL_KEY_ALIAS; +import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.annotationFor; +import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.extractHashedName; import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractName; import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractNamespace; +import static com.exactpro.th2.infraoperator.util.KubernetesUtils.isNotActive; import static io.fabric8.kubernetes.client.Watcher.Action.MODIFIED; public abstract class AbstractTh2Operator implements Watcher { - private static final Logger logger = LoggerFactory.getLogger(AbstractTh2Operator.class); + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTh2Operator.class); private static final int REDEPLOY_DELAY = 120; @@ -74,7 +75,7 @@ protected AbstractTh2Operator(KubernetesClient kubClient) { @Override public void eventReceived(Action action, CR resource) { - String resourceLabel = CustomResourceUtils.annotationFor(resource); + String resourceLabel = annotationFor(resource); try { var cachedFingerprint = fingerprints.get(resourceLabel); @@ -82,23 +83,22 @@ public void eventReceived(Action action, CR resource) { if (cachedFingerprint != null && action.equals(MODIFIED) && cachedFingerprint.equals(resourceFingerprint)) { - logger.debug("No changes detected for \"{}\"", resourceLabel); + LOGGER.debug("No changes detected for \"{}\"", resourceLabel); return; } Histogram.Timer processTimer = OperatorMetrics.getCustomResourceEventTimer(resource); try { - logger.debug("refresh-token={}", resourceFingerprint.refreshToken); + LOGGER.debug("refresh-token={}", resourceFingerprint.refreshToken); processEvent(action, resource); } catch (NonTerminalException e) { - logger.error("Non-terminal Exception processing {} event for \"{}\". Will try to redeploy.", + LOGGER.error("Non-terminal Exception processing {} event for \"{}\". Will try to redeploy.", action, resourceLabel, e); String namespace = resource.getMetadata().getNamespace(); - Namespace namespaceObj = kubClient.namespaces().withName(namespace).get(); - if (namespaceObj == null || !namespaceObj.getStatus().getPhase().equals("Active")) { - logger.info("Namespace \"{}\" deleted or not active, cancelling", namespace); + if (isNotActive(kubClient, namespace)) { + LOGGER.info("Namespace \"{}\" deleted or not active, cancelling", namespace); return; } @@ -116,7 +116,7 @@ public void eventReceived(Action action, CR resource) { ); retryableTaskQueue.add(triggerRedeployTask, true); - logger.info("Task \"{}\" added to scheduler, with delay \"{}\" seconds", + LOGGER.info("Task \"{}\" added to scheduler, with delay \"{}\" seconds", triggerRedeployTask.getName(), REDEPLOY_DELAY); } finally { fingerprints.put(resourceLabel, resourceFingerprint); @@ -128,14 +128,13 @@ public void eventReceived(Action action, CR resource) { } catch (Exception e) { String namespace = resource.getMetadata().getNamespace(); - Namespace namespaceObj = kubClient.namespaces().withName(namespace).get(); - if (namespaceObj == null || !namespaceObj.getStatus().getPhase().equals("Active")) { - logger.info("Namespace \"{}\" deleted or not active, cancelling", namespace); + if (isNotActive(kubClient, namespace)) { + LOGGER.info("Namespace \"{}\" deleted or not active, cancelling", namespace); return; } resource.getStatus().failed(e.getMessage()); updateStatus(resource); - logger.error("Terminal Exception processing {} event for {}. Will not try to redeploy", + LOGGER.error("Terminal Exception processing {} event for {}. Will not try to redeploy", action, resourceLabel, e); } } @@ -152,27 +151,36 @@ protected HelmRelease loadKubObj() { protected void processEvent(Action action, CR resource) throws IOException { - String resourceLabel = CustomResourceUtils.annotationFor(resource); - logger.debug("Processing event {} for \"{}\"", action, resourceLabel); + String resourceLabel = annotationFor(resource); + LOGGER.debug("Processing event {} for \"{}\"", action, resourceLabel); if (resource.getSpec().getDisabled()) { try { - logger.info("Resource \"{}\" has been disabled, executing DELETE action", resourceLabel); deletedEvent(resource); // TODO: work with Th2CustomResource should be encapsulated somewhere // to void issues with choosing the right function to extract the name - kubClient.resources(HelmRelease.class) - .inNamespace(extractNamespace(resource)) + String namespace = extractNamespace(resource); + String helmReleaseName = extractHashedName(resource); + Resource helmReleaseResource = kubClient.resources(HelmRelease.class) + .inNamespace(namespace) // name must be hashed if it exceeds the limit - .withName(CustomResourceUtils.extractHashedName(resource)) - .delete(); + .withName(helmReleaseName); + HelmRelease helmRelease = helmReleaseResource.get(); + if (helmRelease == null) { + LOGGER.info("Resource \"{}\" hasn't been deleted because it already doesn't exist", + annotationFor(namespace, helmReleaseName, HelmRelease.class.getSimpleName())); + } else { + String helmReleaseLabel = annotationFor(helmRelease); + helmReleaseResource.delete(); + LOGGER.info("Resource \"{}\" has been deleted", helmReleaseLabel); + } resource.getStatus().disabled("Resource has been disabled"); + LOGGER.info("Resource \"{}\" has been disabled, executing DELETE action", resourceLabel); updateStatus(resource); - logger.info("Resource \"{}\" has been deleted", resourceLabel); } catch (Exception e) { resource.getStatus().failed("Unknown error"); updateStatus(resource); - logger.error("Exception while processing disable feature for: \"{}\"", resourceLabel, e); + LOGGER.error("Exception while processing disable feature for: \"{}\"", resourceLabel, e); } return; } @@ -182,23 +190,23 @@ protected void processEvent(Action action, CR resource) throws IOException { resource.getStatus().installing(); resource = updateStatus(resource); addedEvent(resource); - logger.info("Resource \"{}\" has been added", resourceLabel); + LOGGER.info("Resource \"{}\" has been added", resourceLabel); break; case MODIFIED: resource.getStatus().upgrading(); resource = updateStatus(resource); modifiedEvent(resource); - logger.info("Resource \"{}\" has been modified", resourceLabel); + LOGGER.info("Resource \"{}\" has been modified", resourceLabel); break; case DELETED: deletedEvent(resource); - logger.info("Resource \"{}\" has been deleted", resourceLabel); + LOGGER.info("Resource \"{}\" has been deleted", resourceLabel); break; case ERROR: - logger.warn("Error while processing \"{}\"", resourceLabel); + LOGGER.warn("Error while processing \"{}\"", resourceLabel); resource.getStatus().failed("Unknown error from kubernetes"); resource = updateStatus(resource); errorEvent(resource); @@ -218,23 +226,23 @@ protected void deletedEvent(CR resource) { // kubernetes objects will be removed when custom resource removed (through 'OwnerReference') - String resourceLabel = CustomResourceUtils.annotationFor(resource); + String resourceLabel = annotationFor(resource); fingerprints.remove(resourceLabel); // The HelmRelease name is hashed if Th2CustomResource name exceeds the limit OperatorState.INSTANCE.removeHelmReleaseFromCache( - CustomResourceUtils.extractHashedName(resource), + extractHashedName(resource), extractNamespace(resource) ); } protected void errorEvent(CR resource) { - String resourceLabel = CustomResourceUtils.annotationFor(resource); + String resourceLabel = annotationFor(resource); fingerprints.remove(resourceLabel); } protected CR updateStatus(CR resource) { - String resourceLabel = CustomResourceUtils.annotationFor(resource); + String resourceLabel = annotationFor(resource); var resClient = getResourceClient().getInstance(); try { @@ -243,7 +251,7 @@ protected CR updateStatus(CR resource) { } catch (KubernetesClientException e) { if (HttpCode.ofCode(e.getCode()) == HttpCode.SERVER_CONFLICT) { - logger.warn("Failed to update status for \"{}\" to \"{}\" because it has been " + + LOGGER.warn("Failed to update status for \"{}\" to \"{}\" because it has been " + "already changed on the server. Trying to sync a resource...", resourceLabel, resource.getStatus().getPhase()); var freshRes = resClient.inNamespace(extractNamespace(resource)).list().getItems().stream() @@ -254,11 +262,11 @@ protected CR updateStatus(CR resource) { freshRes.setStatus(resource.getStatus()); var updatedRes = updateStatus(freshRes); fingerprints.put(resourceLabel, new ResourceFingerprint(updatedRes)); - logger.info("Status for \"{}\" resource successfully updated to \"{}\"", + LOGGER.info("Status for \"{}\" resource successfully updated to \"{}\"", resourceLabel, resource.getStatus().getPhase()); return updatedRes; } else { - logger.warn("Unable to update status for \"{}\" resource to \"{}\": resource not present", + LOGGER.warn("Unable to update status for \"{}\" resource to \"{}\": resource not present", resourceLabel, resource.getStatus().getPhase()); return resource; } @@ -268,7 +276,7 @@ protected CR updateStatus(CR resource) { } } - protected void setupAndCreateKubObj(CR resource) throws IOException { + protected void setupAndCreateKubObj(CR resource) { var kubObj = loadKubObj(); @@ -276,9 +284,7 @@ protected void setupAndCreateKubObj(CR resource) throws IOException { createKubObj(extractNamespace(resource), kubObj); - logger.info("Generated \"{}\" based on \"{}\"" - , CustomResourceUtils.annotationFor(kubObj) - , CustomResourceUtils.annotationFor(resource)); + LOGGER.info("Generated \"{}\" based on \"{}\"" , annotationFor(kubObj) , annotationFor(resource)); String kubObjType = kubObj.getClass().getSimpleName(); @@ -291,15 +297,15 @@ protected void setupKubObj(CR resource, HelmRelease helmRelease) { mapProperties(resource, helmRelease); - logger.info("Generated additional properties from \"{}\" for the resource \"{}\"" - , CustomResourceUtils.annotationFor(resource) - , CustomResourceUtils.annotationFor(helmRelease)); + LOGGER.info("Generated additional properties from \"{}\" for the resource \"{}\"" + , annotationFor(resource) + , annotationFor(helmRelease)); helmRelease.getMetadata().setOwnerReferences(List.of(createOwnerReference(resource))); - logger.info("Property \"OwnerReference\" with reference to \"{}\" has been set for the resource \"{}\"" - , CustomResourceUtils.annotationFor(resource) - , CustomResourceUtils.annotationFor(helmRelease)); + LOGGER.info("Property \"OwnerReference\" with reference to \"{}\" has been set for the resource \"{}\"" + , annotationFor(resource) + , annotationFor(helmRelease)); } @@ -308,11 +314,11 @@ protected void mapProperties(CR resource, HelmRelease helmRelease) { var kubObjMD = helmRelease.getMetadata(); var resMD = resource.getMetadata(); String resName = resMD.getName(); - String annotation = CustomResourceUtils.annotationFor(resource); - String finalName = CustomResourceUtils.extractHashedName(resource); + String annotation = annotationFor(resource); + String finalName = extractHashedName(resource); if (!finalName.equals(resName)) { - logger.info("Name of resource \"{}\" exceeds limitations. Will be substituted with \"{}\"", + LOGGER.info("Name of resource \"{}\" exceeds limitations. Will be substituted with \"{}\"", annotation, finalName); } diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/GenericHelmTh2Op.java b/src/main/java/com/exactpro/th2/infraoperator/operator/GenericHelmTh2Op.java index 88e6b354..6be079d6 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/GenericHelmTh2Op.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/GenericHelmTh2Op.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,13 @@ package com.exactpro.th2.infraoperator.operator; import com.exactpro.th2.infraoperator.spec.Th2CustomResource; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import io.fabric8.kubernetes.client.KubernetesClient; public abstract class GenericHelmTh2Op extends HelmReleaseTh2Op { - public GenericHelmTh2Op(KubernetesClient client) { - super(client); + public GenericHelmTh2Op(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + super(kubClient, rabbitMQContext); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/HelmReleaseTh2Op.java b/src/main/java/com/exactpro/th2/infraoperator/operator/HelmReleaseTh2Op.java index 02bf247c..bd0e44bb 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/HelmReleaseTh2Op.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/HelmReleaseTh2Op.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,7 @@ import com.exactpro.th2.infraoperator.spec.shared.PrometheusConfiguration; import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.BindQueueLinkResolver; import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.DeclareQueueResolver; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import com.exactpro.th2.infraoperator.util.JsonUtils; @@ -65,25 +66,25 @@ public abstract class HelmReleaseTh2Op extends Abs public static final String ANNOTATIONS_ALIAS = "commonAnnotations"; //component section - private static final String ROOTLESS_ALIAS = "rootless"; + public static final String ROOTLESS_ALIAS = "rootless"; - private static final String COMPONENT_NAME_ALIAS = "name"; + public static final String COMPONENT_NAME_ALIAS = "name"; - private static final String DOCKER_IMAGE_ALIAS = "image"; + public static final String DOCKER_IMAGE_ALIAS = "image"; - private static final String CUSTOM_CONFIG_ALIAS = "custom"; + public static final String CUSTOM_CONFIG_ALIAS = "custom"; - private static final String SECRET_VALUES_CONFIG_ALIAS = "secretValuesConfig"; + public static final String SECRET_VALUES_CONFIG_ALIAS = "secretValuesConfig"; - private static final String SECRET_PATHS_CONFIG_ALIAS = "secretPathsConfig"; + public static final String SECRET_PATHS_CONFIG_ALIAS = "secretPathsConfig"; - private static final String PROMETHEUS_CONFIG_ALIAS = "prometheus"; + public static final String PROMETHEUS_CONFIG_ALIAS = "prometheus"; public static final String DICTIONARIES_ALIAS = "dictionaries"; public static final String MQ_QUEUE_CONFIG_ALIAS = "mq"; - private static final String GRPC_P2P_CONFIG_ALIAS = "grpc"; + public static final String GRPC_P2P_CONFIG_ALIAS = "grpc"; public static final String MQ_ROUTER_ALIAS = "mqRouter"; @@ -126,13 +127,18 @@ public abstract class HelmReleaseTh2Op extends Abs protected final MixedOperation, Resource> helmReleaseClient; - public HelmReleaseTh2Op(KubernetesClient client) { + protected final DeclareQueueResolver declareQueueResolver; - super(client); + protected final BindQueueLinkResolver bindQueueLinkResolver; - this.grpcConfigFactory = new GrpcRouterConfigFactory(); + public HelmReleaseTh2Op(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + + super(kubClient); - helmReleaseClient = kubClient.resources(HelmRelease.class); + this.grpcConfigFactory = new GrpcRouterConfigFactory(); + this.helmReleaseClient = this.kubClient.resources(HelmRelease.class); + this.declareQueueResolver = new DeclareQueueResolver(rabbitMQContext); + this.bindQueueLinkResolver = new BindQueueLinkResolver(rabbitMQContext); } public abstract SharedIndexInformer generateInformerFromFactory(SharedInformerFactory factory); @@ -324,11 +330,11 @@ protected void addedEvent(CR resource) throws IOException { String namespace = extractNamespace(resource); var lock = OperatorState.INSTANCE.getLock(namespace); + lock.lock(); try { - lock.lock(); - DeclareQueueResolver.resolveAdd(resource); - BindQueueLinkResolver.resolveDeclaredLinks(resource); - BindQueueLinkResolver.resolveHiddenLinks(resource); + declareQueueResolver.resolveAdd(resource); + bindQueueLinkResolver.resolveDeclaredLinks(resource); + bindQueueLinkResolver.resolveHiddenLinks(resource); updateGrpcLinkedResourcesIfNeeded(resource); super.addedEvent(resource); } finally { @@ -342,12 +348,11 @@ protected void modifiedEvent(CR resource) throws IOException { String namespace = extractNamespace(resource); var lock = OperatorState.INSTANCE.getLock(namespace); + lock.lock(); try { - lock.lock(); - - DeclareQueueResolver.resolveAdd(resource); - BindQueueLinkResolver.resolveDeclaredLinks(resource); - BindQueueLinkResolver.resolveHiddenLinks(resource); + declareQueueResolver.resolveAdd(resource); + bindQueueLinkResolver.resolveDeclaredLinks(resource); + bindQueueLinkResolver.resolveHiddenLinks(resource); updateGrpcLinkedResourcesIfNeeded(resource); super.modifiedEvent(resource); } finally { @@ -359,10 +364,10 @@ protected void modifiedEvent(CR resource) throws IOException { protected void deletedEvent(CR resource) { var lock = OperatorState.INSTANCE.getLock(extractNamespace(resource)); + lock.lock(); try { - lock.lock(); super.deletedEvent(resource); - DeclareQueueResolver.resolveDelete(resource); + declareQueueResolver.resolveDelete(resource); } finally { lock.unlock(); } diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/StoreHelmTh2Op.java b/src/main/java/com/exactpro/th2/infraoperator/operator/StoreHelmTh2Op.java index c2c9e44d..abc1b916 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/StoreHelmTh2Op.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/StoreHelmTh2Op.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import com.exactpro.th2.infraoperator.OperatorState; import com.exactpro.th2.infraoperator.spec.Th2CustomResource; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import io.fabric8.kubernetes.client.KubernetesClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,7 +29,7 @@ public abstract class StoreHelmTh2Op extends HelmReleaseTh2Op { - private static final Logger logger = LoggerFactory.getLogger(StoreHelmTh2Op.class); + private static final Logger LOGGER = LoggerFactory.getLogger(StoreHelmTh2Op.class); public static final String EVENT_STORAGE_PIN_ALIAS = "estore-pin"; @@ -38,17 +39,16 @@ public abstract class StoreHelmTh2Op extends HelmR public static final String MESSAGE_STORAGE_BOX_ALIAS = "mstore"; - public StoreHelmTh2Op(KubernetesClient client) { - super(client); + public StoreHelmTh2Op(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + super(kubClient, rabbitMQContext); } private void nameCheck(CR resource) throws IOException { var msNamespace = extractNamespace(resource); var lock = OperatorState.INSTANCE.getLock(msNamespace); + lock.lock(); try { - lock.lock(); - var msName = extractName(resource); var stName = getStorageName(); @@ -57,7 +57,7 @@ private void nameCheck(CR resource) throws IOException { var msg = String.format("%s<%s.%s> has an invalid name, must be '%s'", extractType(resource), msNamespace, msName, stName); - logger.warn(msg); + LOGGER.warn(msg); resource.getStatus().failed(msg); updateStatus(resource); return; diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/BoxHelmTh2Op.java b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/BoxHelmTh2Op.java index c1e188bb..9a35114d 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/BoxHelmTh2Op.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/BoxHelmTh2Op.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import com.exactpro.th2.infraoperator.model.kubernetes.client.impl.BoxClient; import com.exactpro.th2.infraoperator.operator.GenericHelmTh2Op; import com.exactpro.th2.infraoperator.spec.box.Th2Box; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.informers.SharedIndexInformer; @@ -31,9 +32,9 @@ public class BoxHelmTh2Op extends GenericHelmTh2Op { private final BoxClient boxClient; - public BoxHelmTh2Op(KubernetesClient client) { - super(client); - this.boxClient = new BoxClient(client); + public BoxHelmTh2Op(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + super(kubClient, rabbitMQContext); + this.boxClient = new BoxClient(kubClient); } @Override diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/CoreBoxHelmTh2Op.java b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/CoreBoxHelmTh2Op.java index 4e01f53f..19d47f8a 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/CoreBoxHelmTh2Op.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/CoreBoxHelmTh2Op.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import com.exactpro.th2.infraoperator.model.kubernetes.client.impl.CoreBoxClient; import com.exactpro.th2.infraoperator.operator.GenericHelmTh2Op; import com.exactpro.th2.infraoperator.spec.corebox.Th2CoreBox; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.informers.SharedIndexInformer; @@ -31,9 +32,9 @@ public class CoreBoxHelmTh2Op extends GenericHelmTh2Op { private final CoreBoxClient coreBoxClient; - public CoreBoxHelmTh2Op(KubernetesClient client) { - super(client); - this.coreBoxClient = new CoreBoxClient(client); + public CoreBoxHelmTh2Op(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + super(kubClient, rabbitMQContext); + this.coreBoxClient = new CoreBoxClient(kubClient); } @Override diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/EstoreHelmTh2Op.java b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/EstoreHelmTh2Op.java index fa863d54..99018cce 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/EstoreHelmTh2Op.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/EstoreHelmTh2Op.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import com.exactpro.th2.infraoperator.model.kubernetes.client.impl.EstoreClient; import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op; import com.exactpro.th2.infraoperator.spec.estore.Th2Estore; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.informers.SharedIndexInformer; @@ -31,9 +32,9 @@ public class EstoreHelmTh2Op extends StoreHelmTh2Op { private final EstoreClient estoreClient; - public EstoreHelmTh2Op(KubernetesClient client) { - super(client); - this.estoreClient = new EstoreClient(client); + public EstoreHelmTh2Op(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + super(kubClient, rabbitMQContext); + this.estoreClient = new EstoreClient(kubClient); } @Override diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/JobHelmTh2Op.java b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/JobHelmTh2Op.java index a46e1603..41c29d62 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/JobHelmTh2Op.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/JobHelmTh2Op.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import com.exactpro.th2.infraoperator.model.kubernetes.client.impl.JobClient; import com.exactpro.th2.infraoperator.operator.GenericHelmTh2Op; import com.exactpro.th2.infraoperator.spec.job.Th2Job; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.informers.SharedIndexInformer; @@ -31,9 +32,9 @@ public class JobHelmTh2Op extends GenericHelmTh2Op { private final JobClient jobClient; - public JobHelmTh2Op(KubernetesClient client) { - super(client); - this.jobClient = new JobClient(client); + public JobHelmTh2Op(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + super(kubClient, rabbitMQContext); + this.jobClient = new JobClient(kubClient); } @Override diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/MstoreHelmTh2Op.java b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/MstoreHelmTh2Op.java index e0763f9f..19a80201 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/impl/MstoreHelmTh2Op.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/impl/MstoreHelmTh2Op.java @@ -22,6 +22,7 @@ import com.exactpro.th2.infraoperator.model.kubernetes.client.impl.MstoreClient; import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op; import com.exactpro.th2.infraoperator.spec.mstore.Th2Mstore; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.informers.SharedIndexInformer; @@ -31,9 +32,9 @@ public class MstoreHelmTh2Op extends StoreHelmTh2Op { private final MstoreClient mstoreClient; - public MstoreHelmTh2Op(KubernetesClient client) { - super(client); - this.mstoreClient = new MstoreClient(client); + public MstoreHelmTh2Op(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + super(kubClient, rabbitMQContext); + this.mstoreClient = new MstoreClient(kubClient); } @Override diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/ConfigMapEventHandler.java b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/ConfigMapEventHandler.java index e5bef611..cdecc816 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/ConfigMapEventHandler.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/ConfigMapEventHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,6 @@ import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import com.exactpro.th2.infraoperator.util.ExtractUtils; import com.exactpro.th2.infraoperator.util.HelmReleaseUtils; -import com.exactpro.th2.infraoperator.util.Strings; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectReader; @@ -45,15 +44,25 @@ import io.fabric8.kubernetes.client.informers.SharedIndexInformer; import io.fabric8.kubernetes.client.informers.SharedInformerFactory; import io.prometheus.client.Histogram; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.Base64; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; import static com.exactpro.th2.infraoperator.configuration.OperatorConfig.RABBITMQ_SECRET_PASSWORD_KEY; -import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.*; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.CHECKSUM_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.CONFIG_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.CRADLE_MGR_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.GRPC_ROUTER_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.LOGGING_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.MQ_ROUTER_ALIAS; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.annotationFor; import static com.exactpro.th2.infraoperator.util.JsonUtils.JSON_MAPPER; +import static com.exactpro.th2.infraoperator.util.WatcherUtils.createExceptionHandler; public class ConfigMapEventHandler implements Watcher { public static final String SECRET_TYPE_OPAQUE = "Opaque"; @@ -74,7 +83,7 @@ public class ConfigMapEventHandler implements Watcher { public static final String BOOK_CONFIG_CM_NAME = "book-config"; - private static final String DEFAULT_BOOK = "defaultBook"; + public static final String DEFAULT_BOOK = "defaultBook"; private static final Map cmMapping = Map.of( LOGGING_CM_NAME, new ConfigMapMeta(LOGGING_ALIAS, ""), @@ -94,33 +103,43 @@ private ConfigMapMeta(String alias, String dataFileName) { } } - private static final Logger logger = LoggerFactory.getLogger(ConfigMapEventHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigMapEventHandler.class); - private KubernetesClient client; + private final KubernetesClient kubClient; + + private final RabbitMQContext rabbitMQContext; + + private final DefaultWatchManager watchManager; private MixedOperation, Resource> helmReleaseClient; public KubernetesClient getClient() { - return client; + return kubClient; } public static ConfigMapEventHandler newInstance(SharedInformerFactory sharedInformerFactory, KubernetesClient client, + RabbitMQContext rabbitMQContext, + DefaultWatchManager watchManager, EventQueue eventQueue) { - var res = new ConfigMapEventHandler(client); - res.client = client; + var res = new ConfigMapEventHandler(client, rabbitMQContext, watchManager); res.helmReleaseClient = client.resources(HelmRelease.class); SharedIndexInformer configMapInformer = sharedInformerFactory.sharedIndexInformerFor( ConfigMap.class, CustomResourceUtils.RESYNC_TIME); + configMapInformer.exceptionHandler(createExceptionHandler(ConfigMap.class)); configMapInformer.addEventHandler(new GenericResourceEventHandler<>(res, eventQueue)); return res; } - private ConfigMapEventHandler(KubernetesClient client) { - this.client = client; + private ConfigMapEventHandler(KubernetesClient kubClient, + RabbitMQContext rabbitMQContext, + DefaultWatchManager watchManager) { + this.kubClient = kubClient; + this.rabbitMQContext = rabbitMQContext; + this.watchManager = watchManager; } @Override @@ -132,18 +151,18 @@ public void eventReceived(Action action, ConfigMap resource) { if (configMapName.equals(ConfigLoader.getConfig().getRabbitMQConfigMapName())) { try { - logger.info("Processing {} event for \"{}\"", action, resourceLabel); + LOGGER.info("Processing {} event for \"{}\"", action, resourceLabel); var lock = OperatorState.INSTANCE.getLock(namespace); + lock.lock(); try { - lock.lock(); OperatorConfig opConfig = ConfigLoader.getConfig(); ConfigMaps configMaps = ConfigMaps.INSTANCE; RabbitMQConfig rabbitMQConfig = configMaps.getRabbitMQConfig4Namespace(namespace); String configContent = resource.getData().get(RabbitMQConfig.CONFIG_MAP_RABBITMQ_PROP_NAME); - if (Strings.isNullOrEmpty(configContent)) { - logger.error("Key \"{}\" not found in \"{}\"", RabbitMQConfig.CONFIG_MAP_RABBITMQ_PROP_NAME, + if (StringUtils.isBlank(configContent)) { + LOGGER.error("Key \"{}\" not found in \"{}\"", RabbitMQConfig.CONFIG_MAP_RABBITMQ_PROP_NAME, resourceLabel); return; } @@ -155,20 +174,20 @@ public void eventReceived(Action action, ConfigMap resource) { if (!Objects.equals(rabbitMQConfig, newRabbitMQConfig)) { Histogram.Timer processTimer = OperatorMetrics.getConfigMapEventTimer(resource); configMaps.setRabbitMQConfig4Namespace(namespace, newRabbitMQConfig); - RabbitMQContext.setUpRabbitMqForNamespace(namespace); - logger.info("RabbitMQ ConfigMap has been updated in namespace \"{}\". Updating all boxes", + rabbitMQContext.setUpRabbitMqForNamespace(namespace); + LOGGER.info("RabbitMQ ConfigMap has been updated in namespace \"{}\". Updating all boxes", namespace); - DefaultWatchManager.getInstance().refreshBoxes(namespace); - logger.info("box-definition(s) have been updated"); + watchManager.refreshBoxes(namespace); + LOGGER.info("box-definition(s) have been updated"); processTimer.observeDuration(); } else { - logger.info("RabbitMQ ConfigMap data hasn't changed"); + LOGGER.info("RabbitMQ ConfigMap data hasn't changed"); } } finally { lock.unlock(); } } catch (Exception e) { - logger.error("Exception processing {} event for \"{}\"", action, resourceLabel, e); + LOGGER.error("Exception processing {} event for \"{}\"", action, resourceLabel, e); } } else if (configMapName.equals(BOOK_CONFIG_CM_NAME)) { updateDefaultBookName(action, namespace, resource, resourceLabel); @@ -182,7 +201,7 @@ public void eventReceived(Action action, ConfigMap resource) { private void updateConfigMap(Action action, String namespace, ConfigMap resource, final String cmName, String resourceLabel) { try { - logger.info("Processing {} event for \"{}\"", action, resourceLabel); + LOGGER.info("Processing {} event for \"{}\"", action, resourceLabel); if (isActionInvalid(action, resourceLabel)) { return; } @@ -192,8 +211,8 @@ private void updateConfigMap(Action action, String namespace, ConfigMap resource String dataFileName = titles.dataFileName; var lock = OperatorState.INSTANCE.getLock(namespace); + lock.lock(); try { - lock.lock(); String oldChecksum = OperatorState.INSTANCE.getConfigChecksum(namespace, alias); String newChecksum = ExtractUtils.fullSourceHash(resource); if (!newChecksum.equals(oldChecksum)) { @@ -204,27 +223,27 @@ private void updateConfigMap(Action action, String namespace, ConfigMap resource cmData = resource.getData().get(dataFileName); OperatorState.INSTANCE.putConfigData(namespace, alias, cmData); } - logger.info("\"{}\" has been updated. Updating all boxes", resourceLabel); + LOGGER.info("\"{}\" has been updated. Updating all boxes", resourceLabel); int refreshedBoxesCount = updateResourceChecksumAndData(namespace, newChecksum, cmData, alias); - logger.info("{} HelmRelease(s) have been updated", refreshedBoxesCount); + LOGGER.info("{} HelmRelease(s) have been updated", refreshedBoxesCount); processTimer.observeDuration(); } } finally { lock.unlock(); } } catch (Exception e) { - logger.error("Exception processing {} event for \"{}\"", action, resourceLabel, e); + LOGGER.error("Exception processing {} event for \"{}\"", action, resourceLabel, e); } } private boolean isActionInvalid(Action action, String resourceLabel) { boolean isInvalid = false; if (action == Action.DELETED) { - logger.error("DELETED action is not supported for \"{}\". ", resourceLabel); + LOGGER.error("DELETED action is not supported for \"{}\". ", resourceLabel); isInvalid = true; } else if (action == Action.ERROR) { - logger.error("Received ERROR action for \"{}\" Canceling update", resourceLabel); + LOGGER.error("Received ERROR action for \"{}\" Canceling update", resourceLabel); isInvalid = true; } return isInvalid; @@ -232,29 +251,29 @@ private boolean isActionInvalid(Action action, String resourceLabel) { private void updateDefaultBookName(Action action, String namespace, ConfigMap resource, String resourceLabel) { try { - logger.info("Processing {} event for \"{}\"", action, resourceLabel); + LOGGER.info("Processing {} event for \"{}\"", action, resourceLabel); if (isActionInvalid(action, resourceLabel)) { return; } var lock = OperatorState.INSTANCE.getLock(namespace); + lock.lock(); try { - lock.lock(); String oldBookName = OperatorState.INSTANCE.getBookName(namespace); String newBookName = resource.getData().get(DEFAULT_BOOK); if (!newBookName.equals(oldBookName)) { Histogram.Timer processTimer = OperatorMetrics.getConfigMapEventTimer(resource); OperatorState.INSTANCE.setBookName(namespace, newBookName); - logger.info("\"{}\" has been updated. Updating all boxes", resourceLabel); - DefaultWatchManager.getInstance().refreshBoxes(namespace); - logger.info("box-definition(s) have been updated"); + LOGGER.info("\"{}\" has been updated. Updating all boxes", resourceLabel); + watchManager.refreshBoxes(namespace); + LOGGER.info("box-definition(s) have been updated"); processTimer.observeDuration(); } } finally { lock.unlock(); } } catch (Exception e) { - logger.error("Exception processing {} event for \"{}\"", action, resourceLabel, e); + LOGGER.error("Exception processing {} event for \"{}\"", action, resourceLabel, e); } } @@ -276,9 +295,9 @@ private int updateResourceChecksumAndData(String namespace, String checksum, Str config.put(CHECKSUM_ALIAS, checksum); hr.addComponentValue(key, config); - logger.debug("Updating \"{}\" resource", CustomResourceUtils.annotationFor(hr)); + LOGGER.debug("Updating \"{}\" resource", CustomResourceUtils.annotationFor(hr)); createKubObj(namespace, hr); - logger.debug("\"{}\" Updated", CustomResourceUtils.annotationFor(hr)); + LOGGER.debug("\"{}\" Updated", CustomResourceUtils.annotationFor(hr)); } return helmReleases.size(); } @@ -288,7 +307,7 @@ protected void createKubObj(String namespace, HelmRelease helmRelease) { OperatorState.INSTANCE.putHelmReleaseInCache(helmRelease, namespace); } - private Map getConfigFromCR(CustomResource customResource, String key) { + private Map getConfigFromCR(CustomResource customResource, String key) { Th2Spec spec = (Th2Spec) customResource.getSpec(); switch (key) { case MQ_ROUTER_ALIAS: @@ -308,7 +327,7 @@ public void onClose(WatcherException cause) { private String readRabbitMQPasswordForSchema(String namespace, String secretName) throws Exception { - Secret secret = client.secrets().inNamespace(namespace).withName(secretName).get(); + Secret secret = kubClient.secrets().inNamespace(namespace).withName(secretName).get(); if (secret == null) { throw new Exception(String.format("Secret not found \"%s\"", annotationFor(namespace, "Secret", secretName))); diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/DefaultWatchManager.java b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/DefaultWatchManager.java index bf5ccf39..ba49de7a 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/DefaultWatchManager.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/DefaultWatchManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,31 +20,36 @@ import com.exactpro.th2.infraoperator.model.kubernetes.client.ResourceClient; import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op; import com.exactpro.th2.infraoperator.spec.Th2CustomResource; +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext; import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import com.exactpro.th2.infraoperator.util.Strings; import com.fasterxml.uuid.Generators; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.informers.SharedIndexInformer; import io.fabric8.kubernetes.client.informers.SharedInformerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; +import java.util.function.BiFunction; import java.util.function.Supplier; import java.util.stream.Collectors; import static com.exactpro.th2.infraoperator.operator.AbstractTh2Operator.REFRESH_TOKEN_ALIAS; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.annotationFor; -import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractName; +import static com.exactpro.th2.infraoperator.util.WatcherUtils.createExceptionHandler; -public class DefaultWatchManager { +public class DefaultWatchManager implements AutoCloseable { - private static final Logger logger = LoggerFactory.getLogger(DefaultWatchManager.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultWatchManager.class); private boolean isWatching = false; @@ -54,29 +59,29 @@ public class DefaultWatchManager { private final SharedInformerFactory sharedInformerFactory; - private static DefaultWatchManager instance; - private final EventDispatcher eventDispatcher; - private final KubernetesClient client; + private final KubernetesClient kubClient; + + private final RabbitMQContext rabbitMQContext; - private synchronized SharedInformerFactory getInformerFactory() { + private SharedInformerFactory getInformerFactory() { return sharedInformerFactory; } - private DefaultWatchManager(KubernetesClient client) { - this.sharedInformerFactory = client.informers(); + public DefaultWatchManager(KubernetesClient kubClient, RabbitMQContext rabbitMQContext) { + this.sharedInformerFactory = kubClient.informers(); this.eventDispatcher = new EventDispatcher(); - this.client = client; + this.kubClient = kubClient; + this.rabbitMQContext = rabbitMQContext; sharedInformerFactory.addSharedInformerEventListener(exception -> { - logger.error("Exception in InformerFactory : {}", exception.getMessage()); + LOGGER.error("Exception in InformerFactory : {}", exception.getMessage()); }); - instance = this; } - public void startInformers() { - logger.info("Starting all informers..."); + public synchronized void startInformers() { + LOGGER.info("Starting all informers..."); SharedInformerFactory sharedInformerFactory = getInformerFactory(); @@ -87,11 +92,11 @@ public void startInformers() { isWatching = true; sharedInformerFactory.startAllRegisteredInformers(); - logger.info("All informers has been started"); + LOGGER.info("All informers has been started"); } - public void stopInformers() { - logger.info("Shutting down informers"); + private void stopInformers() { + LOGGER.info("Shutting down informers"); getInformerFactory().stopAllRegisteredInformers(); } @@ -102,10 +107,10 @@ private void loadResources(EventHandlerContext context) { private void loadConfigMaps(EventHandlerContext context) { var configMapEventHandler = (ConfigMapEventHandler) context.getHandler(ConfigMapEventHandler.class); - List configMaps = client.configMaps().inAnyNamespace().list().getItems(); + List configMaps = kubClient.configMaps().inAnyNamespace().list().getItems(); configMaps = filterByNamespace(configMaps); for (var configMap : configMaps) { - logger.info("Loading \"{}\"", annotationFor(configMap)); + LOGGER.info("Loading \"{}\"", annotationFor(configMap)); configMapEventHandler.eventReceived(Watcher.Action.ADDED, configMap); } } @@ -133,11 +138,13 @@ private EventHandlerContext registerInformers(SharedInformerFactory sharedInform EventHandlerContext context = new EventHandlerContext(); - context.addHandler(NamespaceEventHandler.newInstance(sharedInformerFactory, eventDispatcher.getEventQueue())); - context.addHandler(Th2DictionaryEventHandler.newInstance(sharedInformerFactory, client, + context.addHandler(NamespaceEventHandler.newInstance(sharedInformerFactory, + rabbitMQContext, eventDispatcher.getEventQueue())); - context.addHandler(ConfigMapEventHandler.newInstance(sharedInformerFactory, client, + context.addHandler(Th2DictionaryEventHandler.newInstance(sharedInformerFactory, kubClient, eventDispatcher.getEventQueue())); + context.addHandler(ConfigMapEventHandler.newInstance(sharedInformerFactory, kubClient, rabbitMQContext, + this, eventDispatcher.getEventQueue())); /* resourceClients initialization should be done first @@ -157,46 +164,36 @@ private EventHandlerContext registerInformers(SharedInformerFactory sharedInform var handler = new BoxResourceEventHandler<>( helmReleaseTh2Op, eventDispatcher.getEventQueue()); - helmReleaseTh2Op.generateInformerFromFactory(getInformerFactory()).addEventHandler(handler); + + SharedIndexInformer customResourceInformer = + helmReleaseTh2Op.generateInformerFromFactory(getInformerFactory()); + customResourceInformer.exceptionHandler(createExceptionHandler(Th2CustomResource.class)); + customResourceInformer.addEventHandler(handler); context.addHandler(handler); } return context; } - public boolean isWatching() { + public synchronized boolean isWatching() { return isWatching; } - public void addTarget( - Function> operator) { + public synchronized void addTarget( + BiFunction> operator) { helmWatchersCommands.add(() -> { // T extends Th2CustomResource -> T is a Th2CustomResource @SuppressWarnings("unchecked") - var th2ResOp = (HelmReleaseTh2Op) operator.apply(client); + var th2ResOp = (HelmReleaseTh2Op) operator.apply(kubClient, rabbitMQContext); return th2ResOp; }); } - void refreshBoxes(String namespace) { - refreshBoxes(namespace, null, true); - } - - void refreshBoxes(String namespace, Set boxes) { - refreshBoxes(namespace, boxes, false); - } - - private void refreshBoxes(String namespace, Set boxes, boolean refreshAllBoxes) { - - if (!refreshAllBoxes && (boxes == null || boxes.size() == 0)) { - logger.warn("Empty set of boxes was given to refresh"); - return; - } - + synchronized void refreshBoxes(String namespace) { if (!isWatching()) { - logger.warn("Not watching for resources yet"); + LOGGER.warn("Not watching for resources yet"); return; } @@ -204,14 +201,12 @@ private void refreshBoxes(String namespace, Set boxes, boolean refreshAl for (var resourceClient : resourceClients) { var mixedOperation = resourceClient.getInstance(); for (var res : mixedOperation.inNamespace(namespace).list().getItems()) { - if (refreshAllBoxes || boxes.contains(extractName(res))) { - createResource(namespace, res, resourceClient); - refreshedBoxes++; - } + createResource(namespace, res, resourceClient); + refreshedBoxes++; } } - logger.info("{} boxes updated", refreshedBoxes); + LOGGER.info("{} boxes updated", refreshedBoxes); } private void createResource(String linkNamespace, Th2CustomResource resource, @@ -222,20 +217,16 @@ private void createResource(String linkNamespace, Th2CustomResource resource, resMeta.setAnnotations(Objects.nonNull(resMeta.getAnnotations()) ? resMeta.getAnnotations() : new HashMap<>()); resMeta.getAnnotations().put(REFRESH_TOKEN_ALIAS, refreshToken); resClient.getInstance().inNamespace(linkNamespace).resource(resource).createOrReplace(); - logger.debug("refreshed \"{}\" with refresh-token={}", + LOGGER.debug("refreshed \"{}\" with refresh-token={}", CustomResourceUtils.annotationFor(resource), refreshToken); } - public static synchronized DefaultWatchManager getInstance() { - if (instance == null) { - instance = new DefaultWatchManager(new KubernetesClientBuilder().build()); - } - - return instance; - } - - public void close() { + @Override + public synchronized void close() throws InterruptedException { + stopInformers(); eventDispatcher.interrupt(); - client.close(); + eventDispatcher.join(5_000); + resourceClients.clear(); + helmWatchersCommands.clear(); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/GenericResourceEventHandler.java b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/GenericResourceEventHandler.java index 7426b3ce..118b4970 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/GenericResourceEventHandler.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/GenericResourceEventHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,18 +29,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.util.Objects.requireNonNull; + public class GenericResourceEventHandler implements ResourceEventHandler, Watcher { private static final Logger logger = LoggerFactory.getLogger(GenericResourceEventHandler.class); - private Watcher watcher; + private final Watcher watcher; - private EventQueue eventQueue; + private final EventQueue eventQueue; private final OperatorConfig config = ConfigLoader.getConfig(); public GenericResourceEventHandler(Watcher watcher, EventQueue eventQueue) { - this.watcher = watcher; - this.eventQueue = eventQueue; + this.watcher = requireNonNull(watcher, "watcher can't be null"); + this.eventQueue = requireNonNull(eventQueue, "event queue can't be null"); } @Override @@ -146,6 +148,7 @@ public void eventReceived(Action action, T resource) { @Override public void onClose(WatcherException cause) { + logger.error("Watcher for '{}' has been closed", watcher.getClass().getSimpleName()); throw new AssertionError("This method should not be called"); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/NamespaceEventHandler.java b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/NamespaceEventHandler.java index 9a6289fb..53da675b 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/NamespaceEventHandler.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/NamespaceEventHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,26 +32,32 @@ import org.slf4j.LoggerFactory; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.RESYNC_TIME; +import static com.exactpro.th2.infraoperator.util.WatcherUtils.createExceptionHandler; public class NamespaceEventHandler implements ResourceEventHandler, Watcher { - private static final Logger logger = LoggerFactory.getLogger(NamespaceEventHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(NamespaceEventHandler.class); - private EventQueue eventQueue; + private final RabbitMQContext rabbitMQContext; + + private final EventQueue eventQueue; private final OperatorConfig config = ConfigLoader.getConfig(); public static NamespaceEventHandler newInstance(SharedInformerFactory sharedInformerFactory, + RabbitMQContext rabbitMQContext, EventQueue eventQueue) { SharedIndexInformer namespaceInformer = sharedInformerFactory.sharedIndexInformerFor( Namespace.class, RESYNC_TIME); - var res = new NamespaceEventHandler(eventQueue); + var res = new NamespaceEventHandler(rabbitMQContext, eventQueue); + namespaceInformer.exceptionHandler(createExceptionHandler(Namespace.class)); namespaceInformer.addEventHandler(res); return res; } - public NamespaceEventHandler(EventQueue eventQueue) { + public NamespaceEventHandler(RabbitMQContext rabbitMQContext, EventQueue eventQueue) { + this.rabbitMQContext = rabbitMQContext; this.eventQueue = eventQueue; } @@ -62,7 +68,7 @@ public void onAdd(Namespace namespace) { return; } - logger.debug("Received ADDED event for namespace: \"{}\"", namespace.getMetadata().getName()); + LOGGER.debug("Received ADDED event for namespace: \"{}\"", namespace.getMetadata().getName()); } @Override @@ -74,7 +80,7 @@ public void onUpdate(Namespace oldNamespace, Namespace newNamespace) { return; } - logger.debug("Received MODIFIED event for namespace: \"{}\"", newNamespace.getMetadata().getName()); + LOGGER.debug("Received MODIFIED event for namespace: \"{}\"", newNamespace.getMetadata().getName()); } @Override @@ -87,7 +93,7 @@ public void onDelete(Namespace namespace, boolean deletedFinalStateUnknown) { String resourceLabel = String.format("namespace:%s", namespaceName); String eventId = EventCounter.newEvent(); - logger.debug("Received DELETED event for namespace: \"{}\"", namespaceName); + LOGGER.debug("Received DELETED event for namespace: \"{}\"", namespaceName); eventQueue.addEvent(EventQueue.generateEvent( eventId, @@ -109,23 +115,22 @@ public void eventReceived(Action action, Namespace resource) { String resourceLabel = String.format("namespace:%s", namespaceName); + lock.lock(); try { - lock.lock(); - - logger.debug("Processing {} event for namespace: \"{}\"", action, namespaceName); - RabbitMQContext.cleanupRabbit(namespaceName); - logger.info("Deleted namespace {}", namespaceName); + LOGGER.info("Processing {} event for namespace: \"{}\"", action, namespaceName); + rabbitMQContext.cleanupRabbit(namespaceName); + LOGGER.info("Deleted namespace {}", namespaceName); } catch (Exception e) { - logger.error("Exception processing event for \"{}\"", resourceLabel, e); + LOGGER.error("Exception processing event for \"{}\"", resourceLabel, e); } finally { lock.unlock(); } long duration = System.currentTimeMillis() - startDateTime; - logger.info("Event for \"{}\" processed in {}ms", resourceLabel, duration); + LOGGER.info("Event for \"{}\" processed in {}ms", resourceLabel, duration); } catch (Exception e) { - logger.error("Exception processing event", e); + LOGGER.error("Exception processing event", e); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/Th2DictionaryEventHandler.java b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/Th2DictionaryEventHandler.java index 39638fca..0a9b8266 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/Th2DictionaryEventHandler.java +++ b/src/main/java/com/exactpro/th2/infraoperator/operator/manager/impl/Th2DictionaryEventHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ import com.exactpro.th2.infraoperator.util.CustomResourceUtils; import com.exactpro.th2.infraoperator.util.ExtractUtils; import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.Namespace; import io.fabric8.kubernetes.api.model.KubernetesResourceList; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.client.KubernetesClient; @@ -38,19 +37,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.*; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.DICTIONARIES_ALIAS; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.RESYNC_TIME; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.annotationFor; import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractName; import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractNamespace; import static com.exactpro.th2.infraoperator.util.HelmReleaseUtils.extractDictionariesConfig; +import static com.exactpro.th2.infraoperator.util.KubernetesUtils.isNotActive; +import static com.exactpro.th2.infraoperator.util.WatcherUtils.createExceptionHandler; public class Th2DictionaryEventHandler implements Watcher { - private static final Logger logger = LoggerFactory.getLogger(Th2DictionaryEventHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(Th2DictionaryEventHandler.class); private KubernetesClient kubClient; @@ -72,6 +76,7 @@ public static Th2DictionaryEventHandler newInstance(SharedInformerFactory shared sharedInformerFactory.sharedIndexInformerFor( Th2Dictionary.class, RESYNC_TIME); + dictionaryInformer.exceptionHandler(createExceptionHandler(Th2Dictionary.class)); dictionaryInformer.addEventHandler(new GenericResourceEventHandler<>(res, eventQueue)); return res; } @@ -95,7 +100,7 @@ public void eventReceived(Action action, Th2Dictionary dictionary) { } } catch (Exception e) { String resourceLabel = annotationFor(dictionary); - logger.error("Terminal Exception processing {} event for {}. Will not try to redeploy", + LOGGER.error("Terminal Exception processing {} event for {}. Will not try to redeploy", action, resourceLabel, e); } finally { //observe event processing time for only operator @@ -109,9 +114,9 @@ private void processAdded(Th2Dictionary dictionary) { String newChecksum = ExtractUtils.fullSourceHash(dictionary); //create or replace corresponding config map from Kubernetes - logger.debug("Creating config map for: \"{}\"", resourceLabel); + LOGGER.debug("Creating config map for: \"{}\"", resourceLabel); kubClient.resource(toConfigMap(dictionary)).inNamespace(namespace).createOrReplace(); - logger.debug("Created config map for: \"{}\"", resourceLabel); + LOGGER.debug("Created config map for: \"{}\"", resourceLabel); sourceHashes.put(resourceLabel, newChecksum); } @@ -123,25 +128,25 @@ private void processModified(Th2Dictionary dictionary) { String oldChecksum = sourceHashes.get(resourceLabel); if (oldChecksum != null && oldChecksum.equals(newChecksum)) { - logger.info("Dictionary: \"{}\" has not been changed", resourceLabel); + LOGGER.info("Dictionary: \"{}\" has not been changed", resourceLabel); return; } //update corresponding config map from Kubernetes - logger.debug("Updating config map for: \"{}\"", resourceLabel); + LOGGER.debug("Updating config map for: \"{}\"", resourceLabel); kubClient.resource(toConfigMap(dictionary)).inNamespace(namespace).createOrReplace(); - logger.debug("Updated config map for: \"{}\"", resourceLabel); + LOGGER.debug("Updated config map for: \"{}\"", resourceLabel); sourceHashes.put(resourceLabel, newChecksum); - logger.info("Checking bindings for \"{}\"", resourceLabel); + LOGGER.info("Checking bindings for \"{}\"", resourceLabel); var linkedResources = getLinkedResources(dictionary); int items = linkedResources.size(); if (items == 0) { - logger.info("No boxes needs to be updated"); + LOGGER.info("No boxes needs to be updated"); } else { - logger.info("{} box(es) needs to be updated", items); + LOGGER.info("{} box(es) needs to be updated", items); updateLinkedResources(dictionaryName, namespace, newChecksum, linkedResources); } } @@ -152,10 +157,10 @@ private void processDeleted(Th2Dictionary dictionary) { String resourceLabel = annotationFor(dictionary); //delete corresponding config map from Kubernetes - logger.debug("Deleting config map for: \"{}\"", resourceLabel); + LOGGER.debug("Deleting config map for: \"{}\"", resourceLabel); kubClient.configMaps().inNamespace(namespace).withName(dictionaryName).delete(); sourceHashes.remove(resourceLabel); - logger.debug("Deleted config map for: \"{}\"", resourceLabel); + LOGGER.debug("Deleted config map for: \"{}\"", resourceLabel); } private Set getLinkedResources(Th2Dictionary dictionary) { @@ -191,20 +196,19 @@ private ConfigMap toConfigMap(Th2Dictionary dictionary) { private void updateLinkedResources(String dictionaryName, String namespace, String checksum, Set linkedResources) { - Namespace namespaceObj = kubClient.namespaces().withName(namespace).get(); - if (namespaceObj == null || !namespaceObj.getStatus().getPhase().equals("Active")) { - logger.info("Namespace \"{}\" deleted or not active, cancelling", namespace); + if (isNotActive(kubClient, namespace)) { + LOGGER.info("Namespace \"{}\" deleted or not active, cancelling", namespace); return; } for (var linkedResourceName : linkedResources) { - logger.debug("Checking linked resource: '{}.{}'", namespace, linkedResourceName); + LOGGER.debug("Checking linked resource: '{}.{}'", namespace, linkedResourceName); var hr = OperatorState.INSTANCE.getHelmReleaseFromCache(linkedResourceName, namespace); if (hr == null) { - logger.info("HelmRelease of '{}.{}' resource not found in cache", namespace, linkedResourceName); + LOGGER.info("HelmRelease of '{}.{}' resource not found in cache", namespace, linkedResourceName); continue; } else { - logger.debug("Found HelmRelease \"{}\"", CustomResourceUtils.annotationFor(hr)); + LOGGER.debug("Found HelmRelease \"{}\"", CustomResourceUtils.annotationFor(hr)); } Collection dictionaryConfig = extractDictionariesConfig(hr); @@ -215,11 +219,11 @@ private void updateLinkedResources(String dictionaryName, String namespace, } } hr.addComponentValue(DICTIONARIES_ALIAS, dictionaryConfig); - logger.debug("Updating \"{}\"", CustomResourceUtils.annotationFor(hr)); + LOGGER.debug("Updating \"{}\"", CustomResourceUtils.annotationFor(hr)); createKubObj(namespace, hr); - logger.debug("Updated \"{}\"", CustomResourceUtils.annotationFor(hr)); + LOGGER.debug("Updated \"{}\"", CustomResourceUtils.annotationFor(hr)); } else { - logger.info("Dictionaries config for resource of '{}.{}' was null", namespace, linkedResourceName); + LOGGER.info("Dictionaries config for resource of '{}.{}' was null", namespace, linkedResourceName); } } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/dictionary/Th2Dictionary.java b/src/main/java/com/exactpro/th2/infraoperator/spec/dictionary/Th2Dictionary.java index ae5cb859..0f88387a 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/dictionary/Th2Dictionary.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/dictionary/Th2Dictionary.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.exactpro.th2.infraoperator.spec.dictionary; import com.exactpro.th2.infraoperator.spec.helmrelease.InstantiableMap; +import io.fabric8.kubernetes.api.model.Namespaced; import io.fabric8.kubernetes.client.CustomResource; import io.fabric8.kubernetes.model.annotation.Group; import io.fabric8.kubernetes.model.annotation.Kind; @@ -25,6 +26,6 @@ @Group("th2.exactpro.com") @Version("v2") @Kind("Th2Dictionary") -public class Th2Dictionary extends CustomResource { +public class Th2Dictionary extends CustomResource implements Namespaced { } diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/DeclareQueueResolver.java b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/DeclareQueueResolver.java index 785c878e..cfc566d7 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/DeclareQueueResolver.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/DeclareQueueResolver.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import com.exactpro.th2.infraoperator.util.HelmReleaseUtils; import com.rabbitmq.client.Channel; import com.rabbitmq.http.client.domain.QueueInfo; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,45 +37,48 @@ import java.util.List; import java.util.Set; -import static com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_BOX_ALIAS; -import static com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_PIN_ALIAS; -import static com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_BOX_ALIAS; -import static com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_PIN_ALIAS; -import static com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext.getChannel; +import static com.exactpro.th2.infraoperator.spec.strategy.linkresolver.Util.createEstoreQueue; +import static com.exactpro.th2.infraoperator.spec.strategy.linkresolver.Util.createMstoreQueue; import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractName; public class DeclareQueueResolver { - private static final Logger logger = LoggerFactory.getLogger(DeclareQueueResolver.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DeclareQueueResolver.class); - public static void resolveAdd(Th2CustomResource resource) { + private final RabbitMQContext rabbitMQContext; + + public DeclareQueueResolver(RabbitMQContext rabbitMQContext) { + this.rabbitMQContext = rabbitMQContext; + } + + public void resolveAdd(Th2CustomResource resource) { String namespace = ExtractUtils.extractNamespace(resource); try { declareQueueBunch(namespace, resource); } catch (Exception e) { String message = "Exception while working with rabbitMq"; - logger.error(message, e); + LOGGER.error(message, e); throw new NonTerminalException(message, e); } } - public static void resolveDelete(Th2CustomResource resource) { + public void resolveDelete(Th2CustomResource resource) { String namespace = ExtractUtils.extractNamespace(resource); try { - Channel channel = getChannel(); + Channel channel = rabbitMQContext.getChannel(); //get queues that are associated with current box. Set boxQueueNames = generateBoxQueues(namespace, resource); removeExtinctQueues(channel, boxQueueNames, CustomResourceUtils.annotationFor(resource), namespace); } catch (Exception e) { String message = "Exception while working with rabbitMq"; - logger.error(message, e); + LOGGER.error(message, e); throw new NonTerminalException(message, e); } } - private static void declareQueueBunch(String namespace, Th2CustomResource resource) throws IOException { + private void declareQueueBunch(String namespace, Th2CustomResource resource) throws IOException { - Channel channel = getChannel(); + Channel channel = rabbitMQContext.getChannel(); boolean persistence = ConfigLoader.getConfig().getRabbitMQManagement().getPersistence(); //get queues that are associated with current box and are not linked through Th2Link resources @@ -87,9 +91,9 @@ private static void declareQueueBunch(String namespace, Th2CustomResource resour //remove from set if pin for queue still exists. boxQueues.remove(queueName); var newQueueArguments = RabbitMQContext.generateQueueArguments(pin.getSettings()); - var currentQueue = RabbitMQContext.getQueue(queueName); + var currentQueue = rabbitMQContext.getQueue(queueName); if (currentQueue != null && !currentQueue.getArguments().equals(newQueueArguments)) { - logger.warn("Arguments for queue '{}' were modified. Recreating with new arguments", queueName); + LOGGER.warn("Arguments for queue '{}' were modified. Recreating with new arguments", queueName); channel.queueDelete(queueName); } var declareResult = channel.queueDeclare(queueName @@ -97,14 +101,14 @@ private static void declareQueueBunch(String namespace, Th2CustomResource resour , false , false , newQueueArguments); - logger.info("Queue '{}' of resource {} was successfully declared", + LOGGER.info("Queue '{}' of resource {} was successfully declared", declareResult.getQueue(), extractName(resource)); } //remove from rabbit queues that are left i.e. inactive removeExtinctQueues(channel, boxQueues, CustomResourceUtils.annotationFor(resource), namespace); } - private static Set getBoxPreviousQueues(String namespace, String boxName) { + private Set getBoxPreviousQueues(String namespace, String boxName) { HelmRelease hr = OperatorState.INSTANCE.getHelmReleaseFromCache(boxName, namespace); if (hr == null) { return getBoxQueuesFromRabbit(namespace, boxName); @@ -112,9 +116,13 @@ private static Set getBoxPreviousQueues(String namespace, String boxName return HelmReleaseUtils.extractQueues(hr.getComponentValuesSection()); } - private static Set getBoxQueuesFromRabbit(String namespace, String boxName) { + /** + * Collect all queues related to the {@code namespace} {@code boxName} component in RabbitMQ + * @return mutable set of queues + */ + private @NotNull Set getBoxQueuesFromRabbit(String namespace, String boxName) { - List queueInfoList = RabbitMQContext.getQueues(); + List queueInfoList = rabbitMQContext.getQueues(); Set queueNames = new HashSet<>(); queueInfoList.forEach(q -> { @@ -141,20 +149,20 @@ private static void removeExtinctQueues( String resourceLabel, String namespace ) { - String estoreQueue = new QueueName(namespace, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS).toString(); - String mstoreQueue = new QueueName(namespace, MESSAGE_STORAGE_BOX_ALIAS, MESSAGE_STORAGE_PIN_ALIAS).toString(); + String estoreQueue = createEstoreQueue(namespace); + String mstoreQueue = createMstoreQueue(namespace); if (!extinctQueueNames.isEmpty()) { - logger.info("Trying to delete queues associated with \"{}\"", resourceLabel); + LOGGER.info("Trying to delete queues associated with \"{}\"", resourceLabel); extinctQueueNames .stream() .filter(name -> !name.equals(estoreQueue) && !name.equals(mstoreQueue)) .forEach(queueName -> { try { channel.queueDelete(queueName); - logger.info("Deleted queue: [{}]", queueName); + LOGGER.info("Deleted queue: [{}]", queueName); } catch (IOException e) { - logger.error("Exception deleting queue: [{}]", queueName, e); + LOGGER.error("Exception deleting queue: [{}]", queueName, e); } }); } diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/RabbitMQContext.java b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/RabbitMQContext.java index 0f45c658..7710d02a 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/RabbitMQContext.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/RabbitMQContext.java @@ -29,7 +29,8 @@ import com.exactpro.th2.infraoperator.spec.strategy.redeploy.tasks.RecreateQueuesAndBindings; import com.exactpro.th2.infraoperator.spec.strategy.redeploy.tasks.RetryRabbitSetup; import com.exactpro.th2.infraoperator.spec.strategy.redeploy.tasks.RetryTopicExchangeTask; -import com.exactpro.th2.infraoperator.util.Strings; +import com.exactpro.th2.infraoperator.util.Utils; +import com.rabbitmq.client.BuiltinExchangeType; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; @@ -41,6 +42,7 @@ import com.rabbitmq.http.client.domain.ExchangeInfo; import com.rabbitmq.http.client.domain.QueueInfo; import com.rabbitmq.http.client.domain.UserPermissions; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,160 +50,175 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; -import java.util.*; - -import static com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static com.exactpro.th2.infraoperator.spec.strategy.linkresolver.Util.createEstoreQueue; +import static com.exactpro.th2.infraoperator.spec.strategy.linkresolver.Util.createMstoreQueue; import static com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.QueueName.QUEUE_NAME_REGEXP; +import static com.exactpro.th2.infraoperator.util.Strings.anyPrefixMatch; import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static java.util.Objects.requireNonNullElse; +import static org.apache.commons.lang3.StringUtils.isNoneBlank; -public final class RabbitMQContext { +public final class RabbitMQContext implements AutoCloseable { - private static final Logger logger = LoggerFactory.getLogger(RabbitMQContext.class); + private static final Logger LOGGER = LoggerFactory.getLogger(RabbitMQContext.class); private static final int RETRY_DELAY = 120; - private static volatile RabbitMQManagementConfig managementConfig; + public static final String TOPIC = BuiltinExchangeType.TOPIC.getType(); + + public static final String DIRECT = BuiltinExchangeType.DIRECT.getType(); - private static volatile ChannelContext channelContext; + private final RabbitMQManagementConfig managementConfig; - private static volatile Client rmqClient; + private final ChannelContext channelContext; + + private final Client rmqClient; private static final RetryableTaskQueue retryableTaskQueue = new RetryableTaskQueue(); - private RabbitMQContext() { + public RabbitMQContext(RabbitMQManagementConfig managementConfig) throws MalformedURLException, URISyntaxException { + this.managementConfig = managementConfig; + this.rmqClient = createClient(managementConfig); + this.channelContext = new ChannelContext(this, managementConfig); + } + + public String getTopicExchangeName() { + return managementConfig.getExchangeName(); } - public static void declareTopicExchange() { - String exchangeName = getManagementConfig().getExchangeName(); - RabbitMQManagementConfig rabbitMQManagementConfig = getManagementConfig(); + public void declareTopicExchange() { + String exchangeName = getTopicExchangeName(); try { - getChannel().exchangeDeclare(exchangeName, "topic", rabbitMQManagementConfig.getPersistence()); + getChannel().exchangeDeclare(exchangeName, TOPIC, managementConfig.getPersistence()); } catch (Exception e) { - logger.error("Exception setting up exchange: \"{}\"", exchangeName, e); - RetryTopicExchangeTask retryTopicExchangeTask = new RetryTopicExchangeTask(exchangeName, RETRY_DELAY); + LOGGER.error("Exception setting up exchange: \"{}\"", exchangeName, e); + RetryTopicExchangeTask retryTopicExchangeTask = new RetryTopicExchangeTask(this, exchangeName, RETRY_DELAY); retryableTaskQueue.add(retryTopicExchangeTask, true); - logger.info("Task \"{}\" added to scheduler, with delay \"{}\" seconds", + LOGGER.info("Task \"{}\" added to scheduler, with delay \"{}\" seconds", retryTopicExchangeTask.getName(), RETRY_DELAY); } } - public static void setUpRabbitMqForNamespace(String namespace) { + public void setUpRabbitMqForNamespace(String namespace) { try { createUser(namespace); - declareExchange(namespace); + declareTopicExchange(); + declareExchange(toExchangeName(namespace)); createStoreQueues(namespace); } catch (Exception e) { - logger.error("Exception setting up rabbitMq for namespace: \"{}\"", namespace, e); - RetryRabbitSetup retryRabbitSetup = new RetryRabbitSetup(namespace, RETRY_DELAY); + LOGGER.error("Exception setting up rabbitMq for namespace: \"{}\"", namespace, e); + RetryRabbitSetup retryRabbitSetup = new RetryRabbitSetup(this, namespace, RETRY_DELAY); retryableTaskQueue.add(retryRabbitSetup, true); - logger.info("Task \"{}\" added to scheduler, with delay \"{}\" seconds", + LOGGER.info("Task \"{}\" added to scheduler, with delay \"{}\" seconds", retryRabbitSetup.getName(), RETRY_DELAY); } } - private static void createUser(String namespace) throws Exception { + private void createUser(String namespace) { - RabbitMQManagementConfig rabbitMQManagementConfig = getManagementConfig(); RabbitMQConfig rabbitMQConfig = getRabbitMQConfig(namespace); String password = rabbitMQConfig.getPassword(); - String vHostName = rabbitMQManagementConfig.getVhostName(); + String vHostName = managementConfig.getVhostName(); - if (Strings.isNullOrEmpty(namespace)) { + if (StringUtils.isBlank(namespace)) { return; } try { - Client rmqClient = getClient(); - if (rmqClient.getVhost(vHostName) == null) { - logger.error("vHost: \"{}\" is not present", vHostName); + LOGGER.error("vHost: \"{}\" is not present", vHostName); return; } rmqClient.createUser(namespace, password.toCharArray(), new ArrayList<>()); - logger.info("Created user \"{}\" on vHost \"{}\"", namespace, vHostName); + LOGGER.info("Created user \"{}\" on vHost \"{}\"", namespace, vHostName); // set permissions - RabbitMQNamespacePermissions rabbitMQNamespacePermissions = - rabbitMQManagementConfig.getSchemaPermissions(); + RabbitMQNamespacePermissions rabbitMQNamespacePermissions = managementConfig.getSchemaPermissions(); UserPermissions permissions = new UserPermissions(); permissions.setConfigure(rabbitMQNamespacePermissions.getConfigure()); permissions.setRead(rabbitMQNamespacePermissions.getRead()); permissions.setWrite(rabbitMQNamespacePermissions.getWrite()); rmqClient.updatePermissions(vHostName, namespace, permissions); - logger.info("User \"{}\" permissions set in RabbitMQ", namespace); + LOGGER.info("User \"{}\" permissions set in RabbitMQ", namespace); } catch (Exception e) { - logger.error("Exception setting up user: \"{}\" for vHost: \"{}\"", namespace, vHostName, e); + LOGGER.error("Exception setting up user: \"{}\" for vHost: \"{}\"", namespace, vHostName, e); throw e; } } - private static void declareExchange(String exchangeName) throws Exception { - RabbitMQManagementConfig rabbitMQManagementConfig = getManagementConfig(); + private void declareExchange(String exchangeName) throws Exception { try { - getChannel().exchangeDeclare(exchangeName, "direct", rabbitMQManagementConfig.getPersistence()); + getChannel().exchangeDeclare(exchangeName, DIRECT, managementConfig.getPersistence()); } catch (Exception e) { - logger.error("Exception setting up exchange: \"{}\"", exchangeName, e); + LOGGER.error("Exception setting up exchange: \"{}\"", exchangeName, e); throw e; } } - private static void createStoreQueues(String namespace) throws Exception { + private void createStoreQueues(String namespace) throws Exception { var channel = getChannel(); - RabbitMQManagementConfig rabbitMQManagementConfig = getManagementConfig(); var declareResult = channel.queueDeclare( - new QueueName(namespace, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS).toString(), - rabbitMQManagementConfig.getPersistence(), + createEstoreQueue(namespace), + managementConfig.getPersistence(), false, false, null ); - logger.info("Queue \"{}\" was successfully declared", declareResult.getQueue()); + LOGGER.info("Queue \"{}\" was successfully declared", declareResult.getQueue()); declareResult = channel.queueDeclare( - new QueueName(namespace, MESSAGE_STORAGE_BOX_ALIAS, MESSAGE_STORAGE_PIN_ALIAS).toString(), - rabbitMQManagementConfig.getPersistence(), + createMstoreQueue(namespace), + managementConfig.getPersistence(), false, false, null ); - logger.info("Queue \"{}\" was successfully declared", declareResult.getQueue()); + LOGGER.info("Queue \"{}\" was successfully declared", declareResult.getQueue()); } - public static void cleanupRabbit(String namespace) throws Exception { - removeSchemaExchange(namespace); + public void cleanupRabbit(String namespace) { + removeSchemaExchange(toExchangeName(namespace)); removeSchemaQueues(namespace); removeSchemaUser(namespace); } - private static void removeSchemaUser(String namespace) throws Exception { - RabbitMQManagementConfig rabbitMQManagementConfig = getManagementConfig(); - - String vHostName = rabbitMQManagementConfig.getVhostName(); - - Client rmqClient = getClient(); + private void removeSchemaUser(String namespace) { + String vHostName = managementConfig.getVhostName(); if (rmqClient.getVhost(vHostName) == null) { - logger.error("vHost: \"{}\" is not present", vHostName); + LOGGER.error("vHost: \"{}\" is not present", vHostName); return; } rmqClient.deleteUser(namespace); - logger.info("Deleted user \"{}\" from vHost \"{}\"", namespace, vHostName); + LOGGER.info("Deleted user \"{}\" from vHost \"{}\"", namespace, vHostName); } - private static void removeSchemaExchange(String exchangeName) { + private void removeSchemaExchange(String exchangeName) { try { getChannel().exchangeDelete(exchangeName); } catch (Exception e) { - logger.error("Exception deleting exchange: \"{}\"", exchangeName, e); + LOGGER.error("Exception deleting exchange: \"{}\"", exchangeName, e); } } - private static void removeSchemaQueues(String namespace) { + private void removeSchemaQueues(String namespace) { try { Channel channel = getChannel(); @@ -212,69 +229,23 @@ private static void removeSchemaQueues(String namespace) { if (queue != null && queue.getNamespace().equals(namespace)) { try { channel.queueDelete(queueName); - logger.info("Deleted queue: [{}]", queueName); + LOGGER.info("Deleted queue: [{}]", queueName); } catch (IOException e) { - logger.error("Exception deleting queue: [{}]", queueName, e); + LOGGER.error("Exception deleting queue: [{}]", queueName, e); } } }); } catch (Exception e) { - logger.error("Exception cleaning up queues for: \"{}\"", namespace, e); + LOGGER.error("Exception cleaning up queues for: \"{}\"", namespace, e); } } - public static void cleanUpRabbitBeforeStart() { - try { - if (!getManagementConfig().getCleanUpOnStart()) { - logger.info("Cleanup RabbitMQ before start is skipped by config"); - return; - } - - Channel channel = getChannel(); - List namespacePrefixes = ConfigLoader.getConfig().getNamespacePrefixes(); - - List queueInfoList = getQueues(); - queueInfoList.forEach(q -> { - String queueName = q.getName(); - if (queueName != null && queueName.matches(QUEUE_NAME_REGEXP)) { - try { - channel.queueDelete(queueName); - logger.info("Deleted queue: [{}]", queueName); - } catch (IOException e) { - logger.error("Exception deleting queue: [{}]", queueName, e); - } - } - }); - - List exchangeInfoList = getExchanges(); - exchangeInfoList.forEach(e -> { - String exchangeName = e.getName(); - for (String namespacePrefix : namespacePrefixes) { - if (exchangeName.startsWith(namespacePrefix)) { - try { - channel.exchangeDelete(exchangeName); - break; - } catch (IOException ex) { - logger.error("Exception deleting exchange: [{}]", exchangeName, ex); - break; - } - } - } - }); - } catch (Exception e) { - logger.error("Exception cleaning up rabbit", e); - } + public static String toExchangeName(String namespace) { + return namespace; } - public static Channel getChannel() { - Channel channel = getChannelContext().channel; - if (!channel.isOpen()) { - logger.warn("RabbitMQ connection is broken, trying to reconnect..."); - getChannelContext().close(); - channel = getChannelContext().channel; - logger.info("RabbitMQ connection has been restored"); - } - return channel; + public Channel getChannel() { + return channelContext.getChannel(); } public static Map generateQueueArguments(PinSettings pinSettings) throws NumberFormatException { @@ -292,84 +263,89 @@ public static Map generateQueueArguments(PinSettings pinSettings } } - public static List getQueues() { - - String vHostName = getManagementConfig().getVhostName(); + public @NotNull List getQueues() { + String vHostName = managementConfig.getVhostName(); try { - Client rmqClient = getClient(); - return rmqClient.getQueues(vHostName); + return requireNonNullElse(rmqClient.getQueues(vHostName), emptyList()); } catch (Exception e) { String message = "Exception while fetching queues"; - logger.error(message, e); + LOGGER.error(message, e); throw new NonTerminalException(message, e); } } - public static List getQueueBindings(String queue) { - String vHostName = getManagementConfig().getVhostName(); + public @NotNull List getTh2Queues() { + return getQueues().stream() + .filter(queueInfo -> queueInfo.getName() != null && queueInfo.getName().matches(QUEUE_NAME_REGEXP)) + .collect(Collectors.toList()); + } + + public List getQueueBindings(String queue) { + String vHostName = managementConfig.getVhostName(); try { - Client rmqClient = getClient(); return rmqClient.getQueueBindings(vHostName, queue); } catch (Exception e) { String message = "Exception while fetching bindings"; - logger.error(message, e); + LOGGER.error(message, e); throw new NonTerminalException(message, e); } } - public static List getExchanges() { - + public @NotNull List getExchanges() { try { - Client rmqClient = getClient(); return rmqClient.getExchanges(); } catch (Exception e) { String message = "Exception while fetching exchanges"; - logger.error(message, e); + LOGGER.error(message, e); throw new NonTerminalException(message, e); } } - public static QueueInfo getQueue(String queueName) { + public @NotNull List getTh2Exchanges() { + Collection namespacePrefixes = ConfigLoader.getConfig().getNamespacePrefixes(); + String topicExchange = getTopicExchangeName(); + return getExchanges().stream() + .filter(exchangeInfo -> { + String name = exchangeInfo.getName(); + return isNoneBlank(name) + && (name.equals(topicExchange) || anyPrefixMatch(name, namespacePrefixes)); + + }).collect(Collectors.toList()); + } + + public QueueInfo getQueue(String queueName) { - String vHostName = getManagementConfig().getVhostName(); + String vHostName = managementConfig.getVhostName(); try { - Client rmqClient = getClient(); return rmqClient.getQueue(vHostName, queueName); } catch (Exception e) { String message = "Exception while fetching queue"; - logger.error(message, e); + LOGGER.error(message, e); throw new NonTerminalException(message, e); } } - private static RabbitMQManagementConfig getManagementConfig() { - // we do not need to synchronize as we are assigning immutable object from singleton - if (managementConfig == null) { - managementConfig = ConfigLoader.getConfig().getRabbitMQManagement(); - } - return managementConfig; - } - - private static Client getClient() throws MalformedURLException, URISyntaxException { - if (rmqClient == null) { - RabbitMQManagementConfig rabbitMQMngConfig = getManagementConfig(); - String apiStr = "http://%s:%s/api"; - rmqClient = new Client(new ClientParameters() - .url(format(apiStr, rabbitMQMngConfig.getHost(), rabbitMQMngConfig.getManagementPort())) - .username(rabbitMQMngConfig.getUsername()) - .password(rabbitMQMngConfig.getPassword()) + public static Client createClient( + String host, + int port, + String username, + String password + ) throws MalformedURLException, URISyntaxException { + return new Client(new ClientParameters() + .url(format("http://%s:%s/api", host, port)) + .username(username) + .password(password) ); - } - return rmqClient; } - private static ChannelContext getChannelContext() { - // we do not need to synchronize as we are assigning immutable object from singleton - if (channelContext == null) { - channelContext = new ChannelContext(); - } - return channelContext; + private static Client createClient( + RabbitMQManagementConfig rabbitMQMngConfig + ) throws MalformedURLException, URISyntaxException { + return createClient(rabbitMQMngConfig.getHost(), + rabbitMQMngConfig.getManagementPort(), + rabbitMQMngConfig.getUsername(), + rabbitMQMngConfig.getPassword()); } private static RabbitMQConfig getRabbitMQConfig(String namespace) { @@ -382,49 +358,77 @@ private static RabbitMQConfig getRabbitMQConfig(String namespace) { return rabbitMQConfig; } - static class ChannelContext { + @Override + public void close() { + Utils.close(channelContext, "AMQP channel context"); + } + + static class ChannelContext implements AutoCloseable { + + private final Lock lock = new ReentrantLock(); + + private final RabbitMQContext rabbitMQContext; + + private final RabbitMQManagementConfig rabbitMQManagementConfig; private Connection connection; private Channel channel; - ChannelContext() { - ConnectionFactory connectionFactory = createConnectionFactory(); + ChannelContext(RabbitMQContext rabbitMQContext, RabbitMQManagementConfig rabbitMQManagementConfig) { + this.rabbitMQContext = rabbitMQContext; + this.rabbitMQManagementConfig = rabbitMQManagementConfig; + getChannel(); + } + + public Channel getChannel() { + lock.lock(); try { - this.connection = connectionFactory.newConnection(); - this.connection.addShutdownListener(new RmqClientShutdownEventListener()); - this.channel = connection.createChannel(); + if (connection == null || !connection.isOpen()) { + close(); + LOGGER.warn("RabbitMQ connection is broken, trying to reconnect..."); + connection = createConnection(); + } + if (channel == null || !channel.isOpen()) { + channel = connection.createChannel(); + } + return channel; } catch (Exception e) { close(); String message = "Exception while creating rabbitMq channel"; - logger.error(message, e); + LOGGER.error(message, e); throw new NonTerminalException(message, e); + } finally { + lock.unlock(); } } - synchronized void close() { + @Override + public void close() { + lock.lock(); try { if (channel != null && channel.isOpen()) { - channel.close(); + Utils.close(channel, "AMQP channel"); } - } catch (Exception e) { - logger.error("Exception closing RabbitMQ channel", e); - } - try { if (connection != null && connection.isOpen()) { - connection.close(); + Utils.close(connection, "AMQP connection"); } - } catch (Exception e) { - logger.error("Exception closing RabbitMQ connection for", e); + connection = null; + channel = null; + } finally { + lock.unlock(); } - channel = null; - connection = null; - channelContext = null; + } + + private Connection createConnection() throws IOException, TimeoutException { + ConnectionFactory connectionFactory = createConnectionFactory(); + Connection connection = connectionFactory.newConnection(); + connection.addShutdownListener(new RmqClientShutdownEventListener(rabbitMQContext)); + return connection; } @NotNull - private static ConnectionFactory createConnectionFactory() { - RabbitMQManagementConfig rabbitMQManagementConfig = getManagementConfig(); + private ConnectionFactory createConnectionFactory() { ConnectionFactory connectionFactory = new ConnectionFactory(); connectionFactory.setHost(rabbitMQManagementConfig.getHost()); connectionFactory.setPort(rabbitMQManagementConfig.getApplicationPort()); @@ -436,16 +440,22 @@ private static ConnectionFactory createConnectionFactory() { } private static class RmqClientShutdownEventListener implements ShutdownListener { + private final RabbitMQContext rabbitMQContext; + + private RmqClientShutdownEventListener(RabbitMQContext rabbitMQContext) { + this.rabbitMQContext = rabbitMQContext; + } @Override public void shutdownCompleted(ShutdownSignalException cause) { - logger.error("Detected Rabbit mq connection lose", cause); + LOGGER.error("Detected Rabbit mq connection lose", cause); RecreateQueuesAndBindings recreateQueuesAndBindingsTask = new RecreateQueuesAndBindings( + rabbitMQContext, OperatorState.INSTANCE.getAllBoxResources(), RETRY_DELAY ); retryableTaskQueue.add(recreateQueuesAndBindingsTask, true); - logger.info("Task \"{}\" added to scheduler, with delay \"{}\" seconds", + LOGGER.info("Task \"{}\" added to scheduler, with delay \"{}\" seconds", recreateQueuesAndBindingsTask.getName(), RETRY_DELAY); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/ContinuousTaskWorker.java b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/ContinuousTaskWorker.java index bda96929..37f4b137 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/ContinuousTaskWorker.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/ContinuousTaskWorker.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,35 +17,43 @@ package com.exactpro.th2.infraoperator.spec.strategy.redeploy; import com.exactpro.th2.infraoperator.spec.strategy.redeploy.tasks.Task; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -public class ContinuousTaskWorker { - private static final Logger logger = LoggerFactory.getLogger(ContinuousTaskWorker.class); +public class ContinuousTaskWorker implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(ContinuousTaskWorker.class); private static final int THREAD_POOL_SIZE = 2; private final Map taskMap = new HashMap<>(); - private final ScheduledExecutorService taskScheduler = new ScheduledThreadPoolExecutor(THREAD_POOL_SIZE); + private final ScheduledExecutorService taskScheduler = new ScheduledThreadPoolExecutor(THREAD_POOL_SIZE, + new ThreadFactoryBuilder().setNameFormat("worker-%d").build()); public synchronized void add(Task task) { if (!taskMap.containsKey(task.getName())) { taskMap.put(task.getName(), task); taskScheduler.scheduleWithFixedDelay(task, task.getRetryDelay(), task.getRetryDelay(), TimeUnit.SECONDS); - logger.info("Added task '{}' to scheduler", task.getName()); + LOGGER.info("Added task '{}' to scheduler", task.getName()); } else { - logger.info("Task '{}' is already present in scheduler. Will not be added again", task.getName()); + LOGGER.info("Task '{}' is already present in scheduler. Will not be added again", task.getName()); } } - public void shutdown() { + @Override + public void close() throws Exception { taskScheduler.shutdown(); + if (!taskScheduler.awaitTermination(5, TimeUnit.SECONDS)) { + List tasks = taskScheduler.shutdownNow(); + LOGGER.error("The {} tasks in {} are not completed", tasks.size(), ContinuousTaskWorker.class); + } } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/CheckResourceCacheTask.java b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/CheckResourceCacheTask.java index 62762113..1d56f77c 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/CheckResourceCacheTask.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/CheckResourceCacheTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,24 +30,25 @@ import io.fabric8.kubernetes.api.model.DefaultKubernetesResourceList; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Set; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.stamp; +import static com.exactpro.th2.infraoperator.util.KubernetesUtils.createKubernetesClient; public class CheckResourceCacheTask implements Task { private static final Logger logger = LoggerFactory.getLogger(ContinuousTaskWorker.class); private final long retryDelay; - private final KubernetesClient client = new KubernetesClientBuilder().build(); + private final KubernetesClient client = createKubernetesClient(); - private final List nsPrefixes = ConfigLoader.getConfig().getNamespacePrefixes(); + private final Set nsPrefixes = ConfigLoader.getConfig().getNamespacePrefixes(); private final List operations = List.of( client.resources(Th2Box.class), diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RecreateQueuesAndBindings.java b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RecreateQueuesAndBindings.java index 1206a9db..fc7b9dfa 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RecreateQueuesAndBindings.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RecreateQueuesAndBindings.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,11 +24,22 @@ import java.util.Collection; public class RecreateQueuesAndBindings implements Task { + private final RabbitMQContext rabbitMQContext; + + private final DeclareQueueResolver declareQueueResolver; + + private final BindQueueLinkResolver bindQueueLinkResolver; + private final long retryDelay; private final Collection resources; - public RecreateQueuesAndBindings(Collection resources, long retryDelay) { + public RecreateQueuesAndBindings(RabbitMQContext rabbitMQContext, + Collection resources, + long retryDelay) { + this.rabbitMQContext = rabbitMQContext; + this.declareQueueResolver = new DeclareQueueResolver(rabbitMQContext); + this.bindQueueLinkResolver = new BindQueueLinkResolver(rabbitMQContext); this.resources = resources; this.retryDelay = retryDelay; } @@ -45,11 +56,11 @@ public long getRetryDelay() { @Override public void run() { - RabbitMQContext.getChannel(); + rabbitMQContext.getChannel(); resources.forEach(resource -> { - DeclareQueueResolver.resolveAdd(resource); - BindQueueLinkResolver.resolveDeclaredLinks(resource); - BindQueueLinkResolver.resolveHiddenLinks(resource); + declareQueueResolver.resolveAdd(resource); + bindQueueLinkResolver.resolveDeclaredLinks(resource); + bindQueueLinkResolver.resolveHiddenLinks(resource); }); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RetryRabbitSetup.java b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RetryRabbitSetup.java index 3f5e558e..610cfd77 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RetryRabbitSetup.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RetryRabbitSetup.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,14 @@ public class RetryRabbitSetup implements Task { + private final RabbitMQContext rabbitMQContext; + private final long retryDelay; private final String namespace; - public RetryRabbitSetup(String namespace, long retryDelay) { + public RetryRabbitSetup(RabbitMQContext rabbitMQContext, String namespace, long retryDelay) { + this.rabbitMQContext = rabbitMQContext; this.retryDelay = retryDelay; this.namespace = namespace; } @@ -41,6 +44,6 @@ public long getRetryDelay() { @Override public void run() { - RabbitMQContext.setUpRabbitMqForNamespace(namespace); + rabbitMQContext.setUpRabbitMqForNamespace(namespace); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RetryTopicExchangeTask.java b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RetryTopicExchangeTask.java index f67cd9ab..b47d2583 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RetryTopicExchangeTask.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/RetryTopicExchangeTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,14 @@ public class RetryTopicExchangeTask implements Task { + private final RabbitMQContext rabbitMQContext; + private final long retryDelay; private final String exchangeName; - public RetryTopicExchangeTask(String exchangeName, long retryDelay) { + public RetryTopicExchangeTask(RabbitMQContext rabbitMQContext, String exchangeName, long retryDelay) { + this.rabbitMQContext = rabbitMQContext; this.retryDelay = retryDelay; this.exchangeName = exchangeName; } @@ -41,6 +44,6 @@ public long getRetryDelay() { @Override public void run() { - RabbitMQContext.declareTopicExchange(); + rabbitMQContext.declareTopicExchange(); } } diff --git a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/TriggerRedeployTask.java b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/TriggerRedeployTask.java index b764765d..457739bd 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/TriggerRedeployTask.java +++ b/src/main/java/com/exactpro/th2/infraoperator/spec/strategy/redeploy/tasks/TriggerRedeployTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,13 +30,12 @@ import java.util.HashMap; import static com.exactpro.th2.infraoperator.operator.AbstractTh2Operator.REFRESH_TOKEN_ALIAS; +import static com.exactpro.th2.infraoperator.util.KubernetesUtils.PHASE_ACTIVE; public class TriggerRedeployTask implements Task { private static final Logger logger = LoggerFactory.getLogger(TriggerRedeployTask.class); - public static final String PHASE_ACTIVE = "Active"; - private final ResourceClient resourceClient; private final KubernetesClient kubClient; diff --git a/src/main/java/com/exactpro/th2/infraoperator/util/CustomResourceUtils.java b/src/main/java/com/exactpro/th2/infraoperator/util/CustomResourceUtils.java index b028a553..7a0ad151 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/util/CustomResourceUtils.java +++ b/src/main/java/com/exactpro/th2/infraoperator/util/CustomResourceUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,23 +16,17 @@ package com.exactpro.th2.infraoperator.util; -import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractName; -import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractNamespace; -import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractType; - import com.exactpro.th2.infraoperator.spec.Th2CustomResource; import com.exactpro.th2.infraoperator.spec.helmrelease.HelmRelease; -import com.exactpro.th2.infraoperator.spec.shared.pin.*; import io.fabric8.kubernetes.api.model.HasMetadata; - -import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.List; -import java.util.Objects; + +import static com.exactpro.th2.infraoperator.util.ExtractUtils.extractName; public class CustomResourceUtils { @@ -40,7 +34,7 @@ public class CustomResourceUtils { public static final long RESYNC_TIME = 180000; - private static final String GIT_COMMIT_HASH = "th2.exactpro.com/git-commit-hash"; + public static final String GIT_COMMIT_HASH = "th2.exactpro.com/git-commit-hash"; private static final int SHORT_HASH_LENGTH = 8; @@ -55,7 +49,7 @@ public static String annotationFor(String namespace, String kind, String resourc return String.format("%s:%s/%s(commit-%s)", namespace, kind, resourceName, commitHash); } - public static String annotationFor(HasMetadata resource) { + public static String annotationFor(@NotNull HasMetadata resource) { return annotationFor( resource.getMetadata().getNamespace(), resource.getKind(), @@ -64,47 +58,15 @@ public static String annotationFor(HasMetadata resource) { ); } - @Nullable - public static HelmRelease search(List helmReleases, Th2CustomResource resource) { - String resFullName = extractHashedFullName(resource); - return helmReleases.stream() - .filter(hr -> { - var owner = extractOwnerFullName(hr); - return Objects.nonNull(owner) && owner.equals(resFullName); - }).findFirst() - .orElse(null); - } - public static String extractHashedName(Th2CustomResource customResource) { return hashNameIfNeeded(extractName(customResource)); } - private static String extractHashedFullName(Th2CustomResource customResource) { - return concatFullName(extractNamespace(customResource), extractHashedName(customResource)); - } - - @Nullable - private static String extractOwnerFullName(HelmRelease helmRelease) { - var ownerReferences = helmRelease.getMetadata().getOwnerReferences(); - if (ownerReferences.size() > 0) { - return concatFullName(extractNamespace(helmRelease), ownerReferences.get(0).getName()); - } else { - logger.warn("[{}<{}>] doesn't have owner resource", extractType(helmRelease), extractFullName(helmRelease)); - return null; - } - } - - private static String extractFullName(HasMetadata obj) { - return concatFullName(extractNamespace(obj), extractName(obj)); - } - - private static String concatFullName(String namespace, String name) { - return namespace + "." + name; - } - private static String hashNameIfNeeded(String resName) { if (resName.length() >= HelmRelease.NAME_LENGTH_LIMIT) { - return digest(resName); + String result = digest(resName); + logger.debug("Resource '{}' name has been hashed to '{}'", resName, result); + return result; } return resName; } diff --git a/src/main/java/com/exactpro/th2/infraoperator/util/ExtractUtils.java b/src/main/java/com/exactpro/th2/infraoperator/util/ExtractUtils.java index 303f404c..36b60511 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/util/ExtractUtils.java +++ b/src/main/java/com/exactpro/th2/infraoperator/util/ExtractUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,11 @@ import java.util.Map; import io.fabric8.kubernetes.api.model.HasMetadata; +import org.apache.commons.lang3.StringUtils; public class ExtractUtils { - static final String KEY_SOURCE_HASH = "th2.exactpro.com/source-hash"; + public static final String KEY_SOURCE_HASH = "th2.exactpro.com/source-hash"; public static final String REFRESH_TOKEN_ALIAS = "refresh-token"; @@ -73,7 +74,7 @@ public static String fullSourceHash(HasMetadata res) { public static String shortSourceHash(HasMetadata res) { String fullHash = fullSourceHash(res); - if (Strings.isNullOrEmpty(fullHash)) { + if (StringUtils.isBlank(fullHash)) { return fullHash; } return "[" + fullHash.substring(0, 8) + "]"; diff --git a/src/main/java/com/exactpro/th2/infraoperator/util/HelmReleaseUtils.java b/src/main/java/com/exactpro/th2/infraoperator/util/HelmReleaseUtils.java index e2b697dc..2803e7b8 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/util/HelmReleaseUtils.java +++ b/src/main/java/com/exactpro/th2/infraoperator/util/HelmReleaseUtils.java @@ -39,7 +39,12 @@ import java.util.Set; import java.util.stream.Collectors; -import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.*; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.DICTIONARIES_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.ENABLED_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.EXTENDED_SETTINGS_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.EXTERNAL_BOX_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.MQ_QUEUE_CONFIG_ALIAS; +import static com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.SERVICE_ALIAS; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.annotationFor; import static com.exactpro.th2.infraoperator.util.JsonUtils.JSON_MAPPER; import static com.exactpro.th2.infraoperator.util.JsonUtils.YAML_MAPPER; diff --git a/src/main/java/com/exactpro/th2/infraoperator/util/Strings.java b/src/main/java/com/exactpro/th2/infraoperator/util/Strings.java index 61929a65..a03ee67d 100644 --- a/src/main/java/com/exactpro/th2/infraoperator/util/Strings.java +++ b/src/main/java/com/exactpro/th2/infraoperator/util/Strings.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ import com.exactpro.th2.infraoperator.model.box.dictionary.DictionaryEntity; import org.apache.commons.text.lookup.StringLookup; -import java.util.List; +import java.util.Collection; import java.util.Map; import java.util.Set; @@ -30,14 +30,17 @@ public class Strings { private Strings() { } - public static boolean isNullOrEmpty(String s) { - return (s == null || s.isEmpty()); + public static boolean anyPrefixMatch(String namespace, Collection prefixes) { + return (namespace != null + && prefixes != null + && !prefixes.isEmpty() + && prefixes.stream().anyMatch(namespace::startsWith)); } - public static boolean nonePrefixMatch(String namespace, List prefixes) { + public static boolean nonePrefixMatch(String namespace, Collection prefixes) { return (namespace != null && prefixes != null - && prefixes.size() > 0 + && !prefixes.isEmpty() && prefixes.stream().noneMatch(namespace::startsWith)); } diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/ConfigLoader.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/ConfigLoader.kt index 2706f041..6d5ef569 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/ConfigLoader.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/ConfigLoader.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ import com.exactpro.th2.infraoperator.util.JsonUtils.YAML_MAPPER import com.fasterxml.jackson.core.JsonParseException import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException import com.fasterxml.jackson.module.kotlin.readValue -import mu.KotlinLogging +import io.github.oshai.kotlinlogging.KotlinLogging import org.apache.commons.text.StringSubstitutor import org.apache.commons.text.lookup.StringLookupFactory import java.io.FileInputStream @@ -54,13 +54,12 @@ object ConfigLoader { return YAML_MAPPER.readValue(content) } } catch (e: UnrecognizedPropertyException) { - logger.error( - "Bad configuration: unknown property(\"{}\") specified in configuration file", - e.propertyName - ) + logger.error(e) { + "Bad configuration: unknown property('${e.propertyName}') specified in configuration file" + } throw e } catch (e: JsonParseException) { - logger.error("Bad configuration: exception while parsing configuration file") + logger.error { "Bad configuration: exception while parsing configuration file" } throw e } } diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/OperatorConfig.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/OperatorConfig.kt index b66b5465..7d31bbe0 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/OperatorConfig.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/OperatorConfig.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ data class OperatorConfig( var chart: Any? = null, var rabbitMQManagement: RabbitMQManagementConfig = RabbitMQManagementConfig(), var schemaSecrets: SchemaSecrets = SchemaSecrets(), - var namespacePrefixes: List = ArrayList(), + var namespacePrefixes: Set = emptySet(), var rabbitMQConfigMapName: String = DEFAULT_RABBITMQ_CONFIGMAP_NAME, var k8sUrl: String = "", var prometheusConfiguration: PrometheusConfiguration = PrometheusConfiguration.createDefault("true"), diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQConfig.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQConfig.kt index c5bc0c88..af70258d 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQConfig.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQConfig.kt @@ -16,12 +16,14 @@ package com.exactpro.th2.infraoperator.configuration.fields +import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.annotation.JsonDeserialize @JsonDeserialize data class RabbitMQConfig( val port: Int, val host: String, + @get:JsonProperty("vHost") val vHost: String, val exchangeName: String, val username: String, diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQManagementConfig.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQManagementConfig.kt index 059cd646..b2a371d6 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQManagementConfig.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQManagementConfig.kt @@ -25,6 +25,5 @@ data class RabbitMQManagementConfig( val username: String = "", val password: String = "", val persistence: Boolean = false, - val cleanUpOnStart: Boolean = false, val schemaPermissions: RabbitMQNamespacePermissions = RabbitMQNamespacePermissions() ) diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQNamespacePermissions.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQNamespacePermissions.kt index 22e5f667..16a69df5 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQNamespacePermissions.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/configuration/fields/RabbitMQNamespacePermissions.kt @@ -26,7 +26,7 @@ data class RabbitMQNamespacePermissions( ) { companion object { - const val DEFAULT_CONFIGURE_PERMISSION = "" + const val DEFAULT_CONFIGURE_PERMISSION = ".*" const val DEFAULT_READ_PERMISSION = ".*" const val DEFAULT_WRITE_PERMISSION = ".*" } diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactory.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactory.kt index 3440f23d..3636b314 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactory.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactory.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,10 @@ package com.exactpro.th2.infraoperator.model.box.mq.factory import com.exactpro.th2.infraoperator.model.LinkDescription import com.exactpro.th2.infraoperator.model.box.mq.MessageRouterConfiguration import com.exactpro.th2.infraoperator.model.box.mq.QueueConfiguration -import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op import com.exactpro.th2.infraoperator.spec.Th2CustomResource import com.exactpro.th2.infraoperator.spec.shared.PinAttribute +import com.exactpro.th2.infraoperator.spec.shared.pin.PinSpec +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createEstoreRoutingKeyName import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.QueueName import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.RoutingKeyName import com.exactpro.th2.infraoperator.util.ExtractUtils @@ -42,7 +43,7 @@ abstract class MessageRouterConfigFactory { protected fun generatePublishToEstorePin(namespace: String, boxName: String) = QueueConfiguration( LinkDescription( QueueName.EMPTY, - RoutingKeyName(namespace, boxName, StoreHelmTh2Op.EVENT_STORAGE_PIN_ALIAS), + createEstoreRoutingKeyName(namespace, boxName), namespace ), setOf(PinAttribute.publish.name, PinAttribute.event.name), diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryBox.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryBox.kt index 97fe18c1..3f1986d0 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryBox.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryBox.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import com.exactpro.th2.infraoperator.model.box.mq.MessageRouterConfiguration import com.exactpro.th2.infraoperator.model.box.mq.QueueConfiguration import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_PIN_ALIAS import com.exactpro.th2.infraoperator.spec.Th2CustomResource +import com.exactpro.th2.infraoperator.spec.shared.pin.PinSpec import com.exactpro.th2.infraoperator.util.ExtractUtils /** diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryEstore.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryEstore.kt index ed8e30e5..578f31c4 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryEstore.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryEstore.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,8 @@ import com.exactpro.th2.infraoperator.model.box.mq.QueueConfiguration import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_PIN_ALIAS import com.exactpro.th2.infraoperator.spec.Th2CustomResource import com.exactpro.th2.infraoperator.spec.shared.PinAttribute -import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.QueueName +import com.exactpro.th2.infraoperator.spec.shared.pin.PinSpec +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createEstoreQueueName import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.RoutingKeyName import com.exactpro.th2.infraoperator.util.ExtractUtils @@ -47,7 +48,7 @@ class MessageRouterConfigFactoryEstore : MessageRouterConfigFactory() { // add event storage pin config for each resource queues[EVENT_STORAGE_PIN_ALIAS] = QueueConfiguration( LinkDescription( - QueueName(namespace, boxName, EVENT_STORAGE_PIN_ALIAS), + createEstoreQueueName(namespace, boxName), RoutingKeyName.EMPTY, namespace ), diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryMstore.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryMstore.kt index c0e3f162..70bc4a5d 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryMstore.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/model/box/mq/factory/MessageRouterConfigFactoryMstore.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,8 @@ import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_PIN_ import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_PIN_ALIAS import com.exactpro.th2.infraoperator.spec.Th2CustomResource import com.exactpro.th2.infraoperator.spec.shared.PinAttribute -import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.QueueName +import com.exactpro.th2.infraoperator.spec.shared.pin.PinSpec +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createMstoreQueueName import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.RoutingKeyName import com.exactpro.th2.infraoperator.util.ExtractUtils @@ -49,7 +50,7 @@ class MessageRouterConfigFactoryMstore : MessageRouterConfigFactory() { queues[EVENT_STORAGE_PIN_ALIAS] = generatePublishToEstorePin(namespace, boxName) queues[MESSAGE_STORAGE_PIN_ALIAS] = QueueConfiguration( LinkDescription( - QueueName(namespace, boxName, MESSAGE_STORAGE_PIN_ALIAS), + createMstoreQueueName(namespace, boxName), RoutingKeyName.EMPTY, namespace ), diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/Util.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/Util.kt new file mode 100644 index 00000000..a676d3cd --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/Util.kt @@ -0,0 +1,66 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JvmName("Util") + +package com.exactpro.th2.infraoperator.spec.strategy.linkresolver + +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_BOX_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_PIN_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_BOX_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_PIN_ALIAS +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.QueueName +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.RoutingKeyName + +fun createEstoreQueueName(namespace: String): QueueName = QueueName( + namespace, + EVENT_STORAGE_BOX_ALIAS, + EVENT_STORAGE_PIN_ALIAS +) + +fun createEstoreQueue(namespace: String): String = createEstoreQueueName(namespace).toString() + +fun createEstoreQueueName(namespace: String, component: String): QueueName = QueueName( + namespace, + component, + EVENT_STORAGE_PIN_ALIAS +) + +fun createEstoreQueue(namespace: String, component: String): String = + createEstoreQueueName(namespace, component).toString() + +fun createEstoreRoutingKeyName(namespace: String, component: String) = RoutingKeyName( + namespace, + component, + EVENT_STORAGE_PIN_ALIAS +) + +fun createMstoreQueueName(namespace: String) = QueueName( + namespace, + MESSAGE_STORAGE_BOX_ALIAS, + MESSAGE_STORAGE_PIN_ALIAS +) + +fun createMstoreQueue(namespace: String): String = createMstoreQueueName(namespace).toString() + +fun createMstoreQueueName(namespace: String, component: String) = QueueName( + namespace, + component, + MESSAGE_STORAGE_PIN_ALIAS +) + +fun createMstoreQueue(namespace: String, component: String): String = + createMstoreQueueName(namespace, component).toString() diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/BindQueueLinkResolver.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/BindQueueLinkResolver.kt index 3ec3e759..0c2c3177 100644 --- a/src/main/kotlin/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/BindQueueLinkResolver.kt +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/spec/strategy/linkresolver/mq/BindQueueLinkResolver.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,24 +18,24 @@ package com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq import com.exactpro.th2.infraoperator.model.LinkDescription import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_BOX_ALIAS -import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_PIN_ALIAS import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_BOX_ALIAS -import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_PIN_ALIAS import com.exactpro.th2.infraoperator.spec.Th2CustomResource import com.exactpro.th2.infraoperator.spec.shared.PinAttribute import com.exactpro.th2.infraoperator.spec.shared.pin.Link +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createEstoreQueueName +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createEstoreRoutingKeyName +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createMstoreQueueName import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.QueueName import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.RoutingKeyName import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.queue.RoutingKeyName.ROUTING_KEY_REGEXP import com.exactpro.th2.infraoperator.spec.strategy.redeploy.NonTerminalException import com.exactpro.th2.infraoperator.util.CustomResourceUtils import com.exactpro.th2.infraoperator.util.CustomResourceUtils.annotationFor -import mu.KotlinLogging +import io.github.oshai.kotlinlogging.KotlinLogging -object BindQueueLinkResolver { - private val logger = KotlinLogging.logger { } - - @JvmStatic +class BindQueueLinkResolver( + private val rabbitMQContext: RabbitMQContext, +) { fun resolveDeclaredLinks(resource: Th2CustomResource) { val namespace = resource.metadata.namespace val resourceName = resource.metadata.name @@ -54,7 +54,6 @@ object BindQueueLinkResolver { } } - @JvmStatic fun resolveHiddenLinks(resource: Th2CustomResource) { val namespace = resource.metadata.namespace val resourceName = resource.metadata.name @@ -65,14 +64,14 @@ object BindQueueLinkResolver { } // create event storage link for each resource val estoreLinkDescription = LinkDescription( - QueueName(namespace, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS), - RoutingKeyName(namespace, resourceName, EVENT_STORAGE_PIN_ALIAS), + createEstoreQueueName(namespace), + createEstoreRoutingKeyName(namespace, resourceName), namespace ) bindQueues(estoreLinkDescription, commitHash) val currentLinks: MutableList = ArrayList() - val queueName = QueueName(namespace, MESSAGE_STORAGE_BOX_ALIAS, MESSAGE_STORAGE_PIN_ALIAS) + val queueName = createMstoreQueueName(namespace) // create message store link for only resources that need it for ((pinName, attributes) in resource.spec.pins.mq.publishers) { if (checkStorePinAttributes(attributes, resourceLabel, pinName)) { @@ -93,19 +92,17 @@ object BindQueueLinkResolver { return false } if (attributes.contains(PinAttribute.parsed.name)) { - logger.warn( - "Detected a pin: {}:{} with incorrect store configuration. attribute 'parsed' not allowed", - resourceLabel, - pinName - ) + K_LOGGER.warn { + "Detected a pin: $resourceLabel:$pinName with incorrect store configuration. " + + "attribute 'parsed' not allowed" + } return false } if (!attributes.contains(PinAttribute.raw.name)) { - logger.warn( - "Detected a pin: {}:{} with incorrect store configuration. attribute 'raw' is missing", - resourceLabel, - pinName - ) + K_LOGGER.warn { + "Detected a pin: $resourceLabel:$pinName with incorrect store configuration. " + + "attribute 'raw' is missing" + } return false } return true @@ -113,23 +110,20 @@ object BindQueueLinkResolver { private fun bindQueues(queue: LinkDescription, commitHash: String) { try { - val channel = RabbitMQContext.getChannel() + val channel = rabbitMQContext.channel val queueName = queue.queueName.toString() - val currentQueue = RabbitMQContext.getQueue(queueName) + val currentQueue = rabbitMQContext.getQueue(queueName) if (currentQueue == null) { - logger.info("Queue '{}' does not yet exist. skipping binding", queueName) + K_LOGGER.info { "Queue '$queueName' does not yet exist. skipping binding" } return } channel.queueBind(queue.queueName.toString(), queue.exchange, queue.routingKey.toString()) - logger.info( - "Queue '{}' successfully bound to '{}' (commit-{})", - queueName, - queue.routingKey.toString(), - commitHash - ) + K_LOGGER.info { + "Queue '$queueName' successfully bound to '${queue.routingKey}' (commit-$commitHash)" + } } catch (e: Exception) { val message = "Exception while working with rabbitMq" - logger.error(message, e) + K_LOGGER.error(e) { message } throw NonTerminalException(message, e) } } @@ -141,7 +135,7 @@ object BindQueueLinkResolver { resName: String? = null ) { val queueName = queue.toString() - val bindingOnRabbit = RabbitMQContext.getQueueBindings(queueName) + val bindingOnRabbit = rabbitMQContext.getQueueBindings(queueName) ?.map { it.routingKey } ?.filter { it.matches(ROUTING_KEY_REGEXP.toRegex()) && @@ -151,22 +145,26 @@ object BindQueueLinkResolver { RoutingKeyName(queue.namespace, it.box, it.pin).toString() } try { - val channel = RabbitMQContext.getChannel() + val channel = rabbitMQContext.channel bindingOnRabbit?.forEach { if (!currentBindings.contains(it)) { - val currentQueue = RabbitMQContext.getQueue(queueName) + val currentQueue = rabbitMQContext.getQueue(queueName) if (currentQueue == null) { - logger.info("Queue '{}' already removed. skipping unbinding", queueName) + K_LOGGER.info { "Queue '$queueName' already removed. skipping unbinding" } return } channel.queueUnbind(queueName, queue.namespace, it) - logger.info("Unbind queue '{}' -> '{}'. (commit-{})", it, queueName, commitHash) + K_LOGGER.info { "Unbind queue '$it' -> '$queueName'. (commit-$commitHash)" } } } } catch (e: Exception) { val message = "Exception while removing extinct bindings" - logger.error(message, e) + K_LOGGER.error(e) { message } throw NonTerminalException(message, e) } } + + companion object { + private val K_LOGGER = KotlinLogging.logger { } + } } diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/util/KubernetesUtils.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/util/KubernetesUtils.kt new file mode 100644 index 00000000..5f0ac77b --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/util/KubernetesUtils.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JvmName("KubernetesUtils") + +package com.exactpro.th2.infraoperator.util + +import com.exactpro.th2.infraoperator.spec.Th2CustomResource +import com.exactpro.th2.infraoperator.spec.box.Th2Box +import com.exactpro.th2.infraoperator.spec.corebox.Th2CoreBox +import com.exactpro.th2.infraoperator.spec.estore.Th2Estore +import com.exactpro.th2.infraoperator.spec.job.Th2Job +import com.exactpro.th2.infraoperator.spec.mstore.Th2Mstore +import io.fabric8.kubernetes.api.model.Namespace +import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.KubernetesClientBuilder +import kotlin.streams.toList + +const val PHASE_ACTIVE = "Active" + +val CUSTOM_RESOURCE_KINDS: Set> = + setOf(Th2Estore::class.java, Th2Mstore::class.java, Th2CoreBox::class.java, Th2Box::class.java, Th2Job::class.java) + +fun createKubernetesClient(): KubernetesClient = KubernetesClientBuilder().build() + +fun KubernetesClient.namespaces(namespacePrefixes: Set): Set = + namespaces() + .list() + .items + .map { it.metadata.name } + .filter { ns -> Strings.anyPrefixMatch(ns, namespacePrefixes) } + .toSet() + +fun KubernetesClient.customResources(namespace: String): List = + CUSTOM_RESOURCE_KINDS + .stream() + .flatMap { + resources(it) + .inNamespace(namespace) + .resources() + }.map { it.get() as Th2CustomResource } + .toList() + +fun KubernetesClient.isNotActive(namespace: String): Boolean { + val namespaceObj: Namespace? = namespaces().withName(namespace).get() + return namespaceObj == null || namespaceObj.status.phase != PHASE_ACTIVE +} diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/util/RabbitMQUtils.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/util/RabbitMQUtils.kt new file mode 100644 index 00000000..64f3c6f5 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/util/RabbitMQUtils.kt @@ -0,0 +1,170 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JvmName("RabbitMQUtils") + +package com.exactpro.th2.infraoperator.util + +import com.exactpro.th2.infraoperator.configuration.ConfigLoader +import com.exactpro.th2.infraoperator.model.box.mq.QueueConfiguration +import com.exactpro.th2.infraoperator.model.box.mq.factory.MessageRouterConfigFactory +import com.exactpro.th2.infraoperator.model.box.mq.factory.MessageRouterConfigFactoryBox +import com.exactpro.th2.infraoperator.model.box.mq.factory.MessageRouterConfigFactoryEstore +import com.exactpro.th2.infraoperator.model.box.mq.factory.MessageRouterConfigFactoryMstore +import com.exactpro.th2.infraoperator.spec.Th2CustomResource +import com.exactpro.th2.infraoperator.spec.estore.Th2Estore +import com.exactpro.th2.infraoperator.spec.mstore.Th2Mstore +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createEstoreQueue +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createMstoreQueue +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext +import com.rabbitmq.client.Channel +import com.rabbitmq.http.client.domain.ExchangeInfo +import com.rabbitmq.http.client.domain.QueueInfo +import io.fabric8.kubernetes.client.KubernetesClient +import io.github.oshai.kotlinlogging.KotlinLogging +import java.io.IOException + +private val K_LOGGER = KotlinLogging.logger { } + +fun deleteRabbitMQRubbish( + kubernetesClient: KubernetesClient, + rabbitMQContext: RabbitMQContext, +) { + try { + val resourceHolder = collectRabbitMQResources( + rabbitMQContext.th2Queues, + rabbitMQContext.th2Exchanges, + ) + + if (resourceHolder.isHolderEmpty()) { + return + } + + val namespacePrefixes = ConfigLoader.loadConfiguration().namespacePrefixes + val topicExchange = rabbitMQContext.topicExchangeName + + resourceHolder.filterRubbishResources( + kubernetesClient, + namespacePrefixes, + topicExchange, + ) + K_LOGGER.info { "RabbitMQ rubbish: $resourceHolder" } + deleteRabbitMQRubbish(resourceHolder, rabbitMQContext::getChannel) + } catch (e: Exception) { + K_LOGGER.error(e) { "Delete RabbitMQ rubbish failure" } + } +} + +internal fun collectRabbitMQResources( + th2Queues: Collection, + th2Exchanges: Collection, +): ResourceHolder = ResourceHolder().apply { + th2Queues.asSequence() + .map(QueueInfo::getName) + .forEach(queues::add) + + th2Exchanges.asSequence() + .map(ExchangeInfo::getName) + .forEach(exchanges::add) + + K_LOGGER.debug { "Actual set in RabbitMQ, queues: $queues, exchanges: $exchanges" } +} + +internal fun ResourceHolder.filterRubbishResources( + client: KubernetesClient, + namespacePrefixes: Set, + topicExchange: String, +): ResourceHolder = apply { + val namespaces: Set = client.namespaces(namespacePrefixes) + if (namespaces.isEmpty()) { + return@apply + } + exchanges.remove(topicExchange) // FIXME: topic exchange should be declare after each namespace creation + + K_LOGGER.debug { "Search RabbitMQ resources in $namespaces namespaces" } + + val factories: Map, MessageRouterConfigFactory> = createFactories() + namespaces.forEach { namespace -> + queues.remove(createEstoreQueue(namespace)) + queues.remove(createMstoreQueue(namespace)) + exchanges.remove(namespace) + + client.customResources(namespace).asSequence() + .flatMap { cr -> + factories[cr.javaClass]?.createConfig(cr)?.queues?.values + ?: error("MQ config factory isn't present for ${cr.javaClass.simpleName}") + }.map(QueueConfiguration::getQueueName) + .filter(String::isNotBlank) + .forEach(queues::remove) + + K_LOGGER.debug { + "Survived RabbitMQ resources after '$namespace' namespace process, " + + "queues: $queues, exchanges: $exchanges" + } + } +} + +internal fun deleteRabbitMQRubbish( + resourceHolder: ResourceHolder, + getChannel: () -> Channel +) { + if (resourceHolder.isHolderEmpty()) { + return + } + + val channel: Channel = getChannel() + + resourceHolder.queues.forEach { queue -> + try { + channel.queueDelete(queue) + K_LOGGER.info { "Deleted '$queue' queue" } + } catch (e: IOException) { + K_LOGGER.error(e) { "'$queue' queue delete failure" } + } + } + + resourceHolder.exchanges.forEach { exchange -> + try { + channel.exchangeDelete(exchange) + K_LOGGER.info { "Deleted '$exchange' exchange" } + } catch (e: IOException) { + K_LOGGER.error(e) { "'$exchange' queue delete failure" } + } + } +} + +private fun createFactories(): Map, MessageRouterConfigFactory> { + val defaultFactory = MessageRouterConfigFactoryBox() + return CUSTOM_RESOURCE_KINDS + .asSequence() + .map { + it to + when (it) { + Th2Mstore::class.java -> MessageRouterConfigFactoryMstore() + Th2Estore::class.java -> MessageRouterConfigFactoryEstore() + else -> defaultFactory + } + }.toMap() +} + +internal data class ResourceHolder( + val queues: MutableSet = hashSetOf(), + val exchanges: MutableSet = hashSetOf(), +) { + fun isHolderEmpty() = queues.isEmpty() and exchanges.isEmpty() + + override fun toString(): String = "queues=$queues, exchanges=$exchanges" +} diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/util/Utils.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/util/Utils.kt new file mode 100644 index 00000000..de37964c --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/util/Utils.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JvmName("Utils") + +package com.exactpro.th2.infraoperator.util + +import io.github.oshai.kotlinlogging.KotlinLogging + +private val K_LOGGER = KotlinLogging.logger {} + +fun close(closeable: AutoCloseable, name: String) { + try { + closeable.close() + K_LOGGER.info { "$name closed" } + } catch (e: Exception) { + K_LOGGER.error(e) { "$name close failure" } + } +} diff --git a/src/main/kotlin/com/exactpro/th2/infraoperator/util/WatcherUtils.kt b/src/main/kotlin/com/exactpro/th2/infraoperator/util/WatcherUtils.kt new file mode 100644 index 00000000..81626fee --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/infraoperator/util/WatcherUtils.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JvmName("WatcherUtils") + +package com.exactpro.th2.infraoperator.util + +import io.fabric8.kubernetes.client.WatcherException +import io.fabric8.kubernetes.client.informers.ExceptionHandler +import io.github.oshai.kotlinlogging.KotlinLogging + +private val K_LOGGER = KotlinLogging.logger { } + +fun createExceptionHandler(clazz: Class<*>): ExceptionHandler { + return ExceptionHandler { isStarted: Boolean, t: Throwable -> + K_LOGGER.error(t) { "${clazz.simpleName} informer catch error, isStarted: $isStarted" } + // Default condition copied from io.fabric8.kubernetes.client.informers.impl.cache.Reflector.handler. + // We should monitor caught errors in real cluster + // after that change the condition to maintain a component in working order + isStarted && t !is WatcherException + } +} diff --git a/src/test/java/com/exactpro/th2/infraoperator/configuration/ConfigurationTests.java b/src/test/java/com/exactpro/th2/infraoperator/configuration/ConfigurationTests.java index 92776a39..ac52ed19 100644 --- a/src/test/java/com/exactpro/th2/infraoperator/configuration/ConfigurationTests.java +++ b/src/test/java/com/exactpro/th2/infraoperator/configuration/ConfigurationTests.java @@ -21,12 +21,14 @@ import com.exactpro.th2.infraoperator.configuration.fields.SchemaSecrets; import org.junit.jupiter.api.Test; -import java.util.Arrays; -import java.util.Collections; +import java.util.Set; import static com.exactpro.th2.infraoperator.configuration.ConfigLoader.CONFIG_FILE_SYSTEM_PROPERTY; -import static org.junit.jupiter.api.Assertions.*; -import static com.exactpro.th2.infraoperator.configuration.OperatorConfig.*; +import static com.exactpro.th2.infraoperator.configuration.OperatorConfig.DEFAULT_RABBITMQ_CONFIGMAP_NAME; +import static java.util.Collections.emptySet; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; class ConfigurationTests { @@ -63,14 +65,13 @@ void testFullConfig() { "username", "password", true, - false, new RabbitMQNamespacePermissions( "configure", "read", "write" ) ) ); expected.setSchemaSecrets(new SchemaSecrets("rabbitMQ", "cassandra")); - expected.setNamespacePrefixes(Arrays.asList("string1", "string2")); + expected.setNamespacePrefixes(Set.of("string1", "string2")); expected.setRabbitMQConfigMapName("rabbit-mq-app"); assertEquals(expected, loadConfiguration()); @@ -80,7 +81,7 @@ void testFullConfig() { void testNsPrefixes() { beforeEach("nsPrefixesConfig.yml"); - expected.setNamespacePrefixes(Arrays.asList("string1", "string2")); + expected.setNamespacePrefixes(Set.of("string1", "string2")); assertEquals(expected, loadConfiguration()); } @@ -99,7 +100,6 @@ void testRabbitMQManagementConfig() { "username", "password", true, - false, new RabbitMQNamespacePermissions( "configure", "read", "write" ) @@ -121,7 +121,7 @@ void testSchemaSecretsConfig() { @Test void testDefaultConfig() { OperatorConfig config = new OperatorConfig(); - assertEquals(Collections.emptyList(), + assertEquals(emptySet(), config.getNamespacePrefixes()); assertTrue(config.getRabbitMQManagement().getHost().isEmpty()); diff --git a/src/test/java/com/exactpro/th2/infraoperator/util/CustomResourceUtilsTests.java b/src/test/java/com/exactpro/th2/infraoperator/util/CustomResourceUtilsTests.java index de96bf7c..96c0259b 100644 --- a/src/test/java/com/exactpro/th2/infraoperator/util/CustomResourceUtilsTests.java +++ b/src/test/java/com/exactpro/th2/infraoperator/util/CustomResourceUtilsTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,74 +16,36 @@ package com.exactpro.th2.infraoperator.util; +import com.exactpro.th2.infraoperator.spec.Th2CustomResource; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import org.junit.jupiter.api.Test; + import static com.exactpro.th2.infraoperator.spec.helmrelease.HelmRelease.NAME_LENGTH_LIMIT; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.digest; import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.extractHashedName; -import static com.exactpro.th2.infraoperator.util.CustomResourceUtils.search; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Test; - -import com.exactpro.th2.infraoperator.spec.Th2CustomResource; -import com.exactpro.th2.infraoperator.spec.helmrelease.HelmRelease; - -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.OwnerReference; - class CustomResourceUtilsTests { - private final String sourceNamespace = "namespace"; + private static final String SOURCE_NAMESPACE = "namespace"; - private final String sourceName = "123456789_123456789_123456789"; + private static final String SOURCE_NAME = "123456789_123456789_123456789"; @Test void extractHashedNameTest() { - var currentName = sourceName.substring(0, NAME_LENGTH_LIMIT - 1); + var currentName = SOURCE_NAME.substring(0, NAME_LENGTH_LIMIT - 1); assertEquals(currentName, extractHashedName(createTh2CustomResource(currentName))); - currentName = sourceName.substring(0, NAME_LENGTH_LIMIT); + currentName = SOURCE_NAME.substring(0, NAME_LENGTH_LIMIT); assertEquals(digest(currentName), extractHashedName(createTh2CustomResource(currentName))); - currentName = sourceName.substring(0, NAME_LENGTH_LIMIT + 1); + currentName = SOURCE_NAME.substring(0, NAME_LENGTH_LIMIT + 1); assertEquals(digest(currentName), extractHashedName(createTh2CustomResource(currentName))); } - @Test - void searchTest() { - List helmReleases = new ArrayList<>(); - helmReleases.add(createHelmRelease(sourceName.substring(0, NAME_LENGTH_LIMIT - 1))); - helmReleases.add(createHelmRelease(digest(sourceName.substring(0, NAME_LENGTH_LIMIT)))); - helmReleases.add(createHelmRelease(digest(sourceName.substring(0, NAME_LENGTH_LIMIT + 1)))); - - var th2CustomResource = createTh2CustomResource(sourceName.substring(0, NAME_LENGTH_LIMIT - 1)); - assertEquals(helmReleases.get(0), search(helmReleases, th2CustomResource)); - - th2CustomResource = createTh2CustomResource(sourceName.substring(0, NAME_LENGTH_LIMIT)); - assertEquals(helmReleases.get(1), search(helmReleases, th2CustomResource)); - - th2CustomResource = createTh2CustomResource(sourceName.substring(0, NAME_LENGTH_LIMIT + 1)); - assertEquals(helmReleases.get(2), search(helmReleases, th2CustomResource)); - } - - private HelmRelease createHelmRelease(String name) { - HelmRelease helmRelease = createResource(HelmRelease.class, name); - - List ownerReferences = new ArrayList<>(); - var ownerReference = new OwnerReference(); - ownerReference.setName(name); - ownerReferences.add(ownerReference); - - when(helmRelease.getMetadata().getOwnerReferences()).thenReturn(ownerReferences); - - return helmRelease; - } - private Th2CustomResource createTh2CustomResource(String name) { return createResource(Th2CustomResource.class, name); } @@ -91,7 +53,7 @@ private Th2CustomResource createTh2CustomResource(String name) { private T createResource(Class instanceClass, String name) { T resource = mock(instanceClass); ObjectMeta metaData = mock(ObjectMeta.class); - when(metaData.getNamespace()).thenReturn(sourceNamespace); + when(metaData.getNamespace()).thenReturn(SOURCE_NAMESPACE); when(metaData.getName()).thenReturn(name); when(resource.getMetadata()).thenReturn(metaData); diff --git a/src/test/java/com/exactpro/th2/infraoperator/util/StringsTests.java b/src/test/java/com/exactpro/th2/infraoperator/util/StringsTests.java index c8f6e476..d8301ae7 100644 --- a/src/test/java/com/exactpro/th2/infraoperator/util/StringsTests.java +++ b/src/test/java/com/exactpro/th2/infraoperator/util/StringsTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,13 +25,13 @@ class StringsTests { @Test - void isNullOrEmptyTest() { - String testStr = null; - assertTrue(Strings.isNullOrEmpty(testStr)); - testStr = ""; - assertTrue(Strings.isNullOrEmpty(testStr)); - testStr = "notNullOrEmpty"; - assertFalse(Strings.isNullOrEmpty(testStr)); + void anyPrefixMatchTest() { + List prefixes = List.of("a", "b", "c", "D"); + String namespace = "dev-someone"; + assertFalse(Strings.anyPrefixMatch(namespace, prefixes)); + prefixes = List.of("dev", "not-dev"); + assertTrue(Strings.anyPrefixMatch(namespace, prefixes)); + assertFalse(Strings.anyPrefixMatch(null, null)); } @Test diff --git a/src/test/kotlin/com/exactpro/th2/infraoperator/integration/DeleteRubbishOnStartTest.kt b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/DeleteRubbishOnStartTest.kt new file mode 100644 index 00000000..305e5de7 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/DeleteRubbishOnStartTest.kt @@ -0,0 +1,266 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.infraoperator.integration + +import com.exactpro.th2.infraoperator.Th2CrdController +import com.exactpro.th2.infraoperator.configuration.fields.RabbitMQNamespacePermissions +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_BOX_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_PIN_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_BOX_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_PIN_ALIAS +import com.exactpro.th2.infraoperator.spec.box.Th2Box +import com.exactpro.th2.infraoperator.spec.shared.status.RolloutPhase +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext.DIRECT +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext.toExchangeName +import com.exactpro.th2.infraoperator.util.createKubernetesClient +import com.rabbitmq.client.AMQP +import com.rabbitmq.client.Connection +import com.rabbitmq.http.client.Client +import io.fabric8.kubernetes.client.KubernetesClient +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Timeout +import org.junit.jupiter.api.assertAll +import org.junit.jupiter.api.io.TempDir +import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.k3s.K3sContainer +import java.nio.file.Path +import kotlin.test.Test + +@Tag("integration-test") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class DeleteRubbishOnStartTest { + + private lateinit var k3sContainer: K3sContainer + + private lateinit var rabbitMQContainer: RabbitMQContainer + + private lateinit var kubeClient: KubernetesClient + + private lateinit var rabbitMQClient: Client + + private lateinit var rabbitMQConnection: Connection + + @BeforeAll + @Timeout(30_000) + fun beforeAll(@TempDir tempDir: Path) { + k3sContainer = createK3sContainer() + rabbitMQContainer = createRabbitMQContainer() + + prepareTh2CfgDir( + k3sContainer.kubeConfigYaml, + createOperatorConfig( + rabbitMQContainer, + setOf(TH2_PREFIX), + RABBIT_MQ_V_HOST, + RABBIT_MQ_TOPIC_EXCHANGE, + RABBIT_MQ_NAMESPACE_PERMISSIONS, + ), + tempDir, + ) + + kubeClient = createKubernetesClient().apply { configureK3s() } + rabbitMQClient = createRabbitMQClient(rabbitMQContainer) + rabbitMQConnection = createRabbitMQConnection(rabbitMQContainer, RABBIT_MQ_V_HOST) + } + + @AfterAll + @Timeout(30_000) + fun afterAll() { + if (this::kubeClient.isInitialized) { + kubeClient.close() + } + if (this::k3sContainer.isInitialized) { + k3sContainer.stop() + } + if (this::rabbitMQContainer.isInitialized) { + rabbitMQContainer.stop() + } + if (this::rabbitMQConnection.isInitialized && rabbitMQConnection.isOpen) { + rabbitMQConnection.close() + } + } + + @Test + @Timeout(30_000) + fun deleteAllTest() { + val namespaces = listOf( + "${TH2_PREFIX}test-b", + "${TH2_PREFIX}test-c" + ) + val component = "test-component" + + rabbitMQConnection.createChannel().use { channel -> + val queues = mutableListOf() + val exchanges = mutableListOf() + + namespaces.forEach { namespace -> + channel.createQueue(namespace, "rubbish-component", PIN_NAME) + .assertQueue().queue.also(queues::add) + channel.createQueue(namespace, component, "rubbish-pin") + .assertQueue().queue.also(queues::add) + channel.createQueue(namespace, component, PIN_NAME) + .assertQueue().queue.also(queues::add) + channel.createQueue(namespace, MESSAGE_STORAGE_BOX_ALIAS, MESSAGE_STORAGE_PIN_ALIAS) + .assertQueue().queue.also(queues::add) + channel.createQueue(namespace, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS) + .assertQueue().queue.also(queues::add) + + toExchangeName(namespace).apply { + channel.createExchange(this, DIRECT) + rabbitMQClient.assertExchange(this, DIRECT, RABBIT_MQ_V_HOST) + exchanges.add(this) + } + } + + Th2CrdController().use { + assertAll( + queues.map { queue -> + { + rabbitMQClient.assertNoQueue(queue, RABBIT_MQ_V_HOST) + } + } + exchanges.map { exchange -> + { + rabbitMQClient.assertNoExchange(exchange, RABBIT_MQ_V_HOST) + } + } + listOf( + { rabbitMQClient.assertNoQueues("link\\[.*\\]", RABBIT_MQ_V_HOST) }, + { rabbitMQClient.assertNoExchanges("${TH2_PREFIX}.*", RABBIT_MQ_V_HOST) } + ) + ) + } + } + } + + @Test + fun deleteRubbishTest() { + var gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + + val namespaceB = "${TH2_PREFIX}test-b" + val namespaceC = "${TH2_PREFIX}test-c" + + val exchangeB = toExchangeName(namespaceB) + val exchangeC = toExchangeName(namespaceC) + + val component = "test-component" + + prepareNamespace(gitHash, namespaceB) + + rabbitMQConnection.createChannel().use { channel -> + channel.confirmSelect() + /** queue of not existed component */ + val queue01 = channel.createQueue(namespaceB, "rubbish-component", PIN_NAME).assertQueue() + + /** queue of not exited pin */ + val queue02 = channel.createQueue(namespaceB, component, "rubbish-pin").assertQueue() + + /** mstore queue of not existed namespace */ + val queue03 = channel.createQueue( + namespaceC, + MESSAGE_STORAGE_BOX_ALIAS, + MESSAGE_STORAGE_PIN_ALIAS + ).assertQueue() + + /** mstore queue of existed namespace */ + val queue11 = channel.createQueue( + namespaceB, + MESSAGE_STORAGE_BOX_ALIAS, + MESSAGE_STORAGE_PIN_ALIAS + ).assertQueue() + .also { + channel.basicPublish("", it.queue, null, "test-content".toByteArray()) + } + + /** queue of exited component and pin */ + val queue12 = channel.createQueue(namespaceB, component, PIN_NAME).assertQueue() + .also { + channel.basicPublish("", it.queue, null, "test-content".toByteArray()) + } + + /** exchange of not exited namespace */ + channel.createExchange(exchangeC, DIRECT) + rabbitMQClient.assertExchange(exchangeC, DIRECT, RABBIT_MQ_V_HOST) + + // TODO: add routing keys check: + // * existed key to exited queue + // * existed exchange A to existed queue B + // * not existed key to exited queue + + gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + val spec = """ + imageName: "ghcr.io/th2-net/th2-component" + imageVersion: "0.0.0" + type: th2-codec + pins: + mq: + subscribers: + - name: $PIN_NAME + attributes: [subscribe] + """.trimIndent() + kubeClient.createTh2CustomResource(exchangeB, component, gitHash, spec, ::Th2Box) + + Th2CrdController().use { + kubeClient.awaitPhase(exchangeB, component, RolloutPhase.SUCCEEDED, Th2Box::class.java) + + rabbitMQClient.assertNoQueue(queue01.queue, RABBIT_MQ_V_HOST) + rabbitMQClient.assertNoQueue(queue02.queue, RABBIT_MQ_V_HOST) + rabbitMQClient.assertNoQueue(queue03.queue, RABBIT_MQ_V_HOST) + + rabbitMQClient.assertQueue(queue11.queue, RABBIT_MQ_QUEUE_CLASSIC_TYPE, RABBIT_MQ_V_HOST) + rabbitMQClient.awaitQueueSize(queue11.queue, RABBIT_MQ_V_HOST, 1) + rabbitMQClient.assertQueue(queue12.queue, RABBIT_MQ_QUEUE_CLASSIC_TYPE, RABBIT_MQ_V_HOST) + rabbitMQClient.awaitQueueSize(queue12.queue, RABBIT_MQ_V_HOST, 1) + + rabbitMQClient.assertNoExchange(exchangeC, RABBIT_MQ_V_HOST) + } + } + } + + private fun prepareNamespace(gitHash: String, namespace: String) { + kubeClient.createNamespace(namespace) + kubeClient.createRabbitMQSecret(namespace, gitHash) + kubeClient.createRabbitMQAppConfigCfgMap( + namespace, + gitHash, + createRabbitMQConfig(rabbitMQContainer, RABBIT_MQ_V_HOST, toExchangeName(namespace), namespace) + ) + + kubeClient.createBookConfigCfgMap(namespace, gitHash, TH2_BOOK) + kubeClient.createLoggingCfgMap(namespace, gitHash) + kubeClient.createMQRouterCfgMap(namespace, gitHash) + kubeClient.createGrpcRouterCfgMap(namespace, gitHash) + kubeClient.createCradleManagerCfgMap(namespace, gitHash) + } + + private fun AMQP.Queue.DeclareOk.assertQueue(): AMQP.Queue.DeclareOk { + rabbitMQClient.assertQueue(queue, RABBIT_MQ_QUEUE_CLASSIC_TYPE, RABBIT_MQ_V_HOST) + return this + } + + companion object { + private const val TH2_PREFIX = "th2-" + private const val TH2_BOOK = "test_book" + + private val RABBIT_MQ_NAMESPACE_PERMISSIONS = RabbitMQNamespacePermissions() + private const val RABBIT_MQ_V_HOST = "/" + private const val RABBIT_MQ_TOPIC_EXCHANGE = "test-global-exchange" + + private const val PIN_NAME = "test-pin" + } +} diff --git a/src/test/kotlin/com/exactpro/th2/infraoperator/integration/IntegrationTest.kt b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/IntegrationTest.kt new file mode 100644 index 00000000..92ce56e1 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/IntegrationTest.kt @@ -0,0 +1,830 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.infraoperator.integration + +import com.exactpro.th2.infraoperator.Th2CrdController +import com.exactpro.th2.infraoperator.configuration.fields.RabbitMQNamespacePermissions +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.BOOK_CONFIG_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.BOOK_NAME_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.CHECKSUM_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.COMPONENT_NAME_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.CONFIG_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.CRADLE_MGR_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.CUSTOM_CONFIG_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.DICTIONARIES_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.DOCKER_IMAGE_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.EXTENDED_SETTINGS_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.GRPC_P2P_CONFIG_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.GRPC_ROUTER_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.IS_JOB_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.LOGGING_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.MQ_QUEUE_CONFIG_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.MQ_ROUTER_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.PROMETHEUS_CONFIG_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.PULL_SECRETS_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.ROOTLESS_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.SCHEMA_SECRETS_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.SECRET_PATHS_CONFIG_ALIAS +import com.exactpro.th2.infraoperator.operator.HelmReleaseTh2Op.SECRET_VALUES_CONFIG_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_BOX_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.EVENT_STORAGE_PIN_ALIAS +import com.exactpro.th2.infraoperator.operator.StoreHelmTh2Op.MESSAGE_STORAGE_BOX_ALIAS +import com.exactpro.th2.infraoperator.operator.manager.impl.Th2DictionaryEventHandler.DICTIONARY_SUFFIX +import com.exactpro.th2.infraoperator.spec.Th2CustomResource +import com.exactpro.th2.infraoperator.spec.box.Th2Box +import com.exactpro.th2.infraoperator.spec.corebox.Th2CoreBox +import com.exactpro.th2.infraoperator.spec.estore.Th2Estore +import com.exactpro.th2.infraoperator.spec.helmrelease.HelmRelease +import com.exactpro.th2.infraoperator.spec.helmrelease.HelmRelease.NAME_LENGTH_LIMIT +import com.exactpro.th2.infraoperator.spec.job.Th2Job +import com.exactpro.th2.infraoperator.spec.mstore.Th2Mstore +import com.exactpro.th2.infraoperator.spec.shared.status.RolloutPhase.DISABLED +import com.exactpro.th2.infraoperator.spec.shared.status.RolloutPhase.SUCCEEDED +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createEstoreQueue +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createMstoreQueue +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext.DIRECT +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext.TOPIC +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext.toExchangeName +import com.exactpro.th2.infraoperator.util.CustomResourceUtils.extractHashedName +import com.exactpro.th2.infraoperator.util.createKubernetesClient +import com.rabbitmq.http.client.Client +import io.fabric8.kubernetes.api.model.ConfigMap +import io.fabric8.kubernetes.client.KubernetesClient +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Timeout +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import org.junit.jupiter.params.provider.ValueSource +import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.k3s.K3sContainer +import strikt.api.Assertion +import strikt.api.expectThat +import strikt.assertions.getValue +import strikt.assertions.hasSize +import strikt.assertions.isA +import strikt.assertions.isEmpty +import strikt.assertions.isEqualTo +import strikt.assertions.isNotNull +import strikt.assertions.isNull +import java.nio.file.Path +import java.util.concurrent.TimeUnit.MINUTES + +@Tag("integration-test") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class IntegrationTest { + + private lateinit var k3sContainer: K3sContainer + + private lateinit var rabbitMQContainer: RabbitMQContainer + + private lateinit var kubeClient: KubernetesClient + + private lateinit var rabbitMQClient: Client + + private lateinit var controller: Th2CrdController + + @BeforeAll + @Timeout(30_000) + fun beforeAll(@TempDir tempDir: Path) { + k3sContainer = createK3sContainer() + rabbitMQContainer = createRabbitMQContainer() + + prepareTh2CfgDir( + k3sContainer.kubeConfigYaml, + createOperatorConfig( + rabbitMQContainer, + setOf(TH2_PREFIX), + RABBIT_MQ_V_HOST, + RABBIT_MQ_TOPIC_EXCHANGE, + RABBIT_MQ_NAMESPACE_PERMISSIONS, + ), + tempDir, + ) + + kubeClient = createKubernetesClient().apply { configureK3s() } + rabbitMQClient = createRabbitMQClient(rabbitMQContainer) + controller = Th2CrdController() + + rabbitMQClient.assertExchange(RABBIT_MQ_TOPIC_EXCHANGE, TOPIC, RABBIT_MQ_V_HOST) + } + + @AfterAll + @Timeout(30_000) + fun afterAll() { + if (this::kubeClient.isInitialized) { + kubeClient.close() + } + if (this::k3sContainer.isInitialized) { + k3sContainer.stop() + } + if (this::rabbitMQContainer.isInitialized) { + rabbitMQContainer.stop() + } + if (this::controller.isInitialized) { + controller.close() + } + } + + @BeforeEach + @Timeout(30_000) + fun beforeEach() { + val gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + kubeClient.createNamespace(TH2_NAMESPACE) + kubeClient.createRabbitMQSecret(TH2_NAMESPACE, gitHash) + kubeClient.createRabbitMQAppConfigCfgMap( + TH2_NAMESPACE, + gitHash, + createRabbitMQConfig( + rabbitMQContainer, + RABBIT_MQ_V_HOST, + RABBIT_MQ_TH2_EXCHANGE, + TH2_NAMESPACE, + ) + ) + + rabbitMQClient.assertUser(TH2_NAMESPACE, RABBIT_MQ_V_HOST, RABBIT_MQ_NAMESPACE_PERMISSIONS) + rabbitMQClient.assertExchange(RABBIT_MQ_TH2_EXCHANGE, DIRECT, RABBIT_MQ_V_HOST) + rabbitMQClient.assertQueue(createMstoreQueue(TH2_NAMESPACE), RABBIT_MQ_QUEUE_CLASSIC_TYPE, RABBIT_MQ_V_HOST) + rabbitMQClient.assertQueue(createEstoreQueue(TH2_NAMESPACE), RABBIT_MQ_QUEUE_CLASSIC_TYPE, RABBIT_MQ_V_HOST) + + kubeClient.createBookConfigCfgMap(TH2_NAMESPACE, gitHash, TH2_BOOK) + kubeClient.createLoggingCfgMap(TH2_NAMESPACE, gitHash) + kubeClient.createMQRouterCfgMap(TH2_NAMESPACE, gitHash) + kubeClient.createGrpcRouterCfgMap(TH2_NAMESPACE, gitHash) + kubeClient.createCradleManagerCfgMap(TH2_NAMESPACE, gitHash) + } + + @AfterEach + @Timeout(30_000) + fun afterEach() { + kubeClient.deleteNamespace(TH2_NAMESPACE, 1, MINUTES) + // FIXME: Secret not found "th2-test:Secret/rabbitMQ" + + rabbitMQClient.assertNoQueues("link\\[.*\\]", RABBIT_MQ_V_HOST) + rabbitMQClient.assertNoExchange(toExchangeName(TH2_NAMESPACE), RABBIT_MQ_V_HOST) + rabbitMQClient.assertNoUser(TH2_NAMESPACE) + + kubeClient.awaitNoResources(TH2_NAMESPACE) + kubeClient.awaitNoResources(TH2_NAMESPACE) + kubeClient.awaitNoResources(TH2_NAMESPACE) + kubeClient.awaitNoResources(TH2_NAMESPACE) + kubeClient.awaitNoResources(TH2_NAMESPACE) + kubeClient.awaitNoResources(TH2_NAMESPACE) + } + + interface StoreComponentTest { + fun `add component`() + } + + abstract inner class ComponentTest { + abstract val resourceClass: Class + abstract val specType: String + abstract val runAsJob: Boolean + abstract fun createResources(): T + + abstract fun add(name: String) + abstract fun disable(name: String) + abstract fun enable(name: String) + abstract fun `mq link`( + subClass: Class, + subConstructor: () -> Th2CustomResource, + subSpecType: String, + subRunAsJob: Boolean + ) + abstract fun `grpc link`( + clientClass: Class, + clientConstructor: () -> Th2CustomResource, + clientSpecType: String, + clientRunAsJob: Boolean + ) + + protected fun addTest(name: String) { + val gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + val spec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $specType + """.trimIndent() + + val resource = kubeClient.createTh2CustomResource(TH2_NAMESPACE, name, gitHash, spec, this::createResources) + kubeClient.awaitPhase(TH2_NAMESPACE, name, SUCCEEDED, resourceClass) + kubeClient.awaitResource( + TH2_NAMESPACE, + extractHashedName(resource) + ).assertMinCfg(name, runAsJob) + rabbitMQClient.assertBindings( + createEstoreQueue(TH2_NAMESPACE), + RABBIT_MQ_V_HOST, + setOf( + formatQueue(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS), + formatRoutingKey(TH2_NAMESPACE, name, EVENT_STORAGE_PIN_ALIAS) + ) + ) + } + + protected fun disableTest(name: String) { + var gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + var spec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $specType + disabled: false + """.trimIndent() + + val resource = kubeClient.createTh2CustomResource(TH2_NAMESPACE, name, gitHash, spec, this::createResources) + kubeClient.awaitPhase(TH2_NAMESPACE, name, SUCCEEDED, resourceClass) + kubeClient.awaitResource( + TH2_NAMESPACE, + extractHashedName(resource) + ).assertMinCfg(name, runAsJob) + rabbitMQClient.assertBindings( + createEstoreQueue(TH2_NAMESPACE), + RABBIT_MQ_V_HOST, + setOf( + formatQueue(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS), + formatRoutingKey(TH2_NAMESPACE, name, EVENT_STORAGE_PIN_ALIAS), + ) + ) + + gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + spec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $specType + disabled: true + """.trimIndent() + + kubeClient.modifyTh2CustomResource(TH2_NAMESPACE, name, gitHash, spec, resourceClass) + kubeClient.awaitPhase(TH2_NAMESPACE, name, DISABLED, resourceClass) + kubeClient.awaitNoResource(TH2_NAMESPACE, name) + rabbitMQClient.assertBindings( + createEstoreQueue(TH2_NAMESPACE), + RABBIT_MQ_V_HOST, + setOf( + formatQueue(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS), + formatRoutingKey(TH2_NAMESPACE, name, EVENT_STORAGE_PIN_ALIAS), + ) + ) + } + + protected fun enableTest(name: String) { + var gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + var spec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $specType + disabled: true + """.trimIndent() + + kubeClient.createTh2CustomResource(TH2_NAMESPACE, name, gitHash, spec, this::createResources) + kubeClient.awaitPhase(TH2_NAMESPACE, name, DISABLED, resourceClass) + kubeClient.awaitNoResource(TH2_NAMESPACE, name) + rabbitMQClient.assertBindings( + createEstoreQueue(TH2_NAMESPACE), + RABBIT_MQ_V_HOST, + setOf(formatQueue(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS)) + ) + + gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + spec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $specType + disabled: false + """.trimIndent() + + val resource = kubeClient.modifyTh2CustomResource(TH2_NAMESPACE, name, gitHash, spec, resourceClass) + kubeClient.awaitPhase(TH2_NAMESPACE, name, SUCCEEDED, resourceClass) + kubeClient.awaitResource( + TH2_NAMESPACE, + extractHashedName(resource) + ).assertMinCfg(name, runAsJob) + rabbitMQClient.assertBindings( + createEstoreQueue(TH2_NAMESPACE), + RABBIT_MQ_V_HOST, + setOf( + formatQueue(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS), + formatRoutingKey(TH2_NAMESPACE, name, EVENT_STORAGE_PIN_ALIAS) + ) + ) + } + + protected fun mkLinkTest( + subClass: Class, + subConstructor: () -> Th2CustomResource, + subSpecType: String, + subRunAsJob: Boolean, + ) { + val gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + val pubName = "test-publisher" + val subName = "test-subscriber" + + val pubSpec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $specType + pins: + mq: + publishers: + - name: $PUBLISH_PIN + attributes: [publish] + """.trimIndent() + + val subSpec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $subSpecType + pins: + mq: + subscribers: + - name: $SUBSCRIBE_PIN + attributes: [subscribe] + linkTo: + - box: $pubName + pin: $PUBLISH_PIN + """.trimIndent() + + kubeClient.createTh2CustomResource(TH2_NAMESPACE, pubName, gitHash, pubSpec, ::createResources) + kubeClient.createTh2CustomResource(TH2_NAMESPACE, subName, gitHash, subSpec, subConstructor) + + kubeClient.awaitPhase(TH2_NAMESPACE, pubName, SUCCEEDED, resourceClass) + kubeClient.awaitPhase(TH2_NAMESPACE, subName, SUCCEEDED, subClass) + + val queueName = formatQueue(TH2_NAMESPACE, subName, SUBSCRIBE_PIN) + val routingKey = formatRoutingKey(TH2_NAMESPACE, pubName, PUBLISH_PIN) + + kubeClient.awaitResource(TH2_NAMESPACE, pubName).assertMinCfg( + pubName, + runAsJob, + queues = createQueueCfg(PUBLISH_PIN, listOf("publish"), routingKey = routingKey, queueName = "") + ) + kubeClient.awaitResource(TH2_NAMESPACE, subName).assertMinCfg( + subName, + subRunAsJob, + queues = createQueueCfg(SUBSCRIBE_PIN, listOf("subscribe"), routingKey = "", queueName = queueName) + ) + + rabbitMQClient.assertBindings( + queueName, + RABBIT_MQ_V_HOST, + setOf(formatQueue(TH2_NAMESPACE, subName, SUBSCRIBE_PIN), routingKey) + ) + rabbitMQClient.assertBindings( + createEstoreQueue(TH2_NAMESPACE), + RABBIT_MQ_V_HOST, + setOf( + formatQueue(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS), + formatRoutingKey(TH2_NAMESPACE, pubName, EVENT_STORAGE_PIN_ALIAS), + formatRoutingKey(TH2_NAMESPACE, subName, EVENT_STORAGE_PIN_ALIAS), + ) + ) + } + + protected fun grpcLinkTest( + clientClass: Class, + clientConstructor: () -> Th2CustomResource, + clientSpecType: String, + clientRunAsJob: Boolean, + ) { + val gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + val serverName = "test-server" + val clientName = "test-client" + + val serverSpec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $specType + pins: + grpc: + server: + - name: $SERVER_PIN + serviceClasses: [$GRPC_SERVICE] + """.trimIndent() + + val clientSpec = """ + imageName: $IMAGE + imageVersion: $VERSION + type: $clientSpecType + pins: + grpc: + client: + - name: $CLIENT_PIN + serviceClass: $GRPC_SERVICE + strategy: robin + linkTo: + - box: $serverName + pin: $SERVER_PIN + """.trimIndent() + + kubeClient.createTh2CustomResource(TH2_NAMESPACE, serverName, gitHash, serverSpec, ::createResources) + kubeClient.createTh2CustomResource(TH2_NAMESPACE, clientName, gitHash, clientSpec, clientConstructor) + + kubeClient.awaitPhase(TH2_NAMESPACE, serverName, SUCCEEDED, resourceClass) + kubeClient.awaitPhase(TH2_NAMESPACE, clientName, SUCCEEDED, clientClass) + + kubeClient.awaitResource(TH2_NAMESPACE, serverName).assertMinCfg(serverName, runAsJob) + kubeClient.awaitResource(TH2_NAMESPACE, clientName).assertMinCfg( + clientName, + clientRunAsJob, + services = createGrpcCfg(serverName) + ) + + rabbitMQClient.assertBindings( + createEstoreQueue(TH2_NAMESPACE), + RABBIT_MQ_V_HOST, + setOf( + formatQueue(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS), + formatRoutingKey(TH2_NAMESPACE, serverName, EVENT_STORAGE_PIN_ALIAS), + formatRoutingKey(TH2_NAMESPACE, clientName, EVENT_STORAGE_PIN_ALIAS), + ) + ) + } + } + + @Nested + inner class Mstore : StoreComponentTest { + + @Test + @Timeout(30_000) + override fun `add component`() { + val gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + val spec = """ + imageName: ghcr.io/th2-net/th2-mstore + imageVersion: 0.0.0 + """.trimIndent() + kubeClient.createTh2CustomResource(TH2_NAMESPACE, MESSAGE_STORAGE_BOX_ALIAS, gitHash, spec, ::Th2Mstore) + kubeClient.awaitPhase(TH2_NAMESPACE, MESSAGE_STORAGE_BOX_ALIAS, SUCCEEDED) + kubeClient.awaitResource(TH2_NAMESPACE, MESSAGE_STORAGE_BOX_ALIAS) + // FIXME: estore should have binding +// println("Bindings: ${rabbitMQClient.getQueueBindings(RABBIT_MQ_V_HOST, createEstoreQueue(TH2_NAMESPACE))}") + } + } + + @Nested + inner class Estore : StoreComponentTest { + @Test + @Timeout(30_000) + override fun `add component`() { + val gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + val spec = """ + imageName: ghcr.io/th2-net/th2-estore + imageVersion: 0.0.0 + """.trimIndent() + kubeClient.createTh2CustomResource(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, gitHash, spec, ::Th2Estore) + kubeClient.awaitPhase(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, SUCCEEDED) + kubeClient.awaitResource(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS) + rabbitMQClient.assertBindings( + createEstoreQueue(TH2_NAMESPACE), + RABBIT_MQ_V_HOST, + setOf( + formatQueue(TH2_NAMESPACE, EVENT_STORAGE_BOX_ALIAS, EVENT_STORAGE_PIN_ALIAS), + ) + ) + } + } + + @Nested + inner class CoreComponent : ComponentTest() { + override val resourceClass: Class + get() = Th2CoreBox::class.java + override val specType: String + get() = "th2-rpt-data-provider" + override val runAsJob: Boolean + get() = false + + override fun createResources(): Th2CoreBox = Th2CoreBox() + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-core-component", "th2-core-component-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun add(name: String) = addTest(name) + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-core-component", "th2-core-component-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun disable(name: String) = disableTest(name) + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-core-component", "th2-core-component-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun enable(name: String) = enableTest(name) + + @Timeout(30_000) + @ParameterizedTest + @MethodSource("com.exactpro.th2.infraoperator.integration.IntegrationTest#mqLinkArguments") + override fun `mq link`( + subClass: Class, + subConstructor: () -> Th2CustomResource, + subSpecType: String, + subRunAsJob: Boolean + ) = mkLinkTest(subClass, subConstructor, subSpecType, subRunAsJob) + + @Timeout(30_000) + @ParameterizedTest + @MethodSource("com.exactpro.th2.infraoperator.integration.IntegrationTest#mqLinkArguments") + override fun `grpc link`( + clientClass: Class, + clientConstructor: () -> Th2CustomResource, + clientSpecType: String, + clientRunAsJob: Boolean + ) = grpcLinkTest(clientClass, clientConstructor, clientSpecType, clientRunAsJob) + } + + @Nested + inner class Component : ComponentTest() { + override val resourceClass: Class + get() = Th2Box::class.java + override val specType: String + get() = "th2-codec" + override val runAsJob: Boolean + get() = false + + override fun createResources(): Th2Box = Th2Box() + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-component", "th2-component-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun add(name: String) = addTest(name) + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-component", "th2-component-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun disable(name: String) = disableTest(name) + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-component", "th2-component-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun enable(name: String) = enableTest(name) + + @Timeout(30_000) + @ParameterizedTest + @MethodSource("com.exactpro.th2.infraoperator.integration.IntegrationTest#mqLinkArguments") + override fun `mq link`( + subClass: Class, + subConstructor: () -> Th2CustomResource, + subSpecType: String, + subRunAsJob: Boolean + ) = mkLinkTest(subClass, subConstructor, subSpecType, subRunAsJob) + + @Timeout(30_000) + @ParameterizedTest + @MethodSource("com.exactpro.th2.infraoperator.integration.IntegrationTest#mqLinkArguments") + override fun `grpc link`( + clientClass: Class, + clientConstructor: () -> Th2CustomResource, + clientSpecType: String, + clientRunAsJob: Boolean + ) = grpcLinkTest(clientClass, clientConstructor, clientSpecType, clientRunAsJob) + } + + @Nested + inner class Job : ComponentTest() { + override val resourceClass: Class + get() = Th2Job::class.java + override val specType: String + get() = "th2-job" // supported values: "th2-job" + override val runAsJob: Boolean + get() = true + + override fun createResources(): Th2Job = Th2Job() + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-job", "th2-job-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun add(name: String) = addTest(name) + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-job", "th2-job-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun disable(name: String) = disableTest(name) + + @Timeout(30_000) + @ParameterizedTest + @ValueSource(strings = ["th2-job", "th2-job-more-than-$NAME_LENGTH_LIMIT-characters"]) + override fun enable(name: String) = enableTest(name) + + @Timeout(30_000) + @ParameterizedTest + @MethodSource("com.exactpro.th2.infraoperator.integration.IntegrationTest#mqLinkArguments") + override fun `mq link`( + subClass: Class, + subConstructor: () -> Th2CustomResource, + subSpecType: String, + subRunAsJob: Boolean + ) = mkLinkTest(subClass, subConstructor, subSpecType, subRunAsJob) + + @Timeout(30_000) + @ParameterizedTest + @MethodSource("com.exactpro.th2.infraoperator.integration.IntegrationTest#mqLinkArguments") + override fun `grpc link`( + clientClass: Class, + clientConstructor: () -> Th2CustomResource, + clientSpecType: String, + clientRunAsJob: Boolean + ) = grpcLinkTest(clientClass, clientConstructor, clientSpecType, clientRunAsJob) + } + + @Nested + inner class Dictionary { + + @Test + @Timeout(30_000) + fun `add dictionary (short name)`() { + val gitHash = RESOURCE_GIT_HASH_COUNTER.incrementAndGet().toString() + val name = "th2-dictionary" + val spec = """ + data: $DICTIONARY_CONTENT + """.trimIndent() + + val annotations = createAnnotations(gitHash, spec.hashCode().toString()) + kubeClient.createTh2Dictionary( + TH2_NAMESPACE, + name, + annotations, + spec + ) + kubeClient.awaitResource(TH2_NAMESPACE, "$name$DICTIONARY_SUFFIX").also { configMap -> + expectThat(configMap) { + get { metadata }.and { + get { this.annotations } isEqualTo annotations + } + get { data }.isA>().and { + hasSize(1) + getValue("$name$DICTIONARY_SUFFIX") isEqualTo DICTIONARY_CONTENT + } + } + } + } + } + + companion object { + private const val TH2_PREFIX = "th2-" + private const val TH2_NAMESPACE = "${TH2_PREFIX}test" + private const val TH2_BOOK = "test_book" + + private val RABBIT_MQ_NAMESPACE_PERMISSIONS = RabbitMQNamespacePermissions() + private const val RABBIT_MQ_V_HOST = "/" + private const val RABBIT_MQ_TOPIC_EXCHANGE = "test-global-exchange" + private val RABBIT_MQ_TH2_EXCHANGE = toExchangeName(TH2_NAMESPACE) + + private const val PUBLISH_PIN = "test-publish-pin" + private const val SUBSCRIBE_PIN = "test-subscribe-pin" + + private const val SERVER_PIN = "test-server-pin" + private const val CLIENT_PIN = "test-client-pin" + private const val GRPC_SERVICE = "com.exactpro.th2.test.grpc.TestService" + + private const val IMAGE = "ghcr.io/th2-net/th2-estore" + private const val VERSION = "0.0.0" + + private const val DICTIONARY_CONTENT = "test-dictionary-content" + + @JvmStatic + fun mqLinkArguments() = listOf( + Arguments.of(Th2Job::class.java, ::Th2Job, "th2-job", true), + Arguments.of(Th2Box::class.java, ::Th2Box, "th2-codec", false), + Arguments.of(Th2CoreBox::class.java, ::Th2CoreBox, "th2-rpt-data-provider", false), + ) + + private fun createQueueCfg( + pinName: String, + attributes: List, + routingKey: String, + queueName: String, + ) = mapOf( + pinName to mapOf( + "attributes" to attributes, + "exchange" to RABBIT_MQ_TH2_EXCHANGE, + "filters" to emptyList(), + "name" to routingKey, + "queue" to queueName, + ) + ) + + private fun createGrpcCfg(serverName: String) = mapOf( + CLIENT_PIN to mapOf( + "endpoints" to mapOf( + "$serverName-endpoint" to mapOf( + "attributes" to emptyList(), + "host" to serverName, + "port" to 8080 + ) + ), + "filters" to emptyList(), + "service-class" to GRPC_SERVICE, + "strategy" to mapOf( + "endpoints" to listOf("$serverName-endpoint"), + "name" to "robin", + ) + ) + ) + + private fun HelmRelease.assertMinCfg( + name: String, + runAsJob: Boolean, + queues: Map> = emptyMap(), + services: Map> = emptyMap(), + ) { + expectThat(componentValuesSection) { + getValue(BOOK_CONFIG_ALIAS).isA>().hasSize(1).and { + getValue(BOOK_NAME_ALIAS) isEqualTo TH2_BOOK + } + getValue(CRADLE_MGR_ALIAS).isA>().hasSize(2).and { + getValue(CHECKSUM_ALIAS).isNotNull() + getValue(CONFIG_ALIAS).isNull() // FIXME: shouldn't be null + } + getValue(CUSTOM_CONFIG_ALIAS).isA>().isEmpty() + getValue(DICTIONARIES_ALIAS).isA>().isEmpty() // FIXME + getValue(PULL_SECRETS_ALIAS).isA>().isEmpty() // FIXME + getValue(EXTENDED_SETTINGS_ALIAS).isA>().isEmpty() + verifyGrpcCfg(services) + getValue(IS_JOB_ALIAS) isEqualTo runAsJob + getValue(SECRET_PATHS_CONFIG_ALIAS).isA>().isEmpty() + getValue(SECRET_VALUES_CONFIG_ALIAS).isA>().isEmpty() + getValue(SCHEMA_SECRETS_ALIAS).isA>().and { + getValue("cassandra") isEqualTo "cassandra" + getValue("rabbitMQ") isEqualTo "rabbitMQ" + } + getValue(GRPC_ROUTER_ALIAS).isA>().hasSize(2).and { + getValue(CHECKSUM_ALIAS).isNotNull() + getValue(CONFIG_ALIAS).isNull() // FIXME + } + getValue(DOCKER_IMAGE_ALIAS) isEqualTo "$IMAGE:$VERSION" + getValue(COMPONENT_NAME_ALIAS) isEqualTo name + getValue(ROOTLESS_ALIAS).isA>().hasSize(1).and { + getValue("enabled") isEqualTo false // FIXME + } + getValue(PROMETHEUS_CONFIG_ALIAS).isA>().hasSize(1).and { + getValue("enabled") isEqualTo true // FIXME + } + getValue(LOGGING_ALIAS).isA>().hasSize(2).and { + getValue(CHECKSUM_ALIAS).isNotNull() + getValue(CONFIG_ALIAS).isNull() // FIXME + } + getValue(MQ_ROUTER_ALIAS).isA>().hasSize(2).and { + getValue(CHECKSUM_ALIAS).isNotNull() + getValue(CONFIG_ALIAS).isNull() // FIXME + } + verifyMqCfg(name, queues) + } + } + + private fun Assertion.Builder>.verifyGrpcCfg(services: Map>) { + getValue(GRPC_P2P_CONFIG_ALIAS).isA>().hasSize(2).and { + getValue("server").isA>().hasSize(4) and { + getValue("attributes").isNull() // FIXME: add attributes + getValue("host").isNull() // FIXME: add host + getValue("port") isEqualTo 8080 + getValue("workers") isEqualTo 5 + } + getValue("services").isA>() isEqualTo services + } + } + + private fun Assertion.Builder>.verifyMqCfg( + name: String, + queues: Map> + ) { + getValue(MQ_QUEUE_CONFIG_ALIAS).isA>().hasSize(2).and { + getValue("globalNotification").isA>().hasSize(1).and { + getValue("exchange") isEqualTo RABBIT_MQ_TOPIC_EXCHANGE + } + getValue("queues").isA>().hasSize(1 + queues.size).and { + getValue(EVENT_STORAGE_PIN_ALIAS).isA>().hasSize(5).and { + getValue("attributes").isA>() isEqualTo listOf("publish", "event") + getValue("exchange") isEqualTo RABBIT_MQ_TH2_EXCHANGE + getValue("filters").isA>().isEmpty() // FIXME + getValue("name") isEqualTo formatRoutingKey(TH2_NAMESPACE, name, EVENT_STORAGE_PIN_ALIAS) + getValue("queue").isA().isEmpty() + } + queues.forEach { (key, value) -> + getValue(key).isA>() isEqualTo value + } + } + } + } + } +} diff --git a/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestIntegrationUtils.kt b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestIntegrationUtils.kt new file mode 100644 index 00000000..968c8723 --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestIntegrationUtils.kt @@ -0,0 +1,293 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.infraoperator.integration + +import com.exactpro.th2.infraoperator.configuration.ConfigLoader +import com.exactpro.th2.infraoperator.configuration.OperatorConfig +import com.exactpro.th2.infraoperator.configuration.fields.ChartSpec +import com.exactpro.th2.infraoperator.configuration.fields.RabbitMQConfig +import com.exactpro.th2.infraoperator.configuration.fields.RabbitMQManagementConfig +import com.exactpro.th2.infraoperator.configuration.fields.RabbitMQNamespacePermissions +import com.exactpro.th2.infraoperator.operator.manager.impl.ConfigMapEventHandler +import com.exactpro.th2.infraoperator.operator.manager.impl.ConfigMapEventHandler.BOOK_CONFIG_CM_NAME +import com.exactpro.th2.infraoperator.operator.manager.impl.ConfigMapEventHandler.DEFAULT_BOOK +import com.exactpro.th2.infraoperator.spec.shared.PrometheusConfiguration +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext +import com.exactpro.th2.infraoperator.util.JsonUtils +import com.fasterxml.jackson.module.kotlin.readValue +import com.rabbitmq.client.AMQP +import com.rabbitmq.client.Channel +import com.rabbitmq.client.Connection +import com.rabbitmq.client.ConnectionFactory +import com.rabbitmq.http.client.Client +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition +import io.fabric8.kubernetes.client.Config +import io.fabric8.kubernetes.client.KubernetesClient +import io.github.oshai.kotlinlogging.KotlinLogging +import org.slf4j.LoggerFactory +import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.containers.output.Slf4jLogConsumer +import org.testcontainers.k3s.K3sContainer +import org.testcontainers.lifecycle.Startable +import org.testcontainers.utility.DockerImageName +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import java.util.concurrent.atomic.AtomicLong +import kotlin.io.path.absolutePathString +import kotlin.io.path.createDirectories + +private val K3S_DOCKER_IMAGE = DockerImageName.parse("rancher/k3s:v1.21.3-k3s1") +private val RABBITMQ_DOCKER_IMAGE = DockerImageName.parse("rabbitmq:3.12.6-management") + +private val K_LOGGER = KotlinLogging.logger {} + +private val CRD_RESOURCE_NAMES = + setOf( + "helmreleases-crd.yaml", + "th2-box-crd.yaml", + "th2-core-box-crd.yaml", + "th2-dictionary-crd.yaml", + "th2-estore-crd.yaml", + "th2-job-crd.yaml", + "th2-mstore-crd.yaml", + ) + +val RESOURCE_GIT_HASH_COUNTER = AtomicLong(10_000_000) + +fun KubernetesClient.configureK3s() { + CRD_RESOURCE_NAMES + .asSequence() + .map(::loadCrd) + .map(this::resource) + .forEach { crd -> + crd.create() + K_LOGGER.info { "Applied CRD: ${crd.get().metadata.name}" } + } +} + +fun KubernetesClient.createRabbitMQSecret( + namespace: String, + gitHash: String, +) { + val data = mapOf(OperatorConfig.RABBITMQ_SECRET_PASSWORD_KEY to UUID.randomUUID().toString()) + createSecret( + namespace, + "rabbitmq", + createAnnotations(gitHash, data), + data, + ) +} + +fun KubernetesClient.createRabbitMQAppConfigCfgMap( + namespace: String, + gitHash: String, + data: RabbitMQConfig, +) { + val content = JsonUtils.JSON_MAPPER.writeValueAsString(data) + createConfigMap( + namespace, + OperatorConfig.DEFAULT_RABBITMQ_CONFIGMAP_NAME, + createAnnotations(gitHash, content), + mapOf(RabbitMQConfig.CONFIG_MAP_RABBITMQ_PROP_NAME to content), + ) +} + +fun KubernetesClient.createBookConfigCfgMap( + namespace: String, + gitHash: String, + book: String, +) { + createConfigMap( + namespace, + BOOK_CONFIG_CM_NAME, + createAnnotations(gitHash, book), + mapOf(DEFAULT_BOOK to book), + ) +} + +fun KubernetesClient.createLoggingCfgMap( + namespace: String, + gitHash: String, +) { + val data = mapOf("log4j2.properties" to "") + createConfigMap( + namespace, + ConfigMapEventHandler.LOGGING_CM_NAME, + createAnnotations(gitHash, data), + data, + ) +} + +fun KubernetesClient.createMQRouterCfgMap( + namespace: String, + gitHash: String, +) { + val data = mapOf("mq_router.json" to "{}") + createConfigMap( + namespace, + ConfigMapEventHandler.MQ_ROUTER_CM_NAME, + createAnnotations(gitHash, data), + data, + ) +} + +fun KubernetesClient.createGrpcRouterCfgMap( + namespace: String, + gitHash: String, +) { + val data = mapOf("grpc_router.json" to "{}") + createConfigMap( + namespace, + ConfigMapEventHandler.GRPC_ROUTER_CM_NAME, + createAnnotations(gitHash, data), + data, + ) +} + +fun KubernetesClient.createCradleManagerCfgMap( + namespace: String, + gitHash: String, +) { + val data = mapOf("cradle_manager.json" to "{}") + createConfigMap( + namespace, + ConfigMapEventHandler.CRADLE_MANAGER_CM_NAME, + createAnnotations(gitHash, data), + data, + ) +} + +fun createK3sContainer(): K3sContainer = K3sContainer(K3S_DOCKER_IMAGE) + .withLogConsumer(Slf4jLogConsumer(LoggerFactory.getLogger("K3S")).withSeparateOutputStreams()) + .also(Startable::start) + +fun createRabbitMQContainer(): RabbitMQContainer = RabbitMQContainer(RABBITMQ_DOCKER_IMAGE) + .withLogConsumer(Slf4jLogConsumer(LoggerFactory.getLogger("RABBIT_MQ")).withSeparateOutputStreams()) + .also(Startable::start) + +fun createRabbitMQClient(rabbitMQ: RabbitMQContainer): Client = + RabbitMQContext.createClient( + rabbitMQ.host, + rabbitMQ.httpPort, + rabbitMQ.adminUsername, + rabbitMQ.adminPassword, + ) + +fun createRabbitMQConnection(rabbitMQ: RabbitMQContainer, vHost: String): Connection = + ConnectionFactory().apply { + host = rabbitMQ.host + port = rabbitMQ.amqpPort + virtualHost = vHost + username = rabbitMQ.adminUsername + password = rabbitMQ.adminPassword + }.newConnection("integration-test") + +fun createRabbitMQConfig( + rabbitMQ: RabbitMQContainer, + vHost: String, + exchange: String, + user: String, +) = RabbitMQConfig( + rabbitMQ.amqpPort, + rabbitMQ.host, + vHost, + exchange, + user, + "${'$'}{RABBITMQ_PASS}", +) + +fun formatQueue(namespace: String, component: String, pin: String) = "link[$namespace:$component:$pin]" + +fun formatRoutingKey(namespace: String, component: String, pin: String) = "key[$namespace:$component:$pin]" + +fun Channel.createQueue( + namespace: String, + component: String, + pin: String, + durable: Boolean = true, + exclusive: Boolean = false, + autoDelete: Boolean = false, + arguments: Map = emptyMap(), +): AMQP.Queue.DeclareOk = queueDeclare( + formatQueue(namespace, component, pin), + durable, + exclusive, + autoDelete, + arguments +) + +fun Channel.createExchange( + exchange: String, + type: String, + durable: Boolean = true, + exclusive: Boolean = false, + autoDelete: Boolean = false, + arguments: Map = emptyMap(), +): AMQP.Exchange.DeclareOk = exchangeDeclare(exchange, type, durable, exclusive, autoDelete, arguments) + +fun prepareTh2CfgDir( + kubeConfigYaml: String, + operatorConfig: OperatorConfig, + baseDir: Path +) { + val configDir = baseDir.resolve("cfg") + val kubeCfgFile = configDir.resolve("kube-config.yaml") + val operatorCfgFile = configDir.resolve("infra-operator.yml") + configDir.createDirectories() + + Files.writeString(kubeCfgFile, kubeConfigYaml) + JsonUtils.YAML_MAPPER.writeValue(operatorCfgFile.toFile(), operatorConfig) + + System.setProperty(Config.KUBERNETES_KUBECONFIG_FILE, kubeCfgFile.absolutePathString()) + System.setProperty(ConfigLoader.CONFIG_FILE_SYSTEM_PROPERTY, operatorCfgFile.absolutePathString()) +} + +fun createOperatorConfig( + rabbitMQ: RabbitMQContainer, + namespacePrefixes: Set, + vHost: String, + topicExchange: String, + permissions: RabbitMQNamespacePermissions, +) = + OperatorConfig( + chart = ChartSpec(), + namespacePrefixes = namespacePrefixes, + rabbitMQManagement = + RabbitMQManagementConfig( + host = rabbitMQ.host, + managementPort = rabbitMQ.httpPort, + applicationPort = rabbitMQ.amqpPort, + vhostName = vHost, + exchangeName = topicExchange, + username = rabbitMQ.adminUsername, + password = rabbitMQ.adminPassword, + persistence = true, + schemaPermissions = permissions, + ), + prometheusConfiguration = + PrometheusConfiguration( + "0.0.0.0", + "9752", + false.toString(), + ), + ) + +private fun loadCrd(resourceName: String): CustomResourceDefinition = + requireNotNull(IntegrationTest::class.java.classLoader.getResource("crds/$resourceName")) { + "Resource '$resourceName' isn't found" + }.let(JsonUtils.YAML_MAPPER::readValue) diff --git a/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestKubernetesUtils.kt b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestKubernetesUtils.kt new file mode 100644 index 00000000..d60dbdce --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestKubernetesUtils.kt @@ -0,0 +1,236 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.infraoperator.integration + +import com.exactpro.th2.infraoperator.metrics.OperatorMetrics.KEY_DETECTION_TIME +import com.exactpro.th2.infraoperator.operator.manager.impl.ConfigMapEventHandler.SECRET_TYPE_OPAQUE +import com.exactpro.th2.infraoperator.spec.Th2CustomResource +import com.exactpro.th2.infraoperator.spec.Th2Spec +import com.exactpro.th2.infraoperator.spec.dictionary.Th2Dictionary +import com.exactpro.th2.infraoperator.spec.dictionary.Th2DictionarySpec +import com.exactpro.th2.infraoperator.spec.shared.status.RolloutPhase +import com.exactpro.th2.infraoperator.util.CustomResourceUtils.GIT_COMMIT_HASH +import com.exactpro.th2.infraoperator.util.ExtractUtils.KEY_SOURCE_HASH +import com.exactpro.th2.infraoperator.util.JsonUtils.YAML_MAPPER +import io.fabric8.kubernetes.api.model.ConfigMap +import io.fabric8.kubernetes.api.model.HasMetadata +import io.fabric8.kubernetes.api.model.Namespace +import io.fabric8.kubernetes.api.model.ObjectMeta +import io.fabric8.kubernetes.api.model.Secret +import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.dsl.Deletable +import io.github.oshai.kotlinlogging.KotlinLogging +import org.testcontainers.shaded.org.awaitility.Awaitility.await +import java.time.Instant +import java.util.Base64 +import java.util.UUID +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +private val K_LOGGER = KotlinLogging.logger {} + +fun KubernetesClient.createNamespace( + namespace: String, + timeout: Long = 200, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + resource( + Namespace().apply { + metadata = createMeta(namespace, null) + }, + ).create() + + await("createNamespace('$namespace')") + .timeout(timeout, unit) + .until { namespaces().withName(namespace) != null } +} + +fun KubernetesClient.deleteNamespace( + namespace: String, + timeout: Long = 200, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + namespaces() + .withName(namespace) + ?.awaitDeleteResource("deleteNamespace('$namespace')", timeout, unit) +} + +fun KubernetesClient.createSecret( + namespace: String, + name: String, + annotations: Map, + data: Map, +) { + resource( + Secret().apply { + metadata = createMeta(name, namespace, annotations) + this.type = SECRET_TYPE_OPAQUE + this.data = + data.mapValues { (_, value) -> + String(Base64.getEncoder().encode(value.toByteArray())) + } + }, + ).create() +} + +fun KubernetesClient.createConfigMap( + namespace: String, + name: String, + annotations: Map, + data: Map, +) { + resource( + ConfigMap().apply { + metadata = createMeta(name, namespace, annotations) + this.data.putAll(data) + }, + ).create() +} + +inline fun KubernetesClient.awaitResource( + namespace: String, + name: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +): T { + await("awaitResource ($name ${T::class.java})") + .timeout(timeout, unit) + .until { resources(T::class.java).inNamespace(namespace).withName(name).get() != null } + + return resources(T::class.java).inNamespace(namespace).withName(name).get() +} + +inline fun KubernetesClient.awaitNoResources( + namespace: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("awaitNoConfigMaps (${T::class.java})") + .timeout(timeout, unit) + .until { resources(T::class.java).inNamespace(namespace).list().items.isEmpty() } +} + +inline fun KubernetesClient.awaitNoResource( + namespace: String, + name: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("awaitNoResource ($name ${T::class.java})") + .timeout(timeout, unit) + .until { resources(T::class.java).inNamespace(namespace).withName(name).get() == null } +} + +inline fun KubernetesClient.awaitPhase( + namespace: String, + name: String, + phase: RolloutPhase, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) = awaitPhase(namespace, name, phase, T::class.java, timeout, unit) + +fun KubernetesClient.awaitPhase( + namespace: String, + name: String, + phase: RolloutPhase, + resourceType: Class, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("awaitStatus ($name $resourceType $phase)") + .timeout(timeout, unit) + .until { resources(resourceType)?.inNamespace(namespace)?.withName(name)?.get()?.status?.phase == phase } +} + +fun KubernetesClient.createTh2CustomResource( + namespace: String, + name: String, + gitHash: String, + spec: String, + create: () -> T, +): T = create().apply { + this.metadata = createMeta(name, namespace, createAnnotations(gitHash, spec.hashCode().toString())) + this.spec = YAML_MAPPER.readValue(spec, Th2Spec::class.java) +}.also { + resource(it).create() +} + +fun KubernetesClient.modifyTh2CustomResource( + namespace: String, + name: String, + gitHash: String, + spec: String, + resourceType: Class, +): T = resources(resourceType).inNamespace(namespace).withName(name).get().apply { + this.metadata.annotations.putAll(createAnnotations(gitHash, spec.hashCode().toString())) + this.metadata.generation += 1 + this.spec = YAML_MAPPER.readValue(spec, Th2Spec::class.java) +}.also { + resource(it).update() +} + +fun KubernetesClient.createTh2Dictionary( + namespace: String, + name: String, + annotations: Map, + spec: String, +) { + resource( + Th2Dictionary().apply { + this.metadata = createMeta(name, namespace, annotations) + this.spec = YAML_MAPPER.readValue(spec, Th2DictionarySpec::class.java) + } + ).create() +} + +fun createAnnotations( + gitHash: String, + sourceHash: Any, +) = mapOf( + KEY_DETECTION_TIME to System.currentTimeMillis().toString(), + GIT_COMMIT_HASH to gitHash, + KEY_SOURCE_HASH to sourceHash.hashCode().toString(), +) + +private fun Deletable.awaitDeleteResource( + alias: String, + timeout: Long, + unit: TimeUnit, +) { + delete().also { + K_LOGGER.info { "Deleted ($alias)" } + if (it.isEmpty()) return + } + + val counter = AtomicInteger(0) + await(alias) + .timeout(timeout, unit) + .until { delete().also { counter.incrementAndGet() }.isEmpty() } + K_LOGGER.info { "Deleted ($alias) after $counter iterations" } +} + +private fun createMeta( + name: String, + namespace: String?, + annotations: Map = emptyMap(), +): ObjectMeta = ObjectMeta().apply { + this.name = name + this.namespace = namespace + this.annotations.putAll(annotations) + this.uid = UUID.randomUUID().toString() + this.creationTimestamp = Instant.now().toString() +} diff --git a/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestRabbitMQUtils.kt b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestRabbitMQUtils.kt new file mode 100644 index 00000000..be5461ab --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/infraoperator/integration/TestRabbitMQUtils.kt @@ -0,0 +1,249 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.infraoperator.integration + +import com.exactpro.th2.infraoperator.configuration.fields.RabbitMQNamespacePermissions +import com.rabbitmq.http.client.Client +import com.rabbitmq.http.client.domain.DestinationType +import com.rabbitmq.http.client.domain.ExchangeInfo +import com.rabbitmq.http.client.domain.QueueInfo +import io.github.oshai.kotlinlogging.KotlinLogging +import org.junit.jupiter.api.assertAll +import org.testcontainers.shaded.org.awaitility.Awaitility.await +import java.util.concurrent.TimeUnit +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +const val RABBIT_MQ_QUEUE_CLASSIC_TYPE = "classic" + +private val K_LOGGER = KotlinLogging.logger {} + +fun Client.assertUser( + user: String, + vHost: String, + permissions: RabbitMQNamespacePermissions, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("assertUser('$user')") + .timeout(timeout, unit) + .until { users.firstOrNull { it.name == user } != null } + + val userInfo = users.firstOrNull { it.name == user } + assertNotNull(userInfo, "User '$user' isn't found") + assertEquals(emptyList(), userInfo.tags, "User '$user' has tags") + val userPermissions = this.permissions.firstOrNull { it.user == user } + assertNotNull(userPermissions, "User permission '$user' isn't found") + assertEquals(vHost, userPermissions.vhost, "User permission '$user' has incorrect vHost") + assertEquals( + permissions.configure, + userPermissions.configure, + "User permission '$user' has incorrect configure permission", + ) + assertEquals(permissions.read, userPermissions.read, "User permission '$user' has incorrect read permission") + assertEquals(permissions.write, userPermissions.write, "User permission '$user' has incorrect write permission") +} + +fun Client.assertNoUser( + user: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("assertNoUser('$user')") + .timeout(timeout, unit) + .until { users.firstOrNull { it.name == user } == null } +} + +fun Client.assertExchange( + exchange: String, + type: String, + vHost: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("assertExchange('$exchange')") + .timeout(timeout, unit) + .until { exchanges.firstOrNull { it.name == exchange } != null } + + val exchangeInfo = exchanges.firstOrNull { it.name == exchange } + assertNotNull(exchangeInfo, "Exchange '$exchange' isn't found") + assertEquals(type, exchangeInfo.type, "Exchange '$exchange' has incorrect type") + assertEquals(vHost, exchangeInfo.vhost, "Exchange '$exchange' has incorrect vHost") + assertEquals(emptyMap(), exchangeInfo.arguments, "Exchange '$exchange' has arguments") + assertTrue(exchangeInfo.isDurable, "Exchange '$exchange' isn't durable") + assertFalse(exchangeInfo.isInternal, "Exchange '$exchange' is internal") + assertFalse(exchangeInfo.isAutoDelete, "Exchange '$exchange' is auto delete") +} + +fun Client.assertNoExchanges( + exchangePattern: String, + vHost: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + val filter: (ExchangeInfo) -> Boolean = { + exchangeInfo -> + exchangeInfo.name.matches(Regex(exchangePattern)) && exchangeInfo.vhost == vHost + } + await("assertNoExchanges('$exchangePattern')") + .timeout(timeout, unit) + .conditionEvaluationListener { _ -> + K_LOGGER.debug { + "Remaining exchanges by '$exchangePattern': ${exchanges.filter(filter).map(ExchangeInfo::getName)}" + } + } + .until { exchanges.none(filter) } +} + +fun Client.assertNoExchange( + exchange: String, + vHost: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("assertNoExchange('$exchange')") + .timeout(timeout, unit) + .until { exchanges.firstOrNull { it.name == exchange && it.vhost == vHost } == null } +} + +fun Client.assertQueue( + queue: String, + type: String, + vHost: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +): QueueInfo { + await("assertQueue('$queue'") + .timeout(timeout, unit) + .until { getQueue(vHost, queue) != null } + + return assertNotNull(getQueue(vHost, queue), "Queue '$queue' isn't found") + .also { queueInfo -> + assertEquals(type, queueInfo.type, "Queue '$queue' has incorrect type") + assertEquals(emptyMap(), queueInfo.arguments, "Queue '$queue' has arguments") + assertTrue(queueInfo.isDurable, "Queue '$queue' isn't durable") + assertFalse(queueInfo.isExclusive, "Queue '$queue' is exclusive") + assertFalse(queueInfo.isAutoDelete, "Queue '$queue' is auto delete") + } +} + +fun Client.awaitQueueSize( + queue: String, + vHost: String, + size: Long, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("awaitQueueSize('$queue', size: $size") + .timeout(timeout, unit) + .until { getQueue(vHost, queue)?.let { it.messagesReady == size } } +} + +fun Client.assertBindings( + queue: String, + vHost: String, + routingKeys: Set = emptySet(), + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("assertBindings('$queue'), routing keys: $routingKeys") + .timeout(timeout, unit) + .until { getQueueBindings(vHost, queue).size == routingKeys.size } + + val queueBindings = getQueueBindings(vHost, queue) + assertAll( + routingKeys.map { routingKey -> + { + val queueBinding = + assertNotNull( + queueBindings.singleOrNull { it.routingKey == routingKey }, + "Queue '$queue' doesn't contain routing key, actual: $queueBindings", + ) + assertAll( + { + assertEquals( + vHost, + queueBinding.vhost, + "Binding has incorrect vHost for routing key '$routingKey' in queue '$queue'", + ) + }, + { + assertEquals( + emptyMap(), + queueBinding.arguments, + "Binding has arguments for routing key '$routingKey' in queue '$queue'", + ) + }, + { + assertEquals( + routingKey.replace("[", "%5B").replace(":", "%3A").replace("]", "%5D"), + queueBinding.propertiesKey, + "Binding has 'propertiesKey' for routing key '$routingKey' in queue '$queue'", + ) + }, + { + assertEquals( + queue, + queueBinding.destination, + "Binding has incorrect 'destination' for routing key '$routingKey' in queue '$queue'", + ) + }, + { + assertEquals( + DestinationType.QUEUE, + queueBinding.destinationType, + "Binding has incorrect 'destinationType' for routing key '$routingKey' in queue '$queue'", + ) + }, + ) + } + }, + ) +} + +fun Client.assertNoQueues( + queuePattern: String, + vHost: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + val filter: (QueueInfo) -> Boolean = { + queueInfo -> + queueInfo.name.matches(Regex(queuePattern)) && queueInfo.vhost == vHost + } + await("assertNoQueues('$queuePattern')") + .timeout(timeout, unit) + .conditionEvaluationListener { _ -> + K_LOGGER.debug { + "Remaining queues by '$queuePattern': ${queues.filter(filter).map(QueueInfo::getName)}" + } + } + .until { queues.none(filter) } +} + +fun Client.assertNoQueue( + name: String, + vHost: String, + timeout: Long = 5_000, + unit: TimeUnit = TimeUnit.MILLISECONDS, +) { + await("assertNoQueue('$name')") + .timeout(timeout, unit) + .until { getQueue(vHost, name) == null } +} diff --git a/src/test/kotlin/com/exactpro/th2/infraoperator/util/RabbitMQUtilsTest.kt b/src/test/kotlin/com/exactpro/th2/infraoperator/util/RabbitMQUtilsTest.kt new file mode 100644 index 00000000..7c76e44e --- /dev/null +++ b/src/test/kotlin/com/exactpro/th2/infraoperator/util/RabbitMQUtilsTest.kt @@ -0,0 +1,222 @@ +/* + * Copyright 2024-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.infraoperator.util + +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createEstoreQueue +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.createMstoreQueue +import com.exactpro.th2.infraoperator.spec.strategy.linkresolver.mq.RabbitMQContext.toExchangeName +import com.rabbitmq.client.Channel +import io.fabric8.kubernetes.api.model.KubernetesResourceList +import io.fabric8.kubernetes.api.model.Namespace +import io.fabric8.kubernetes.api.model.NamespaceList +import io.fabric8.kubernetes.api.model.ObjectMeta +import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.dsl.MixedOperation +import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation +import io.fabric8.kubernetes.client.dsl.Resource +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import com.exactpro.th2.infraoperator.spec.Th2CustomResource as CR + +private const val TOPIC_EXCHANGE_NAME = "test-global-exchange" + +class RabbitMQUtilsTest { + @Test + fun `no ns and topic exchange`() { + val client: KubernetesClient = mockKubernetesClient() + val actual = + ResourceHolder( + exchanges = hashSetOf(TOPIC_EXCHANGE_NAME), + ).filterRubbishResources( + client, + setOf("th2"), + TOPIC_EXCHANGE_NAME, + ) + val expected = + ResourceHolder( + exchanges = hashSetOf(TOPIC_EXCHANGE_NAME), + ) + + assertEquals(expected, actual) + } + + @Test + fun `no ns and rubbish exchange`() { + val exchangeName = "th2-test-exchange" + val client: KubernetesClient = mockKubernetesClient() + val actual = + ResourceHolder( + exchanges = hashSetOf(exchangeName), + ).filterRubbishResources( + client, + setOf("th2"), + TOPIC_EXCHANGE_NAME, + ) + val expected = + ResourceHolder( + exchanges = hashSetOf(exchangeName), + ) + + assertEquals(expected, actual) + } + + @Test + fun `no ns and rubbish queue`() { + val queueName = "test-link[th2-test-namespace:test-component:test-pin]" + val client: KubernetesClient = mockKubernetesClient() + val actual = + ResourceHolder( + queues = hashSetOf(queueName), + ).filterRubbishResources( + client, + setOf("th2"), + TOPIC_EXCHANGE_NAME, + ) + val expected = + ResourceHolder( + queues = hashSetOf(queueName), + ) + + assertEquals(expected, actual) + } + + @Test + fun `one ns and rubbish exchange`() { + val exchangeName = "th2-test-exchange" + val namespaceName = "th2-test-active-namespace" + val client: KubernetesClient = + mockKubernetesClient( + setOf(namespaceName), + ) + val actual = + ResourceHolder( + exchanges = hashSetOf(exchangeName, toExchangeName(namespaceName), TOPIC_EXCHANGE_NAME), + ).filterRubbishResources( + client, + setOf("th2"), + TOPIC_EXCHANGE_NAME, + ) + val expected = + ResourceHolder( + exchanges = hashSetOf(exchangeName), + ) + + assertEquals(expected, actual) + } + + @Test + fun `one ns and rubbish queue`() { + val queueName = "test-link[th2-test-namespace:test-component:test-pin]" + val namespaceName = "th2-test-active-namespace" + val client: KubernetesClient = + mockKubernetesClient( + setOf(namespaceName), + ) + val actual = + ResourceHolder( + queues = hashSetOf(queueName), + exchanges = hashSetOf(toExchangeName(namespaceName), TOPIC_EXCHANGE_NAME), + ).filterRubbishResources( + client, + setOf("th2"), + TOPIC_EXCHANGE_NAME, + ) + val expected = + ResourceHolder( + queues = hashSetOf(queueName), + ) + + assertEquals(expected, actual) + } + + @Test + fun `empty ns and store queues`() { + val namespaceName = "th2-test-active-namespace" + val client: KubernetesClient = + mockKubernetesClient( + setOf(namespaceName), + ) + val actual = + ResourceHolder( + queues = hashSetOf(createMstoreQueue(namespaceName), createEstoreQueue(namespaceName)), + exchanges = hashSetOf(toExchangeName(namespaceName), TOPIC_EXCHANGE_NAME), + ).filterRubbishResources( + client, + setOf("th2"), + TOPIC_EXCHANGE_NAME, + ) + val expected = ResourceHolder() + + assertEquals(expected, actual) + } + + @Test + fun `delete rubbish`() { + val channel: Channel = mock {} + val resourceHolder = + ResourceHolder( + hashSetOf("queueA", "queueB"), + hashSetOf("exchangeA", "exchangeB"), + ) + deleteRabbitMQRubbish( + resourceHolder, + ) { channel } + + resourceHolder.queues.forEach { + verify(channel).queueDelete(it) + } + + resourceHolder.exchanges.forEach { + verify(channel).exchangeDelete(it) + } + } + + companion object { + fun mockKubernetesClient(namespaceNames: Set = emptySet()): KubernetesClient { + val namespaceList = + NamespaceList().apply { + items = + namespaceNames.map { namespaceName -> + Namespace().apply { + metadata = + ObjectMeta().apply { + name = namespaceName + } + } + } + } + val namespaces: NonNamespaceOperation> = + mock { + on { list() }.thenReturn(namespaceList) + } + + val mixedOperation: + MixedOperation, Resource> = + mock { + on { inNamespace(any()) }.thenReturn(it) + on { resources() }.thenAnswer { emptyList().stream() } + } + return mock { + on { namespaces() }.thenReturn(namespaces) + on { resources(any>()) }.thenReturn(mixedOperation) + } + } + } +} diff --git a/src/test/resources/crds/helmreleases-crd.yaml b/src/test/resources/crds/helmreleases-crd.yaml new file mode 100644 index 00000000..35ec7a94 --- /dev/null +++ b/src/test/resources/crds/helmreleases-crd.yaml @@ -0,0 +1,422 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: helmreleases.helm.fluxcd.io +spec: + conversion: + strategy: None + group: helm.fluxcd.io + names: + kind: HelmRelease + listKind: HelmReleaseList + plural: helmreleases + shortNames: + - hr + - hrs + singular: helmrelease + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Release is the name of the Helm release, as given by Helm. + jsonPath: .status.releaseName + name: Release + type: string + - description: Phase is the current release phase being performed for the HelmRelease. + jsonPath: .status.phase + name: Phase + type: string + - description: ReleaseStatus is the status of the Helm release, as given by Helm. + jsonPath: .status.releaseStatus + name: ReleaseStatus + type: string + - jsonPath: .status.conditions[?(@.type=="Released")].message + name: Message + type: string + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: HelmRelease is a type to represent a Helm release. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + chart: + properties: + chartPullSecret: + description: ChartPullSecret holds the reference to the authentication + secret for accessing the Helm repository using HTTPS basic auth. + NOT IMPLEMENTED! + properties: + name: + type: string + required: + - name + type: object + git: + description: Git URL is the URL of the Git repository, e.g. `git@github.com:org/repo`, + `http://github.com/org/repo`, or `ssh://git@example.com:2222/org/repo.git`. + type: string + name: + description: Name is the name of the Helm chart _without_ an alias, + e.g. redis (for `helm upgrade [flags] stable/redis`). + type: string + path: + description: Path is the path to the chart relative to the repository + root. + type: string + ref: + description: Ref is the Git branch (or other reference) to use. + Defaults to 'master', or the configured default Git ref. + type: string + repository: + description: RepoURL is the URL of the Helm repository, e.g. `https://kubernetes-charts.storage.googleapis.com` + or `https://charts.example.com`. + type: string + secretRef: + description: SecretRef holds the authentication secret for accessing + the Git repository (over HTTPS). The credentials will be added + to an HTTPS GitURL before the mirror is started. + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + skipDepUpdate: + description: SkipDepUpdate will tell the operator to skip running + 'helm dep update' before installing or upgrading the chart, + the chart dependencies _must_ be present for this to succeed. + type: boolean + version: + description: Version is the targeted Helm chart version, e.g. + 7.0.1. + type: string + type: object + disableOpenAPIValidation: + description: DisableOpenAPIValidation controls whether OpenAPI validation + is enforced. + type: boolean + forceUpgrade: + description: Force will mark this Helm release to `--force` upgrades. + This forces the resource updates through delete/recreate if needed. + type: boolean + helmVersion: + description: 'HelmVersion is the version of Helm to target. If not + supplied, the lowest _enabled Helm version_ will be targeted. Valid + HelmVersion values are: "v2", "v3"' + enum: + - v2 + - v3 + type: string + maxHistory: + description: MaxHistory is the maximum amount of revisions to keep + for the Helm release. If not supplied, it defaults to 10. + type: integer + releaseName: + description: ReleaseName is the name of the The Helm release. If not + supplied, it will be generated by affixing the namespace to the + resource name. + type: string + resetValues: + description: ResetValues will mark this Helm release to reset the + values to the defaults of the targeted chart before performing an + upgrade. Not explicitly setting this to `false` equals to `true` + due to the declarative nature of the operator. + type: boolean + rollback: + description: The rollback settings for this Helm release. + properties: + disableHooks: + description: DisableHooks will mark this Helm release to prevent + hooks from running during the rollback. + type: boolean + enable: + description: Enable will mark this Helm release for rollbacks. + type: boolean + force: + description: Force will mark this Helm release to `--force` rollbacks. + This forces the resource updates through delete/recreate if + needed. + type: boolean + maxRetries: + description: MaxRetries is the maximum amount of upgrade retries + the operator should make before bailing. + format: int64 + type: integer + recreate: + description: Recreate will mark this Helm release to `--recreate-pods` + for if applicable. This performs pod restarts. + type: boolean + retry: + description: Retry will mark this Helm release for upgrade retries + after a rollback. + type: boolean + timeout: + description: Timeout is the time to wait for any individual Kubernetes + operation (like Jobs for hooks) during rollback. + format: int64 + type: integer + wait: + description: Wait will mark this Helm release to wait until all + Pods, PVCs, Services, and minimum number of Pods of a Deployment, + StatefulSet, or ReplicaSet are in a ready state before marking + the release as successful. + type: boolean + type: object + skipCRDs: + description: SkipCRDs will mark this Helm release to skip the creation + of CRDs during a Helm 3 installation. + type: boolean + targetNamespace: + description: TargetNamespace overrides the targeted namespace for + the Helm release. The default namespace equals to the namespace + of the HelmRelease resource. + type: string + test: + description: The test settings for this Helm release. + properties: + cleanup: + description: Cleanup, when targeting Helm 2, determines whether + to delete test pods between each test run initiated by the Helm + Operator. + type: boolean + enable: + description: Enable will mark this Helm release for tests. + type: boolean + ignoreFailures: + description: IgnoreFailures will cause a Helm release to be rolled + back if it fails otherwise it will be left in a released state + type: boolean + timeout: + description: Timeout is the time to wait for any individual Kubernetes + operation (like Jobs for hooks) during test. + format: int64 + type: integer + type: object + timeout: + description: Timeout is the time to wait for any individual Kubernetes + operation (like Jobs for hooks) during installation and upgrade + operations. + format: int64 + type: integer + valueFileSecrets: + description: ValueFileSecrets holds the local name references to secrets. + DEPRECATED, use ValuesFrom.secretKeyRef instead. + items: + properties: + name: + type: string + required: + - name + type: object + type: array + values: + description: Values holds the values for this Helm release. + x-kubernetes-preserve-unknown-fields: true + valuesFrom: + items: + properties: + chartFileRef: + description: The reference to a local chart file with release + values. + properties: + optional: + description: Optional will mark this ChartFileSelector as + optional. The result of this are that operations are permitted + without the source, due to it e.g. being temporarily unavailable. + type: boolean + path: + description: Path is the file path to the source relative + to the chart root. + type: string + required: + - path + type: object + configMapKeyRef: + description: The reference to a config map with release values. + properties: + key: + type: string + name: + type: string + namespace: + type: string + optional: + type: boolean + required: + - name + type: object + externalSourceRef: + description: The reference to an external source with release + values. + properties: + optional: + description: Optional will mark this ExternalSourceSelector + as optional. The result of this are that operations are + permitted without the source, due to it e.g. being temporarily + unavailable. + type: boolean + url: + description: URL is the URL of the external source. + type: string + required: + - url + type: object + secretKeyRef: + description: The reference to a secret with release values. + properties: + key: + type: string + name: + type: string + namespace: + type: string + optional: + type: boolean + required: + - name + type: object + type: object + type: array + wait: + description: Wait will mark this Helm release to wait until all Pods, + PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, + or ReplicaSet are in a ready state before marking the release as + successful. + type: boolean + required: + - chart + type: object + status: + description: HelmReleaseStatus contains status information about an HelmRelease. + properties: + conditions: + description: Conditions contains observations of the resource's state, + e.g., has the chart which it refers to been fetched. + items: + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding + to the last status change of this condition. + format: date-time + type: string + lastUpdateTime: + description: LastUpdateTime is the timestamp corresponding to + the last status update of this condition. + format: date-time + type: string + message: + description: Message is a human readable description of the + details of the last transition, complementing reason. + type: string + reason: + description: Reason is a brief machine readable explanation + for the condition's last transition. + type: string + status: + description: Status of the condition, one of ('True', 'False', + 'Unknown'). + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: Type of the condition, one of ('ChartFetched', + 'Deployed', 'Released', 'RolledBack', 'Tested'). + enum: + - ChartFetched + - Deployed + - Released + - RolledBack + - Tested + type: string + required: + - status + - type + type: object + type: array + lastAttemptedRevision: + description: LastAttemptedRevision is the revision of the latest chart + sync, and may be of a failed release. + type: string + observedGeneration: + description: ObservedGeneration is the most recent generation observed + by the operator. + format: int64 + type: integer + phase: + description: Phase the release is in, one of ('ChartFetched', 'ChartFetchFailed', + 'Installing', 'Upgrading', 'Deployed', 'DeployFailed', 'Testing', + 'TestFailed', 'Tested', 'Succeeded', 'RollingBack', 'RolledBack', + 'RollbackFailed') + enum: + - ChartFetched + - ChartFetchFailed + - Installing + - Upgrading + - Deployed + - DeployFailed + - Testing + - TestFailed + - Tested + - Succeeded + - Failed + - RollingBack + - RolledBack + - RollbackFailed + type: string + releaseName: + description: ReleaseName is the name as either supplied or generated. + type: string + releaseStatus: + description: ReleaseStatus is the status as given by Helm for the + release managed by this resource. + type: string + revision: + description: Revision holds the Git hash or version of the chart currently + deployed. + type: string + rollbackCount: + description: RollbackCount records the amount of rollback attempts + made, it is incremented after a rollback failure and reset after + a successful upgrade or revision change. + format: int64 + type: integer + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + listKind: "" + plural: "" + singular: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/src/test/resources/log4j2.properties b/src/test/resources/log4j2.properties new file mode 100644 index 00000000..8009fbc8 --- /dev/null +++ b/src/test/resources/log4j2.properties @@ -0,0 +1,17 @@ +name = CommonJConfig +# Logging level related to initialization of Log4j +status = warn + +# Console appender configuration +appender.console.type = Console +appender.console.name = ConsoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n + +logger.th2.name= com.exactpro.th2 +logger.th2.level= DEBUG + +# Root logger level +rootLogger.level = INFO +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = ConsoleLogger