Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TH2-4267 split common #232

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

TH2-4267 split common #232

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4293919
[TH2-1436] dynamic config and module loading
Xanclry Sep 30, 2022
c64c481
Merge branch 'master' into th2-1436-split-common
Xanclry Sep 30, 2022
fb7afa6
[TH2-1436] version update, small changes
Xanclry Oct 4, 2022
7b8c635
[TH2-1436] using jcommander for cli parsing
Xanclry Nov 29, 2022
ed81c24
[TH2-1436] modules caching, creating config for ConfigurationProvider…
Xanclry Dec 1, 2022
07dc700
[TH2-1436] added kapt plugin
Xanclry Dec 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,12 +174,17 @@ dependencies {
implementation 'com.google.protobuf:protobuf-java-util'
implementation 'com.exactpro.th2:grpc-service-generator:3.2.2'

implementation 'com.exactpro.th2:common-j-cassandra-cradle:0.0.1-th2-1436-init-3181374693-6b81dc9-SNAPSHOT'
implementation "com.exactpro.th2:common-api-j:0.0.1-th2-1436-init-3180908004-e67b670-SNAPSHOT"
// todo remove
// implementation 'com.exactpro.th2:full-cradle-manager:2.22.0-local'
implementation 'com.exactpro.th2:basic-cradle-manager:2.22.0-local'
implementation 'com.exactpro.th2:cradle-manager-api:2.22.0-local'
implementation "com.exactpro.th2:common-api-j:0.0.1-th2-1436-init-3574652642-d3de09f-SNAPSHOT"

annotationProcessor 'com.google.auto.service:auto-service:1.0.1'
compileOnly 'com.google.auto.service:auto-service:1.0.1'

implementation 'com.beust:jcommander:1.82'
implementation "commons-cli:commons-cli"
Xanclry marked this conversation as resolved.
Show resolved Hide resolved

//FIXME: Add these dependencies as api to grpc-... artifacts
implementation "io.grpc:grpc-protobuf"
Expand Down
138 changes: 58 additions & 80 deletions src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,22 @@

package com.exactpro.th2.common.schema.factory;

import com.beust.jcommander.JCommander;
import com.exactpro.th2.common.ConfigurationProvider;
import com.exactpro.th2.common.ConfigurationProviderFactory;
import com.exactpro.th2.common.Module;
import com.exactpro.th2.common.ModuleApi;
import com.exactpro.th2.common.cli.CommonFactoryArgs;
import com.exactpro.th2.common.grpc.EventBatch;
import com.exactpro.th2.common.grpc.MessageBatch;
import com.exactpro.th2.common.grpc.MessageGroupBatch;
import com.exactpro.th2.common.grpc.RawMessageBatch;
import com.exactpro.th2.common.metrics.PrometheusConfiguration;
import com.exactpro.th2.common.module.provider.FileConfigurationProvider;
import com.exactpro.th2.common.module.provider.FileConfigurationProviderConfig;
import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration;
import com.exactpro.th2.common.schema.configuration.Configuration;
import com.exactpro.th2.common.schema.configuration.ConfigurationManager;
import com.exactpro.th2.common.schema.configuration.ConfigurationProviderConfig;
import com.exactpro.th2.common.schema.dictionary.DictionaryType;
import com.exactpro.th2.common.schema.event.EventBatchRouter;
import com.exactpro.th2.common.schema.grpc.configuration.GrpcConfiguration;
Expand All @@ -48,11 +52,6 @@
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.dsl.Resource;
import kotlin.text.Charsets;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -79,8 +78,6 @@
import java.util.stream.Stream;

import static com.exactpro.th2.common.schema.util.ArchiveUtils.getGzipBase64StringDecoder;
import static com.exactpro.th2.common.util.CommandLineUtilsKt.createLongOption;
import static com.exactpro.th2.common.util.CommandLineUtilsKt.createLongOptionWithUnlimitedArgs;
import static java.util.Collections.emptyMap;
import static java.util.Objects.requireNonNull;
import static java.util.Objects.requireNonNullElse;
Expand All @@ -97,12 +94,10 @@ public class CommonFactory extends AbstractCommonFactory implements ModuleApi {
private static final String ROUTER_MQ_FILE_NAME = "mq.json";
private static final String GRPC_FILE_NAME = "grpc.json";
private static final String ROUTER_GRPC_FILE_NAME = "grpc_router.json";
// private static final String CRADLE_CONFIDENTIAL_FILE_NAME = "cradle.json";
private static final String PROMETHEUS_FILE_NAME = "prometheus.json";
private static final String CUSTOM_FILE_NAME = "custom.json";
private static final String BOX_FILE_NAME = "box.json";
private static final String CONNECTION_MANAGER_CONF_FILE_NAME = "mq_router.json";
// private static final String CRADLE_NON_CONFIDENTIAL_FILE_NAME = "cradle_manager.json";

/** @deprecated please use {@link #DICTIONARY_ALIAS_DIR_NAME} */
@Deprecated
Expand Down Expand Up @@ -149,7 +144,7 @@ protected CommonFactory(Class<? extends MessageRouter<MessageBatch>> messageRout
Map<String, String> environmentVariables,
ConfigurationManager configurationManager,
Class<? extends ConfigurationProvider> configurationProviderClass,
String[] configurationProviderArgs) {
ConfigurationProviderConfig providerConfig) {
super(messageRouterParsedBatchClass, messageRouterRawBatchClass, messageRouterMessageGroupBatchClass, eventBatchRouterClass, grpcRouterClass, environmentVariables);

this.custom = defaultPathIfNull(custom, CUSTOM_FILE_NAME);
Expand All @@ -159,7 +154,7 @@ protected CommonFactory(Class<? extends MessageRouter<MessageBatch>> messageRout
this.configurationManager = configurationManager;

var factory = loadFactoryForProvider(configurationProviderClass);
this.configurationProvider = factory.get().createProvider(configurationProviderArgs);
this.configurationProvider = factory.get().create(providerConfig);

start();
}
Expand All @@ -177,7 +172,7 @@ public CommonFactory(FactorySettings settings) {
settings.getVariables(),
createConfigurationManager(settings),
settings.getConfigurationProviderClass(),
settings.getConfigurationProviderArgs());
settings.getConfigurationProviderConfig());
}

/**
Expand All @@ -197,7 +192,7 @@ public CommonFactory(Class<? extends MessageRouter<MessageBatch>> messageRouterP
eventBatchRouterClass,
grpcRouterClass,
FileConfigurationProvider.class,
new String[0],
new FileConfigurationProviderConfig(),
rabbitMQ,
routerMQ,
null,
Expand Down Expand Up @@ -311,48 +306,21 @@ protected ConfigurationManager getConfigurationManager() {
* @throws IllegalArgumentException - Cannot parse command line arguments
*/
public static CommonFactory createFromArguments(String... args) {
Options options = new Options();

Option configOption = new Option("c", "configs", true, null);
options.addOption(configOption);

Option rabbitConfigurationOption = createLongOption(options, "rabbitConfiguration");
Option messageRouterConfigurationOption = createLongOption(options, "messageRouterConfiguration");
Option grpcRouterConfigurationOption = createLongOption(options, "grpcRouterConfiguration");
Option grpcConfigurationOption = createLongOption(options, "grpcConfiguration");
Option grpcRouterConfigOption = createLongOption(options, "grpcRouterConfig");
Option customConfigurationOption = createLongOption(options, "customConfiguration");
Option dictionariesDirOption = createLongOption(options, "dictionariesDir");
Option prometheusConfigurationOption = createLongOption(options, "prometheusConfiguration");
Option boxConfigurationOption = createLongOption(options, "boxConfiguration");
Option namespaceOption = createLongOption(options, "namespace");
Option boxNameOption = createLongOption(options, "boxName");
Option contextNameOption = createLongOption(options, "contextName");
Option dictionariesOption = createLongOptionWithUnlimitedArgs(options, "dictionaries");
Option connectionManagerConfigurationOption = createLongOption(options, "connectionManagerConfiguration");
Option configurationProviderClassOption = createLongOption(options, "configurationProviderClass");

configurationProviderFactoryLoader.forEach(configurationProviderFactory -> {
configurationProviderFactory.addOwnOptionsToCmd(options);
});

try {
CommandLine cmd = new DefaultParser().parse(options, args);
CommonFactoryArgs parsedArgs = parseAsCommonConfig(args);

String configs = cmd.getOptionValue(configOption.getLongOpt());
String configs = parsedArgs.config;

if (cmd.hasOption(namespaceOption.getLongOpt()) && cmd.hasOption(boxNameOption.getLongOpt())) {
String namespace = cmd.getOptionValue(namespaceOption.getLongOpt());
String boxName = cmd.getOptionValue(boxNameOption.getLongOpt());
String contextName = cmd.getOptionValue(contextNameOption.getLongOpt());
if (parsedArgs.namespace != null && parsedArgs.boxName != null) {

Map<DictionaryType, String> dictionaries = new HashMap<>();
if (cmd.hasOption(dictionariesOption.getLongOpt())) {
for (String singleDictionary : cmd.getOptionValues(dictionariesOption.getLongOpt())) {
if (parsedArgs.dictionaries != null) {
for (String singleDictionary : parsedArgs.dictionaries) {
String[] keyValue = singleDictionary.split("=");

if (keyValue.length != 2 || StringUtils.isEmpty(keyValue[0].trim()) || StringUtils.isEmpty(keyValue[1].trim())) {
throw new IllegalStateException(String.format("Argument '%s' in '%s' option has wrong format.", singleDictionary, dictionariesOption.getLongOpt()));
throw new IllegalStateException(String.format("Argument '%s' in '%s' option has wrong format.", singleDictionary, "dictionaries"));
}

String typeStr = keyValue[1].trim();
Expand All @@ -367,43 +335,65 @@ public static CommonFactory createFromArguments(String... args) {
}
}

return createFromKubernetes(namespace, boxName, contextName, dictionaries);
return createFromKubernetes(parsedArgs.namespace, parsedArgs.boxName, parsedArgs.contextName, dictionaries);
}

if (configs != null) {
configureLogger(configs);
}
FactorySettings settings = new FactorySettings();
settings.setRabbitMQ(calculatePath(cmd, rabbitConfigurationOption, configs, RABBIT_MQ_FILE_NAME));
settings.setRouterMQ(calculatePath(cmd, messageRouterConfigurationOption, configs, ROUTER_MQ_FILE_NAME));
settings.setConnectionManagerSettings(calculatePath(cmd, connectionManagerConfigurationOption, configs, CONNECTION_MANAGER_CONF_FILE_NAME));
settings.setGrpc(calculatePath(cmd, grpcConfigurationOption, grpcRouterConfigurationOption, configs, GRPC_FILE_NAME));
settings.setRouterGRPC(calculatePath(cmd, grpcRouterConfigOption, configs, ROUTER_GRPC_FILE_NAME));
settings.setPrometheus(calculatePath(cmd, prometheusConfigurationOption, configs, PROMETHEUS_FILE_NAME));
settings.setBoxConfiguration(calculatePath(cmd, boxConfigurationOption, configs, BOX_FILE_NAME));
settings.setCustom(calculatePath(cmd, customConfigurationOption, configs, CUSTOM_FILE_NAME));
settings.setDictionaryTypesDir(calculatePath(cmd, dictionariesDirOption, configs, DICTIONARY_TYPE_DIR_NAME));
settings.setDictionaryAliasesDir(calculatePath(cmd, dictionariesDirOption, configs, DICTIONARY_ALIAS_DIR_NAME));
String oldDictionariesDir = cmd.getOptionValue(dictionariesDirOption.getLongOpt());
settings.setRabbitMQ(calculatePath(parsedArgs.rabbitConfiguration, configs, RABBIT_MQ_FILE_NAME));
settings.setRouterMQ(calculatePath(parsedArgs.messageRouterConfiguration, configs, ROUTER_MQ_FILE_NAME));
settings.setConnectionManagerSettings(calculatePath(parsedArgs.connectionManagerConfiguration, configs, CONNECTION_MANAGER_CONF_FILE_NAME));
settings.setGrpc(calculatePath(parsedArgs.grpcConfiguration, parsedArgs.grpcRouterConfiguration, configs, GRPC_FILE_NAME));
settings.setRouterGRPC(calculatePath(parsedArgs.grpcRouterConfig, configs, ROUTER_GRPC_FILE_NAME));
settings.setPrometheus(calculatePath(parsedArgs.prometheusConfiguration, configs, PROMETHEUS_FILE_NAME));
settings.setBoxConfiguration(calculatePath(parsedArgs.boxConfiguration, configs, BOX_FILE_NAME));
settings.setCustom(calculatePath(parsedArgs.customConfiguration, configs, CUSTOM_FILE_NAME));
settings.setDictionaryTypesDir(calculatePath(parsedArgs.dictionariesDir, configs, DICTIONARY_TYPE_DIR_NAME));
settings.setDictionaryAliasesDir(calculatePath(parsedArgs.dictionariesDir, configs, DICTIONARY_ALIAS_DIR_NAME));
String oldDictionariesDir = parsedArgs.dictionariesDir;
settings.setOldDictionariesDir(oldDictionariesDir == null ? (configs == null ? CONFIG_DEFAULT_PATH : Path.of(configs)) : Path.of(oldDictionariesDir));

String configurationProviderClassName = cmd.getOptionValue(configurationProviderClassOption.getLongOpt(),
DEFAULT_CONFIGURATION_PROVIDER.getName());
String configurationProviderClassName = defaultIfNull(parsedArgs.configurationProviderClass, DEFAULT_CONFIGURATION_PROVIDER.getName());

var providerClass = (Class<? extends ConfigurationProvider>) Class.forName(configurationProviderClassName);
var factory = loadFactoryForProvider(providerClass);
String[] providerArguments = factory.get().parseCommandLine(cmd);
ConfigurationProviderFactory configurationProviderFactory = factory.get();


ConfigurationProviderConfig configurationProviderConfig =
parseAs(args, configurationProviderFactory.settings());

settings.setConfigurationProviderClass(providerClass);
settings.setConfigurationProviderArgs(providerArguments);
settings.setConfigurationProviderConfig(configurationProviderConfig);

return new CommonFactory(settings);
} catch (ParseException e) {
throw new IllegalArgumentException("Incorrect arguments " + Arrays.toString(args), e);
} catch (ClassNotFoundException e) {
LOGGER.error("Failed to find configuration provider class: " + e.getMessage());
throw new IllegalArgumentException("Incorrect arguments " + Arrays.toString(args), e);
}
}

private static ConfigurationProviderConfig parseAs(String[] args, ConfigurationProviderConfig config) {
JCommander.newBuilder()
.addObject(config)
.addObject(new CommonFactoryArgs())
.acceptUnknownOptions(false)
.build()
.parse(args);
return config;
}
private static CommonFactoryArgs parseAsCommonConfig(String[] args) {
CommonFactoryArgs commonFactoryArgs = new CommonFactoryArgs();
JCommander.newBuilder()
.addObject(commonFactoryArgs)
.acceptUnknownOptions(true)
.build()
.parse(args);
return commonFactoryArgs;
}

private static ServiceLoader.@NotNull Provider<ConfigurationProviderFactory> loadFactoryForProvider(
Xanclry marked this conversation as resolved.
Show resolved Hide resolved
Class<? extends ConfigurationProvider> providerClass) {

Expand All @@ -418,22 +408,14 @@ public static CommonFactory createFromArguments(String... args) {
}

@Override
public <M> M loadModule(@NotNull Class<M> clazz) {
public <M extends Module> @NotNull M loadModule(@NotNull Class<M> clazz) {
Xanclry marked this conversation as resolved.
Show resolved Hide resolved
if (configurationProvider == null) {
Xanclry marked this conversation as resolved.
Show resolved Hide resolved
LOGGER.error("Configuration provider hasn't been provided");
throw new IllegalStateException("Configuration provider hasn't been provided");
}
return configurationManager.getModuleWithConfigurationProvider(clazz, configurationProvider);
}

@Override
public <C extends Configuration> @NotNull C loadConfiguration(@NotNull Class<C> clazz) {
if (configurationProvider == null) {
LOGGER.error("Configuration provider hasn't been provided");
throw new IllegalStateException("Configuration provider hasn't been provided");
}
return configurationManager.getConfigurationWithConfigurationProvider(clazz, configurationProvider);
}

/**
* Create {@link CommonFactory} via configs map from Kubernetes
Expand Down Expand Up @@ -775,11 +757,7 @@ private static Path calculatePath(String path, String configsPath, String fileNa
return path != null ? Path.of(path) : (configsPath != null ? Path.of(configsPath, fileName) : CONFIG_DEFAULT_PATH.resolve(fileName));
}

private static Path calculatePath(CommandLine cmd, Option option, String configs, String fileName) {
return calculatePath(cmd.getOptionValue(option.getLongOpt()), configs, fileName);
}

private static Path calculatePath(CommandLine cmd, Option current, Option deprecated, String configs, String fileName) {
return calculatePath(defaultIfNull(cmd.getOptionValue(current.getLongOpt()), cmd.getOptionValue(deprecated.getLongOpt())), configs, fileName);
private static Path calculatePath(String current, String deprecated, String configs, String fileName) {
return calculatePath(defaultIfNull(current, deprecated), configs, fileName);
}
}
71 changes: 71 additions & 0 deletions src/main/kotlin/com/exactpro/th2/common/cli/CommonFactoryArgs.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
/*
* Copyright 2020-2022 Exactpro (Exactpro Systems Limited)
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.exactpro.th2.common.cli

import com.beust.jcommander.Parameter


class CommonFactoryArgs {

@Parameter(names = ["-c", "--configs"])
lateinit var config: String

@Parameter(names = ["--rabbitConfiguration"])
lateinit var rabbitConfiguration: String

@Parameter(names = ["--messageRouterConfiguration"])
lateinit var messageRouterConfiguration: String

@Parameter(names = ["--grpcRouterConfiguration"])
lateinit var grpcRouterConfiguration: String

@Parameter(names = ["--grpcConfiguration"])
lateinit var grpcConfiguration: String

@Parameter(names = ["--grpcRouterConfig"])
lateinit var grpcRouterConfig: String

@Parameter(names = ["--customConfiguration"])
lateinit var customConfiguration: String

@Parameter(names = ["--dictionariesDir"])
lateinit var dictionariesDir: String

@Parameter(names = ["--prometheusConfiguration"])
lateinit var prometheusConfiguration: String

@Parameter(names = ["--boxConfiguration"])
lateinit var boxConfiguration: String
Comment on lines +23 to +51
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove these properties, please use hard coded names for them


@Parameter(names = ["--namespace"])
lateinit var namespace: String

@Parameter(names = ["--boxName"])
lateinit var boxName: String

@Parameter(names = ["--contextName"])
lateinit var contextName: String
Xanclry marked this conversation as resolved.
Show resolved Hide resolved

@Parameter(names = ["--dictionaries"])
lateinit var dictionaries: List<String>

@Parameter(names = ["--connectionManagerConfiguration"])
lateinit var connectionManagerConfiguration: String
Comment on lines +62 to +66
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove this properties, please use hard coded names for them


@Parameter(names = ["--configurationProviderClass"])
lateinit var configurationProviderClass: String

}
Loading