From 19cd19ced42fdc6d2d0290c005a58679aa0f2dd7 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Fri, 14 Jun 2024 12:28:55 +0400 Subject: [PATCH] Added IDictionaryProvider --- .../schema/factory/AbstractCommonFactory.java | 19 +- .../common/schema/factory/CommonFactory.java | 168 +++----------- .../configuration/IDictionaryProvider.kt | 28 +++ .../configuration/impl/DictionaryProvider.kt | 218 ++++++++++++++++++ 4 files changed, 278 insertions(+), 155 deletions(-) create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/configuration/IDictionaryProvider.kt create mode 100644 src/main/kotlin/com/exactpro/th2/common/schema/configuration/impl/DictionaryProvider.kt diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java index 248abbab7..d2063237d 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/AbstractCommonFactory.java @@ -522,7 +522,9 @@ public T getCustomConfiguration(Class confClass) { * * @return Dictionary as {@link InputStream} * @throws IllegalStateException if can not read dictionary or found more than one target + * @deprecated please use {@link #loadDictionary(String)} */ + @Deprecated(since = "6", forRemoval = true) public abstract InputStream loadSingleDictionary(); /** @@ -551,7 +553,8 @@ public T getCustomConfiguration(Class confClass) { * @param dictionaryType desired type of dictionary * @return Dictionary as {@link InputStream} * @throws IllegalStateException if can not read dictionary - * @deprecated Dictionary types will be removed in future releases of infra, use alias instead + * @deprecated Dictionary types will be removed in future releases of infra, use alias instead. + * Please use {@link #loadDictionary(String)} */ @Deprecated(since = "3.33.0", forRemoval = true) public abstract InputStream readDictionary(DictionaryType dictionaryType); @@ -588,20 +591,6 @@ private EventID createRootEventID() throws IOException { protected abstract ConfigurationManager getConfigurationManager(); - /** - * @return Path to dictionaries with type dir - */ - @Deprecated(since = "3.33.0", forRemoval = true) - protected abstract Path getPathToDictionaryTypesDir(); - - /** - * @return Path to dictionaries with alias dir - */ - protected abstract Path getPathToDictionaryAliasesDir(); - - @Deprecated(since = "3.33.0", forRemoval = true) - protected abstract Path getOldPathToDictionariesDir(); - /** * @return Context for all routers except event router */ diff --git a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java index d7250ddc0..e612cb4d3 100644 --- a/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java +++ b/src/main/java/com/exactpro/th2/common/schema/factory/CommonFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -20,6 +20,9 @@ import com.exactpro.th2.common.schema.box.configuration.BoxConfiguration; import com.exactpro.th2.common.schema.configuration.ConfigurationManager; import com.exactpro.th2.common.schema.configuration.IConfigurationProvider; +import com.exactpro.th2.common.schema.configuration.IDictionaryProvider; +import com.exactpro.th2.common.schema.configuration.impl.DictionaryKind; +import com.exactpro.th2.common.schema.configuration.impl.DictionaryProvider; import com.exactpro.th2.common.schema.configuration.impl.JsonConfigurationProvider; import com.exactpro.th2.common.schema.cradle.CradleConfidentialConfiguration; import com.exactpro.th2.common.schema.cradle.CradleNonConfidentialConfiguration; @@ -44,7 +47,6 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; @@ -52,7 +54,6 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -62,15 +63,12 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Base64; +import java.util.EnumMap; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import static com.exactpro.th2.common.schema.util.ArchiveUtils.getGzipBase64StringDecoder; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; @@ -113,52 +111,29 @@ public class CommonFactory extends AbstractCommonFactory { private static final String CRADLE_MANAGER_CONFIG_MAP = "cradle-manager"; private static final String LOGGING_CONFIG_MAP = "logging-config"; - private final Path dictionaryTypesDir; - private final Path dictionaryAliasesDir; - private final Path oldDictionariesDir; final IConfigurationProvider configurationProvider; + final IDictionaryProvider dictionaryProvider; final ConfigurationManager configurationManager; private static final Logger LOGGER = LoggerFactory.getLogger(CommonFactory.class.getName()); - private CommonFactory(IConfigurationProvider configurationProvider) { + private CommonFactory(@NotNull IConfigurationProvider configurationProvider, @NotNull IDictionaryProvider dictionaryProvider) { super(); - dictionaryTypesDir = defaultPathIfNull(configurationProvider, DICTIONARY_TYPE_DIR_NAME); - dictionaryAliasesDir = defaultPathIfNull(settings.getDictionaryAliasesDir(), settings.getBaseConfigDir(), DICTIONARY_ALIAS_DIR_NAME); - oldDictionariesDir = defaultPathIfNull(settings.getOldDictionariesDir(), settings.getBaseConfigDir(), ""); + this.dictionaryProvider = requireNonNull(dictionaryProvider, "Dictionary provider can't be null"); + this.configurationProvider = requireNonNull(configurationProvider, "Configuration provider can't be null"); + configurationManager = createConfigurationManager(configurationProvider); + start(); } @Deprecated(since = "6", forRemoval = true) public CommonFactory(FactorySettings settings) { - super(settings); - dictionaryTypesDir = defaultPathIfNull(settings.getDictionaryTypesDir(), settings.getBaseConfigDir(), DICTIONARY_TYPE_DIR_NAME); - dictionaryAliasesDir = defaultPathIfNull(settings.getDictionaryAliasesDir(), settings.getBaseConfigDir(), DICTIONARY_ALIAS_DIR_NAME); - oldDictionariesDir = defaultPathIfNull(settings.getOldDictionariesDir(), settings.getBaseConfigDir(), ""); - - configurationProvider = createConfigurationProvider(settings); - configurationManager = createConfigurationManager(configurationProvider); - start(); + this(createConfigurationProvider(settings), createDictionaryProvider(settings)); } public CommonFactory() { this(new FactorySettings()); } - @Override - protected Path getPathToDictionaryTypesDir() { - return dictionaryTypesDir; - } - - @Override - protected Path getPathToDictionaryAliasesDir() { - return dictionaryAliasesDir; - } - - @Override - protected Path getOldPathToDictionariesDir() { - return oldDictionariesDir; - } - @Override protected ConfigurationManager getConfigurationManager() { return configurationManager; @@ -453,122 +428,27 @@ public static CommonFactory createFromKubernetes(String namespace, String boxNam @Override public InputStream loadSingleDictionary() { - Path dictionaryFolder = getPathToDictionaryAliasesDir(); - try { - LOGGER.debug("Loading dictionary from folder: {}", dictionaryFolder); - List dictionaries = null; - if (Files.isDirectory(dictionaryFolder)) { - try (Stream files = Files.list(dictionaryFolder)) { - dictionaries = files.filter(Files::isRegularFile).collect(Collectors.toList()); - } - } - - if (dictionaries==null || dictionaries.isEmpty()) { - throw new IllegalStateException("No dictionary at path: " + dictionaryFolder.toAbsolutePath()); - } else if (dictionaries.size() > 1) { - throw new IllegalStateException("Found several dictionaries at path: " + dictionaryFolder.toAbsolutePath()); - } - - var targetDictionary = dictionaries.get(0); - - return new ByteArrayInputStream(getGzipBase64StringDecoder().decode(Files.readString(targetDictionary))); - } catch (IOException e) { - throw new IllegalStateException("Can not read dictionary from path: " + dictionaryFolder.toAbsolutePath(), e); - } + return dictionaryProvider.load(); } @Override public Set getDictionaryAliases() { - Path dictionaryFolder = getPathToDictionaryAliasesDir(); - try { - if (!Files.isDirectory(dictionaryFolder)) { - return Set.of(); - } - - try (Stream files = Files.list(dictionaryFolder)) { - return files - .filter(Files::isRegularFile) - .map(dictionary -> FilenameUtils.removeExtension(dictionary.getFileName().toString())) - .collect(Collectors.toSet()); - } - } catch (IOException e) { - throw new IllegalStateException("Can not get dictionaries aliases from path: " + dictionaryFolder.toAbsolutePath(), e); - } + return dictionaryProvider.aliases(); } @Override public InputStream loadDictionary(String alias) { - Path dictionaryFolder = getPathToDictionaryAliasesDir(); - try { - LOGGER.debug("Loading dictionary by alias ({}) from folder: {}", alias, dictionaryFolder); - List dictionaries = null; - - if (Files.isDirectory(dictionaryFolder)) { - try (Stream files = Files.list(dictionaryFolder)) { - dictionaries = files - .filter(Files::isRegularFile) - .filter(path -> FilenameUtils.removeExtension(path.getFileName().toString()).equalsIgnoreCase(alias)) - .collect(Collectors.toList()); - } - } - - if (dictionaries==null || dictionaries.isEmpty()) { - throw new IllegalStateException("No dictionary was found by alias '" + alias + "' at path: " + dictionaryFolder.toAbsolutePath()); - } else if (dictionaries.size() > 1) { - throw new IllegalStateException("Found several dictionaries by alias '" + alias + "' at path: " + dictionaryFolder.toAbsolutePath()); - } - - return new ByteArrayInputStream(getGzipBase64StringDecoder().decode(Files.readString(dictionaries.get(0)))); - } catch (IOException e) { - throw new IllegalStateException("Can not read dictionary '" + alias + "' from path: " + dictionaryFolder.toAbsolutePath(), e); - } + return dictionaryProvider.load(alias); } @Override public InputStream readDictionary() { - return readDictionary(DictionaryType.MAIN); + return dictionaryProvider.load(DictionaryType.MAIN); } @Override public InputStream readDictionary(DictionaryType dictionaryType) { - try { - List dictionaries = null; - Path typeFolder = dictionaryType.getDictionary(getPathToDictionaryTypesDir()); - if (Files.isDirectory(typeFolder)) { - try (Stream files = Files.list(typeFolder)) { - dictionaries = files.filter(Files::isRegularFile) - .collect(Collectors.toList()); - } - } - - // Find with old format - Path oldFolder = getOldPathToDictionariesDir(); - if ((dictionaries == null || dictionaries.isEmpty()) && Files.isDirectory(oldFolder)) { - try (Stream files = Files.list(oldFolder)) { - dictionaries = files.filter(path -> Files.isRegularFile(path) && path.getFileName().toString().contains(dictionaryType.name())) - .collect(Collectors.toList()); - } - } - - Path dictionaryAliasFolder = getPathToDictionaryAliasesDir(); - if ((dictionaries == null || dictionaries.isEmpty()) && Files.isDirectory(dictionaryAliasFolder)) { - try (Stream files = Files.list(dictionaryAliasFolder)) { - dictionaries = files.filter(Files::isRegularFile).filter(path -> FilenameUtils.removeExtension(path.getFileName().toString()).equalsIgnoreCase(dictionaryType.name())).collect(Collectors.toList()); - } - } - - if (dictionaries == null || dictionaries.isEmpty()) { - throw new IllegalStateException("No dictionary found with type '" + dictionaryType + "'"); - } else if (dictionaries.size() > 1) { - throw new IllegalStateException("Found several dictionaries satisfying the '" + dictionaryType + "' type"); - } - - var targetDictionary = dictionaries.get(0); - - return new ByteArrayInputStream(getGzipBase64StringDecoder().decode(Files.readString(targetDictionary))); - } catch (IOException e) { - throw new IllegalStateException("Can not read dictionary", e); - } + return dictionaryProvider.load(dictionaryType); } static @NotNull Path getConfigPath() { @@ -674,6 +554,14 @@ private static void createDirectory(Path dir) throws IOException { } } + private static IDictionaryProvider createDictionaryProvider(FactorySettings settings) { + Map paths = new EnumMap<>(DictionaryKind.class); + putIfNotNull(paths, DictionaryKind.OLD, settings.getOldDictionariesDir()); + putIfNotNull(paths, DictionaryKind.TYPE, settings.getDictionaryTypesDir()); + putIfNotNull(paths, DictionaryKind.ALIAS, settings.getDictionaryAliasesDir()); + return new DictionaryProvider(defaultIfNull(settings.getBaseConfigDir(), getConfigPath()), paths); + } + private static IConfigurationProvider createConfigurationProvider(FactorySettings settings) { Map paths = new HashMap<>(); putIfNotNull(paths, CUSTOM_CFG_ALIAS, settings.getCustom()); @@ -707,11 +595,11 @@ private static Path defaultPathIfNull(Path customPath, Path basePath, String nam return customPath == null ? getConfigPath(basePath).resolve(name) : customPath; } - private static void putIfNotNull(@NotNull Map paths, @NotNull String alias, @Nullable Path path) { + private static void putIfNotNull(@NotNull Map paths, @NotNull T key, @Nullable Path path) { requireNonNull(paths, "'Paths' can't be null"); - requireNonNull(alias, "'Alias' can't be null"); + requireNonNull(key, "'Key' can't be null"); if (path != null) { - paths.put(alias, path); + paths.put(key, path); } } diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/configuration/IDictionaryProvider.kt b/src/main/kotlin/com/exactpro/th2/common/schema/configuration/IDictionaryProvider.kt new file mode 100644 index 000000000..7ee0c8484 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/configuration/IDictionaryProvider.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.configuration + +import com.exactpro.th2.common.schema.dictionary.DictionaryType +import java.io.InputStream + +interface IDictionaryProvider { + fun aliases(): Set + fun load(alias: String): InputStream + @Deprecated("Load dictionary by type is deprecated, please use load by alias") + fun load(type: DictionaryType): InputStream + @Deprecated("Load single dictionary is deprecated, please use load by alias") + fun load(): InputStream +} \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/common/schema/configuration/impl/DictionaryProvider.kt b/src/main/kotlin/com/exactpro/th2/common/schema/configuration/impl/DictionaryProvider.kt new file mode 100644 index 000000000..3d4ad2eda --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/common/schema/configuration/impl/DictionaryProvider.kt @@ -0,0 +1,218 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.common.schema.configuration.impl + +import com.exactpro.th2.common.schema.configuration.IDictionaryProvider +import com.exactpro.th2.common.schema.configuration.impl.DictionaryKind.ALIAS +import com.exactpro.th2.common.schema.configuration.impl.DictionaryKind.OLD +import com.exactpro.th2.common.schema.configuration.impl.DictionaryKind.TYPE +import com.exactpro.th2.common.schema.dictionary.DictionaryType +import com.exactpro.th2.common.schema.util.ArchiveUtils +import org.apache.commons.io.FilenameUtils +import java.io.ByteArrayInputStream +import java.io.IOException +import java.io.InputStream +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.io.path.isRegularFile +import kotlin.streams.asSequence + +class DictionaryProvider @JvmOverloads constructor( + private val baseDir: Path, + paths: Map = emptyMap() +): IDictionaryProvider { + init { + require(baseDir.exists()) { "Base dir '$baseDir' doesn't exist" } + require(baseDir.isDirectory()) { "Base dir '$baseDir' isn't a dictionary" } + } + + private val directoryPaths = DictionaryKind.createMapping(baseDir) + .plus(paths.mapValues { (_, value) -> value.toAbsolutePath() }) + + private val dictionaryOldPath: Path + get() = requireNotNull(directoryPaths[OLD]) { + "$OLD dictionary kind isn't present in $directoryPaths" + } + private val dictionaryTypePath: Path + get() = requireNotNull(directoryPaths[TYPE]) { + "$TYPE dictionary kind isn't present in $directoryPaths" + } + + private val dictionaryAliasPath: Path + get() = requireNotNull(directoryPaths[ALIAS]) { + "$ALIAS dictionary kind isn't present in $directoryPaths" + } + private val directories = directoryPaths.values + + override fun aliases(): Set { + try { + if(!dictionaryAliasPath.isDirectory()) { + return emptySet() + } + return Files.walk(dictionaryAliasPath).asSequence() + .filter(Path::isRegularFile) + .map(::toAlias) + .toSet() + } catch (e: IOException) { + throw IllegalStateException("Can not get dictionaries aliases from path: ${dictionaryAliasPath.toAbsolutePath()}", e) + } + } + + override fun load(alias: String): InputStream { + try { + require(alias.isNotBlank()) { + "Dictionary is blank" + } + + check(dictionaryAliasPath.isDirectory()) { + "Dictionary dir doesn't exist or isn't directory, path ${dictionaryAliasPath.toAbsolutePath()}" + } + + val files = searchInAliasDir(alias) + val file = single(listOf(dictionaryAliasPath), files, alias) + + return open(file) + } catch (e: IOException) { + throw IllegalStateException("Can not load dictionary by '$alias' alias from path: ${dictionaryAliasPath.toAbsolutePath()}", e) + } + } + + @Deprecated("Load dictionary by type is deprecated, please use load by alias") + override fun load(type: DictionaryType): InputStream { + try { + var files = searchInAliasDir(type.name) + + if (files.isEmpty()) { + files = searchInTypeDir(type) + } + + if (files.isEmpty()) { + files = searchInOldDir(type.name) + } + + val file = single(directories, files, type.name) + return open(file) + } catch (e: IOException) { + throw IllegalStateException("Can not load dictionary by '$type' type from paths: $directories", e) + } + } + + @Deprecated("Load single dictionary is deprecated, please use load by alias") + override fun load(): InputStream { + val dirs = listOf(dictionaryAliasPath, dictionaryTypePath) + try { + var files: List = if (dictionaryAliasPath.isDirectory()) { + emptyList() + } else { + Files.walk(dictionaryAliasPath).asSequence() + .filter(Path::isRegularFile) + .toList() + } + + if (files.isEmpty()) { + if (dictionaryTypePath.isDirectory()) { + files = Files.walk(dictionaryTypePath).asSequence() + .filter(Path::isDirectory) + .flatMap { dir -> + Files.walk(dir).asSequence() + .filter(Path::isRegularFile) + }.toList() + } + } + + check(files.isNotEmpty()) { + "No dictionary at path(s): $dirs" + } + check(files.size == 1) { + "Found several dictionaries at paths: $dirs" + } + val file = files.single() + return open(file) + } catch (e: IOException) { + throw IllegalStateException("Can not read dictionary from from paths: $directories", e) + } + } + + private fun open(file: Path) = ByteArrayInputStream( + ArchiveUtils.getGzipBase64StringDecoder().decode(Files.readString(file)) + ) + + private fun single(dirs: Collection, files: List, alias: String): Path { + check(files.isNotEmpty()) { + "No dictionary was found by '$alias' name at path(s): $dirs" + } + check(files.size == 1) { + "Found several dictionaries by '$alias' name at path(s): $dirs" + } + return files.single() + } + + private fun searchInOldDir(name: String): List { + if (!dictionaryOldPath.isDirectory()) { + return emptyList() + } + return Files.walk(dictionaryOldPath).asSequence() + .filter(Path::isRegularFile) + .filter { file -> file.fileName.toString().contains(name) } + .toList() + } + + private fun searchInTypeDir(type: DictionaryType): List { + val path = type.getDictionary(dictionaryTypePath) + if (!path.isDirectory()) { + return emptyList() + } + return Files.walk(path).asSequence() + .filter(Path::isRegularFile) + .toList() + } + + private fun searchInAliasDir(alias: String): List { + if (!dictionaryAliasPath.isDirectory()) { + return emptyList() + } + return Files.walk(dictionaryAliasPath).asSequence() + .filter(Path::isRegularFile) + .filter { file -> alias.equals(toAlias(file), true) } + .toList() + } + + companion object { + private fun toAlias(path: Path) = FilenameUtils.removeExtension(path.fileName.toString()) + } +} + +enum class DictionaryKind( + val directoryName: String +) { + OLD(""), + TYPE("dictionary"), + ALIAS("dictionaries"); + + companion object { + fun createMapping(baseDir: Path): Map { + require(baseDir.exists()) { "Base dir '$baseDir' doesn't exist" } + require(baseDir.isDirectory()) { "Base dir '$baseDir' isn't a dictionary" } + return buildMap { + DictionaryKind.values().forEach { + put(it, baseDir.resolve(it.directoryName).toAbsolutePath()) + } + } + } + } +} \ No newline at end of file