Skip to content

Commit

Permalink
Add matchesPluginSystemIndexPattern to SystemIndexRegistry (opensearc…
Browse files Browse the repository at this point in the history 
…h-project#14750) (opensearch-project#14770)

* Add matchesPluginSystemIndexPattern to SystemIndexRegistry

* Add to CHANGELOG

* Use single data structure to keep track of system indices

* Address code review comments

* Add test for getAllDescriptors

* Update server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java

---------

(cherry picked from commit 6ad57bf)

Signed-off-by: Craig Perkins <[email protected]>
Signed-off-by: Craig Perkins <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Andriy Redko <[email protected]>
Signed-off-by: kkewwei <[email protected]>
  • Loading branch information
@opensearch-trigger-bot @github-actions
@reta @kkewwei
3 people authored and kkewwei committed Jul 24, 2024
1 parent 7273af1 commit 78a554a
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 35 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
- Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
- Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
- Add matchesPluginSystemIndexPattern to SystemIndexRegistry ([#14750](https://github.com/opensearch-project/OpenSearch/pull/14750))

### Dependencies
- Update to Apache Lucene 9.11.1 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042), [#14576](https://github.com/opensearch-project/OpenSearch/pull/14576))
Expand Down
36 changes: 23 additions & 13 deletions server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@
import org.opensearch.common.regex.Regex;
import org.opensearch.tasks.TaskResultsService;

import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import static java.util.Collections.singletonList;
Expand All @@ -45,25 +45,35 @@ public class SystemIndexRegistry {
);

private volatile static String[] SYSTEM_INDEX_PATTERNS = new String[0];
volatile static Collection<SystemIndexDescriptor> SYSTEM_INDEX_DESCRIPTORS = Collections.emptyList();
private volatile static Map<String, Collection<SystemIndexDescriptor>> SYSTEM_INDEX_DESCRIPTORS_MAP = Collections.emptyMap();

static void register(Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesDescriptors) {
final Map<String, Collection<SystemIndexDescriptor>> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors);
checkForOverlappingPatterns(descriptorsMap);
List<SystemIndexDescriptor> descriptors = pluginAndModulesDescriptors.values()
.stream()
.flatMap(Collection::stream)
.collect(Collectors.toList());
descriptors.add(TASK_INDEX_DESCRIPTOR);

SYSTEM_INDEX_DESCRIPTORS = descriptors.stream().collect(Collectors.toUnmodifiableList());
SYSTEM_INDEX_PATTERNS = descriptors.stream().map(SystemIndexDescriptor::getIndexPattern).toArray(String[]::new);
SYSTEM_INDEX_DESCRIPTORS_MAP = descriptorsMap;
SYSTEM_INDEX_PATTERNS = getAllDescriptors().stream().map(SystemIndexDescriptor::getIndexPattern).toArray(String[]::new);
}

public static List<String> matchesSystemIndexPattern(String... indexExpressions) {
return Arrays.stream(indexExpressions)
.filter(pattern -> Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, pattern))
.collect(Collectors.toList());
public static Set<String> matchesSystemIndexPattern(Set<String> indexExpressions) {
return indexExpressions.stream().filter(pattern -> Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, pattern)).collect(Collectors.toSet());
}

public static Set<String> matchesPluginSystemIndexPattern(String pluginClassName, Set<String> indexExpressions) {
if (!SYSTEM_INDEX_DESCRIPTORS_MAP.containsKey(pluginClassName)) {
return Collections.emptySet();
}
String[] pluginSystemIndexPatterns = SYSTEM_INDEX_DESCRIPTORS_MAP.get(pluginClassName)
.stream()
.map(SystemIndexDescriptor::getIndexPattern)
.toArray(String[]::new);
return indexExpressions.stream()
.filter(pattern -> Regex.simpleMatch(pluginSystemIndexPatterns, pattern))
.collect(Collectors.toSet());
}

static List<SystemIndexDescriptor> getAllDescriptors() {
return SYSTEM_INDEX_DESCRIPTORS_MAP.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
}

/**
Expand Down
5 changes: 3 additions & 2 deletions server/src/main/java/org/opensearch/indices/SystemIndices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public class SystemIndices {

public SystemIndices(Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesDescriptors) {
SystemIndexRegistry.register(pluginAndModulesDescriptors);
this.runAutomaton = buildCharacterRunAutomaton(SystemIndexRegistry.SYSTEM_INDEX_DESCRIPTORS);
this.runAutomaton = buildCharacterRunAutomaton(SystemIndexRegistry.getAllDescriptors());
}

/**
Expand Down Expand Up @@ -91,7 +91,8 @@ public boolean isSystemIndex(String indexName) {
* @throws IllegalStateException if multiple descriptors match the name
*/
public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) {
final List<SystemIndexDescriptor> matchingDescriptors = SystemIndexRegistry.SYSTEM_INDEX_DESCRIPTORS.stream()
final List<SystemIndexDescriptor> matchingDescriptors = SystemIndexRegistry.getAllDescriptors()
.stream()
.filter(descriptor -> descriptor.matchesIndexPattern(name))
.collect(Collectors.toList());

Expand Down
5 changes: 4 additions & 1 deletion server/src/main/java/org/opensearch/node/Node.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -697,7 +697,10 @@ protected Node(
pluginsService.filterPlugins(SystemIndexPlugin.class)
.stream()
.collect(
Collectors.toMap(plugin -> plugin.getClass().getSimpleName(), plugin -> plugin.getSystemIndexDescriptors(settings))
Collectors.toMap(
plugin -> plugin.getClass().getCanonicalName(),
plugin -> plugin.getSystemIndexDescriptors(settings)
)
)
);
final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap);
Expand Down
100 changes: 81 additions & 19 deletions server/src/test/java/org/opensearch/indices/SystemIndicesTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
Expand Down Expand Up @@ -155,32 +157,61 @@ public void testSystemIndexMatching() {
);

assertThat(
SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index2"),
equalTo(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2))
SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index1", ".system-index2")),
equalTo(Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2))
);
assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index1"), equalTo(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1)));
assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index2"), equalTo(List.of(SystemIndexPlugin2.SYSTEM_INDEX_2)));
assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern1"), equalTo(List.of(".system-index-pattern1")));
assertThat(
SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern-sub*"),
equalTo(List.of(".system-index-pattern-sub*"))
SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index1")),
equalTo(Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1))
);
assertThat(
SystemIndexRegistry.matchesSystemIndexPattern(".system-index-pattern1", ".system-index-pattern2"),
equalTo(List.of(".system-index-pattern1", ".system-index-pattern2"))
SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index2")),
equalTo(Set.of(SystemIndexPlugin2.SYSTEM_INDEX_2))
);
assertThat(
SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index-pattern1"),
equalTo(List.of(".system-index1", ".system-index-pattern1"))
SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index-pattern1")),
equalTo(Set.of(".system-index-pattern1"))
);
assertThat(
SystemIndexRegistry.matchesSystemIndexPattern(".system-index1", ".system-index-pattern1", ".not-system"),
equalTo(List.of(".system-index1", ".system-index-pattern1"))
SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index-pattern-sub*")),
equalTo(Set.of(".system-index-pattern-sub*"))
);
assertThat(
SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index-pattern1", ".system-index-pattern2")),
equalTo(Set.of(".system-index-pattern1", ".system-index-pattern2"))
);
assertThat(
SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index1", ".system-index-pattern1")),
equalTo(Set.of(".system-index1", ".system-index-pattern1"))
);
assertThat(
SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".system-index1", ".system-index-pattern1", ".not-system")),
equalTo(Set.of(".system-index1", ".system-index-pattern1"))
);
assertThat(SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".not-system")), equalTo(Collections.emptySet()));
}

public void testRegisteredSystemIndexGetAllDescriptors() {
SystemIndexPlugin plugin1 = new SystemIndexPlugin1();
SystemIndexPlugin plugin2 = new SystemIndexPlugin2();
SystemIndices pluginSystemIndices = new SystemIndices(
Map.of(
SystemIndexPlugin1.class.getCanonicalName(),
plugin1.getSystemIndexDescriptors(Settings.EMPTY),
SystemIndexPlugin2.class.getCanonicalName(),
plugin2.getSystemIndexDescriptors(Settings.EMPTY)
)
);
assertEquals(
SystemIndexRegistry.getAllDescriptors()
.stream()
.map(SystemIndexDescriptor::getIndexPattern)
.collect(Collectors.toUnmodifiableList()),
List.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2, TASK_INDEX + "*")
);
assertThat(SystemIndexRegistry.matchesSystemIndexPattern(".not-system"), equalTo(Collections.emptyList()));
}

public void testRegisteredSystemIndexExpansion() {
public void testRegisteredSystemIndexMatching() {
SystemIndexPlugin plugin1 = new SystemIndexPlugin1();
SystemIndexPlugin plugin2 = new SystemIndexPlugin2();
SystemIndices pluginSystemIndices = new SystemIndices(
Expand All @@ -191,12 +222,43 @@ public void testRegisteredSystemIndexExpansion() {
plugin2.getSystemIndexDescriptors(Settings.EMPTY)
)
);
List<String> systemIndices = SystemIndexRegistry.matchesSystemIndexPattern(
SystemIndexPlugin1.SYSTEM_INDEX_1,
SystemIndexPlugin2.SYSTEM_INDEX_2
Set<String> systemIndices = SystemIndexRegistry.matchesSystemIndexPattern(
Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2)
);
assertEquals(2, systemIndices.size());
assertTrue(systemIndices.containsAll(List.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2)));
assertTrue(systemIndices.containsAll(Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2)));
}

public void testRegisteredSystemIndexMatchingForPlugin() {
SystemIndexPlugin plugin1 = new SystemIndexPlugin1();
SystemIndexPlugin plugin2 = new SystemIndexPlugin2();
SystemIndices pluginSystemIndices = new SystemIndices(
Map.of(
SystemIndexPlugin1.class.getCanonicalName(),
plugin1.getSystemIndexDescriptors(Settings.EMPTY),
SystemIndexPlugin2.class.getCanonicalName(),
plugin2.getSystemIndexDescriptors(Settings.EMPTY)
)
);
Set<String> systemIndicesForPlugin1 = SystemIndexRegistry.matchesPluginSystemIndexPattern(
SystemIndexPlugin1.class.getCanonicalName(),
Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2, "other-index")
);
assertEquals(1, systemIndicesForPlugin1.size());
assertTrue(systemIndicesForPlugin1.contains(SystemIndexPlugin1.SYSTEM_INDEX_1));

Set<String> systemIndicesForPlugin2 = SystemIndexRegistry.matchesPluginSystemIndexPattern(
SystemIndexPlugin2.class.getCanonicalName(),
Set.of(SystemIndexPlugin1.SYSTEM_INDEX_1, SystemIndexPlugin2.SYSTEM_INDEX_2, "other-index")
);
assertEquals(1, systemIndicesForPlugin2.size());
assertTrue(systemIndicesForPlugin2.contains(SystemIndexPlugin2.SYSTEM_INDEX_2));

Set<String> noMatchingSystemIndices = SystemIndexRegistry.matchesPluginSystemIndexPattern(
SystemIndexPlugin2.class.getCanonicalName(),
Set.of("other-index")
);
assertEquals(0, noMatchingSystemIndices.size());
}

static final class SystemIndexPlugin1 extends Plugin implements SystemIndexPlugin {
Expand Down

0 comments on commit 78a554a

Please sign in to comment.