Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Backport 1097, 1302, 1435, 1441, 1445, 1430 #1470

Merged
merged 9 commits into from
Mar 14, 2024
Merged

Backport 1097, 1302, 1435, 1441, 1445, 1430 #1470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
1bc180b
use notiifcation snapshot for integ test (#822)
lezzago Mar 3, 2023
a4ea8ba
Fix build
engechas Mar 13, 2024
b55003f
optimize doc-level monitor workflow for index patterns
sbcd90 Aug 17, 2023
1c7c073
optimize doc-level monitor execution workflow for datastreams (#1302)
sbcd90 Dec 8, 2023
d763474
Add jvm aware setting and max num docs settings for batching docs for…
eirsep Feb 21, 2024
313248a
optimize to fetch only fields relevant to doc level queries in doc le…
eirsep Feb 22, 2024
9824ef0
optimize sequence number calculation and reduce search requests in do…
eirsep Mar 1, 2024
763d97a
clean up doc level queries on dry run (#1430)
jowg-amazon Feb 27, 2024
be34ced
Fix some tests from cherry-picks
engechas Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 45 additions & 143 deletions alerting/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ apply plugin: 'jacoco'

def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
def usingMultiNode = project.properties.containsKey('numNodes')
String bwcVersion = "2.3.0.0"

ext {
projectSubstitutions = [:]
Expand Down Expand Up @@ -90,7 +91,18 @@ configurations.testImplementation {
exclude module: "securemock"
}

configurations {
zipArchive
}

dependencies {
// Needed for integ tests
zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}"
zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}"

// Needed for BWC tests
zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${bwcVersion}-SNAPSHOT"

compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}"
api "org.opensearch.plugin:percolator-client:${opensearch_version}"

Expand All @@ -103,7 +115,6 @@ dependencies {
implementation "com.github.seancfoley:ipaddress:5.3.3"

testImplementation "org.jetbrains.kotlin:kotlin-test:${kotlin_version}"
testImplementation "org.mockito:mockito-core:4.7.0"
testImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
testImplementation "org.opensearch.plugin:parent-join-client:${opensearch_version}"
}
Expand Down Expand Up @@ -136,12 +147,6 @@ integTest.getClusters().forEach{c -> c.plugin(project.getObjects().fileProperty(

def _numNodes = findProperty('numNodes') as Integer ?: 1

String notificationsFilePath = "src/test/resources/notifications"
String notificationsCoreFilePath = "src/test/resources/notifications-core"
String notificationsPlugin = "opensearch-notifications-" + plugin_no_snapshot + ".zip"
String notificationsCorePlugin = "opensearch-notifications-core-" + plugin_no_snapshot + ".zip"
String notificationsRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsPlugin
String notificationsCoreRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsCorePlugin
testClusters.integTest {
testDistribution = "ARCHIVE"
// Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
Expand All @@ -161,17 +166,9 @@ testClusters.integTest {
new RegularFile() {
@Override
File getAsFile() {
File dir = new File(rootDir.path + "/alerting/" + notificationsCoreFilePath)

if (!dir.exists()) {
dir.mkdirs()
}

File f = new File(dir, notificationsCorePlugin)
if (!f.exists()) {
new URL(notificationsCoreRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
}
fileTree(notificationsCoreFilePath).getSingleFile()
return configurations.zipArchive.asFileTree.matching {
include '**/opensearch-notifications-core*'
}.singleFile
}
}
}))
Expand All @@ -180,17 +177,9 @@ testClusters.integTest {
new RegularFile() {
@Override
File getAsFile() {
File dir = new File(rootDir.path + "/alerting/" + notificationsFilePath)

if (!dir.exists()) {
dir.mkdirs()
}

File f = new File(dir, notificationsPlugin)
if (!f.exists()) {
new URL(notificationsRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
}
fileTree(notificationsFilePath).getSingleFile()
return configurations.zipArchive.asFileTree.matching {
include '**/notifications*'
}.singleFile
}
}
}))
Expand Down Expand Up @@ -261,71 +250,26 @@ task integTestRemote(type: RestIntegTestTask) {
}
integTestRemote.enabled = System.getProperty("tests.rest.cluster") != null

String bwcMinVersion = "1.1.0.0"
String bwcBundleVersion = "1.3.2.0"
Boolean bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null &&
project.properties['customDistributionDownloadType'] == "bundle");
String bwcVersion = bwcBundleTest ? bwcBundleVersion : bwcMinVersion
String bwcCurrentVersion = opensearch_version.replace("-SNAPSHOT", "")
String baseName = "alertingBwcCluster"
String bwcFilePath = "src/test/resources/bwc"
String bwcOpenSearchPlugin = "opensearch-alerting-" + bwcVersion + ".zip"
String bwcRemoteFile = 'https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/' + bwcOpenSearchPlugin

2.times {i ->
testClusters {
"${baseName}$i" {
testDistribution = "ARCHIVE"
versions = ["2.3.0-SNAPSHOT", "2.5.1-SNAPSHOT"]
numberOfNodes = 3
if (bwcBundleTest) {
versions = ["1.3.2", bwcCurrentVersion]
nodes.each { node ->
node.extraConfigFile("kirk.pem", file("src/test/resources/kirk.pem"))
node.extraConfigFile("kirk-key.pem", file("src/test/resources/kirk-key.pem"))
node.extraConfigFile("esnode.pem", file("src/test/resources/esnode.pem"))
node.extraConfigFile("esnode-key.pem", file("src/test/resources/esnode-key.pem"))
node.extraConfigFile("root-ca.pem", file("src/test/resources/root-ca.pem"))
node.setting("plugins.security.disabled", "true")
node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
node.setting("plugins.security.ssl.http.enabled", "true")
node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
node.setting("plugins.security.allow_unsafe_democertificates", "true")
node.setting("plugins.security.allow_default_init_securityindex", "true")
node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
node.setting("plugins.security.audit.type", "internal_elasticsearch")
node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
node.setting("plugins.security.system_indices.enabled", "true")
}
} else {
versions = ["1.1.0", opensearch_version]
plugin(provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
File dir = new File(rootDir.path + "/alerting/" + bwcFilePath + "/alerting/" + bwcVersion)

if (!dir.exists()) {
dir.mkdirs()
}
File f = new File(dir, bwcOpenSearchPlugin)
if (!f.exists()) {
new URL(bwcRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
}
return fileTree(bwcFilePath + "/alerting/" + bwcVersion).getSingleFile()
}
plugin(provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return configurations.zipArchive.asFileTree.matching {
include '**/alerting*'
}.singleFile
}
}
}))
}
}
}))
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
setting 'http.content_type.required', 'true'
}
Expand All @@ -344,35 +288,19 @@ task prepareBwcTests {
new RegularFile() {
@Override
File getAsFile() {
File dir = new File(rootDir.path + "/alerting/" + notificationsCoreFilePath)

if (!dir.exists()) {
dir.mkdirs()
}

File f = new File(dir, notificationsCorePlugin)
if (!f.exists()) {
new URL(notificationsCoreRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
}
fileTree(notificationsCoreFilePath).getSingleFile()
return configurations.zipArchive.asFileTree.matching {
include '**/opensearch-notifications-core*'
}.singleFile
}
}
}),
provider({
new RegularFile() {
@Override
File getAsFile() {
File dir = new File(rootDir.path + "/alerting/" + notificationsFilePath)

if (!dir.exists()) {
dir.mkdirs()
}

File f = new File(dir, notificationsPlugin)
if (!f.exists()) {
new URL(notificationsRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
}
fileTree(notificationsFilePath).getSingleFile()
return configurations.zipArchive.asFileTree.matching {
include '**/notifications*'
}.singleFile
}
}
})
Expand All @@ -383,9 +311,7 @@ task prepareBwcTests {
// Create two test clusters with 3 nodes of the old version
2.times {i ->
task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
if (!bwcBundleTest){
dependsOn 'prepareBwcTests'
}
dependsOn 'prepareBwcTests'
useCluster testClusters."${baseName}$i"
filter {
includeTestsMatching "org.opensearch.alerting.bwc.*IT"
Expand All @@ -404,14 +330,8 @@ task prepareBwcTests {
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
useCluster testClusters."${baseName}0"
dependsOn "${baseName}#oldVersionClusterTask0"
if (bwcBundleTest){
doFirst {
testClusters."${baseName}0".nextNodeToNextVersion()
}
} else {
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
filter {
includeTestsMatching "org.opensearch.alerting.bwc.*IT"
Expand All @@ -429,14 +349,8 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#mixedClusterTask"
useCluster testClusters."${baseName}0"
if (bwcBundleTest){
doFirst {
testClusters."${baseName}0".nextNodeToNextVersion()
}
} else {
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
filter {
includeTestsMatching "org.opensearch.alerting.bwc.*IT"
Expand All @@ -454,14 +368,8 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
useCluster testClusters."${baseName}0"
if (bwcBundleTest){
doFirst {
testClusters."${baseName}0".nextNodeToNextVersion()
}
} else {
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
filter {
includeTestsMatching "org.opensearch.alerting.bwc.*IT"
Expand All @@ -479,14 +387,8 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#oldVersionClusterTask1"
useCluster testClusters."${baseName}1"
if (bwcBundleTest){
doFirst {
testClusters."${baseName}1".goToNextVersion()
}
} else {
doFirst {
testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
}
doFirst {
testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
}
filter {
includeTestsMatching "org.opensearch.alerting.bwc.*IT"
Expand Down
7 changes: 7 additions & 0 deletions alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import org.opensearch.alerting.resthandler.RestSearchEmailGroupAction
import org.opensearch.alerting.resthandler.RestSearchMonitorAction
import org.opensearch.alerting.script.TriggerScript
import org.opensearch.alerting.settings.AlertingSettings
import org.opensearch.alerting.settings.AlertingSettings.Companion.DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE
import org.opensearch.alerting.settings.DestinationSettings
import org.opensearch.alerting.settings.LegacyOpenDistroAlertingSettings
import org.opensearch.alerting.settings.LegacyOpenDistroDestinationSettings
Expand Down Expand Up @@ -83,6 +84,7 @@ import org.opensearch.commons.alerting.model.SearchInput
import org.opensearch.env.Environment
import org.opensearch.env.NodeEnvironment
import org.opensearch.index.IndexModule
import org.opensearch.monitor.jvm.JvmStats
import org.opensearch.painless.spi.PainlessExtension
import org.opensearch.painless.spi.Whitelist
import org.opensearch.painless.spi.WhitelistLoader
Expand Down Expand Up @@ -226,6 +228,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
.registerTriggerService(TriggerService(scriptService))
.registerAlertService(AlertService(client, xContentRegistry, alertIndices))
.registerDocLevelMonitorQueries(DocLevelMonitorQueries(client, clusterService))
.registerJvmStats(JvmStats.jvmStats())
.registerConsumers()
.registerDestinationSettings()
scheduledJobIndices = ScheduledJobIndices(client.admin(), clusterService)
Expand Down Expand Up @@ -273,6 +276,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
AlertingSettings.ALERT_HISTORY_MAX_DOCS,
AlertingSettings.ALERT_HISTORY_RETENTION_PERIOD,
AlertingSettings.ALERTING_MAX_MONITORS,
AlertingSettings.PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT,
DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE,
AlertingSettings.PERCOLATE_QUERY_MAX_NUM_DOCS_IN_MEMORY,
AlertingSettings.REQUEST_TIMEOUT,
AlertingSettings.MAX_ACTION_THROTTLE_VALUE,
AlertingSettings.FILTER_BY_BACKEND_ROLES,
Expand All @@ -293,6 +299,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
LegacyOpenDistroAlertingSettings.REQUEST_TIMEOUT,
LegacyOpenDistroAlertingSettings.MAX_ACTION_THROTTLE_VALUE,
LegacyOpenDistroAlertingSettings.FILTER_BY_BACKEND_ROLES,
AlertingSettings.DOC_LEVEL_MONITOR_FETCH_ONLY_QUERY_FIELDS_ENABLED,
DestinationSettings.EMAIL_USERNAME,
DestinationSettings.EMAIL_PASSWORD,
DestinationSettings.ALLOW_LIST,
Expand Down
Loading
Loading