diff --git a/build.gradle b/build.gradle index 5a66d36b6d..64e4b32644 100644 --- a/build.gradle +++ b/build.gradle @@ -82,6 +82,7 @@ configurations.all { force "org.apache.commons:commons-lang3:3.4" force "org.springframework:spring-core:5.3.20" force "com.google.guava:guava:30.0-jre" + force "com.fasterxml.jackson.core:jackson-databind:2.13.2" } } @@ -218,6 +219,15 @@ testsJar { libsDirName = '.' } +task copyResourcesUsedInTests(type: Copy) { + into("${buildDir}/resources/test") + + from("${projectDir}/plugin-security.policy") { + } +} + +test.dependsOn copyResourcesUsedInTests + test { maxParallelForks = 3 diff --git a/src/main/resources/static_config/static_action_groups.yml b/src/main/resources/static_config/static_action_groups.yml index 4d6afd9615..3bdb1684f9 100644 --- a/src/main/resources/static_config/static_action_groups.yml +++ b/src/main/resources/static_config/static_action_groups.yml @@ -227,3 +227,385 @@ manage_data_streams: - "indices:monitor/data_stream/stats" type: "index" description: "Manage data streams" + +#todo copied action groups, remove it: +SGS_KIBANA_ALL_WRITE: + reserved: true + hidden: false + static: true + allowed_actions: + - "kibana:saved_objects/*/write" + type: "kibana" + description: "Allow writing in all kibana apps" +SGS_KIBANA_ALL_READ: + reserved: true + hidden: false + static: true + allowed_actions: + - "kibana:saved_objects/*/read" + type: "kibana" + description: "Allow reading in all kibana apps" +SGS_CLUSTER_ALL: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:*" + type: "cluster" + description: "Allow everything on cluster level" +SGS_CRUD: + reserved: true + hidden: false + static: true + allowed_actions: + - "SGS_READ" + - "SGS_WRITE" + type: "index" + description: "Allow all read/write operations on data" +SGS_SEARCH: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/read/search*" + - "indices:data/read/msearch*" + - "SGS_SUGGEST" + type: "index" + description: "Allow searching" +SGS_DATA_ACCESS: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/*" + - "SGS_CRUD" + type: "index" + description: "Allow all read/write operations on data" +SGS_CREATE_INDEX: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:admin/create" + - "indices:admin/mapping/put" + - "indices:admin/mapping/auto_put" + - "indices:admin/auto_create" + type: "index" + description: "Allow creating new indices" +SGS_WRITE: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/write*" + - "indices:admin/mapping/put" + - "indices:admin/mapping/auto_put" + type: "index" + description: "Allow writing data" +SGS_MANAGE_ALIASES: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:admin/aliases*" + type: "index" + description: "Allow managing index aliases" +SGS_READ: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/read*" + - "indices:admin/mappings/fields/get*" + - "indices:admin/resolve/index" + type: "index" + description: "Allow all read operations" +SGS_INDICES_ALL: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:*" + type: "index" + description: "Allow readonly everything with indices and data" +SGS_DELETE: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/write/delete*" + type: "index" + description: "Allow deleting documents" +SGS_CLUSTER_COMPOSITE_OPS: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/write/bulk" + - "indices:admin/aliases*" + - "indices:data/write/reindex" + - "SGS_CLUSTER_COMPOSITE_OPS_RO" + type: "cluster" + description: "Allow read/write bulk and m* operations" +SGS_CLUSTER_COMPOSITE_OPS_RO: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/read/mget" + - "indices:data/read/msearch" + - "indices:data/read/mtv" + - "indices:data/read/sql" + - "indices:data/read/sql/translate" + - "indices:data/read/sql/close_cursor" + - "indices:admin/aliases/exists*" + - "indices:admin/aliases/get*" + - "indices:data/read/scroll*" + - "indices:data/read/async_search/*" + type: "cluster" + description: "Allow readonly bulk and m* operations" +SGS_GET: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/read/get*" + - "indices:data/read/mget*" + type: "index" + description: "Allow get" +SGS_MANAGE: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:monitor/*" + - "indices:admin/*" + type: "index" + description: "Allow indices management" +SGS_CLUSTER_MONITOR: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:monitor/*" + type: "cluster" + description: "Allow monitoring the cluster" +SGS_MANAGE_SNAPSHOTS: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin/snapshot/*" + - "cluster:admin/repository/*" + type: "cluster" + description: "Allow snapshots" +SGS_INDEX: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/write/index*" + - "indices:data/write/update*" + - "indices:admin/mapping/put" + - "indices:admin/mapping/auto_put" + - "indices:data/write/bulk*" + type: "index" + description: "Allow indexing" +SGS_UNLIMITED: + reserved: true + hidden: false + static: true + allowed_actions: + - "*" + type: "all" + description: "Allow all" +SGS_INDICES_MONITOR: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:monitor/*" + type: "index" + description: "Allow monitoring indices" +SGS_SUGGEST: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/read/suggest*" + type: "index" + description: "Allow suggestions" +SGS_CLUSTER_MANAGE_ILM: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin/ilm/*" + type: "cluster" + description: "Manage index lifecycles (cluster)" +SGS_CLUSTER_READ_ILM: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin/ilm/get" + - "cluster:admin/ilm/operation_mode/get" + type: "cluster" + description: "Read index lifecycles (cluster)" +SGS_INDICES_MANAGE_ILM: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:admin/ilm/*" + type: "index" + description: "Manage index lifecycles (index)" +SGS_CLUSTER_MANAGE_INDEX_TEMPLATES: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:admin/template/*" + - "indices:admin/index_template/*" + - "cluster:admin/component_template/*" + type: "cluster" + description: "Manage index templates" +SGS_CLUSTER_MANAGE_PIPELINES: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin/ingest/pipeline/*" + type: "cluster" + description: "Manage pipelines" +SGS_SEARCH_TEMPLATES: + reserved: true + hidden: false + static: true + allowed_actions: + - "indices:data/read/search/template" + - "indices:data/read/msearch/template" + type: "cluster" + description: "Use search templates" + +### Auth Tokens + +SGS_CREATE_MANAGE_OWN_AUTH_TOKEN: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin:searchguard:authtoken/_own/create" + - "cluster:admin:searchguard:authtoken/_own/get" + - "cluster:admin:searchguard:authtoken/_own/revoke" + - "cluster:admin:searchguard:authtoken/_own/search" + - "cluster:admin:searchguard:authtoken/info" + type: "cluster" + description: "Allows a user to create and manage auth tokens for themselves" + +SGS_MANAGE_ALL_AUTH_TOKEN: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin:searchguard:authtoken/_own/get" + - "cluster:admin:searchguard:authtoken/_own/revoke" + - "cluster:admin:searchguard:authtoken/_own/search" + - "cluster:admin:searchguard:authtoken/_all/get" + - "cluster:admin:searchguard:authtoken/_all/revoke" + - "cluster:admin:searchguard:authtoken/_all/search" + - "cluster:admin:searchguard:authtoken/info" + type: "cluster" + description: "Allows a user to manage auth tokens of all users" + +### Signals + +SGS_SIGNALS_ALL: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin:searchguard:tenant:signals:*" + - "cluster:admin:searchguard:signals:*" + type: "signals" + description: "Grants alls permissions for Signals Alerting" + +### Watches + +SGS_SIGNALS_WATCH_READ: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin:searchguard:tenant:signals:watch/get" + - "cluster:admin:searchguard:tenant:signals:watch/search" + - "cluster:admin:searchguard:tenant:signals:watch:state/get" + - "cluster:admin:searchguard:tenant:signals:watch:state/search" + type: "signals" + description: "Grants permissions to read Signals Alerting watches" + +SGS_SIGNALS_WATCH_MANAGE: + reserved: true + hidden: false + static: true + allowed_actions: + - SGS_SIGNALS_WATCH_READ + - "cluster:admin:searchguard:tenant:signals:watch/put" + - "cluster:admin:searchguard:tenant:signals:watch/delete" + - "cluster:admin:searchguard:tenant:signals:watch/execute" + - "cluster:admin:searchguard:tenant:signals:watch/activate_deactivate" + - "cluster:admin:searchguard:tenant:signals:watch/ack" + type: "signals" + description: "Grants permissions to write Signals Alerting watches" + +SGS_SIGNALS_WATCH_EXECUTE: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin:searchguard:tenant:signals:watch/execute" + type: "signals" + description: "Grants permissions to execute Signals Alerting watches" + +SGS_SIGNALS_WATCH_ACTIVATE: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin:searchguard:tenant:signals:watch/activate_deactivate" + type: "signals" + description: "Grants permissions to activate and deactivate Signals Alerting watches" + +SGS_SIGNALS_WATCH_ACKNOWLEDGE: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin:searchguard:tenant:signals:watch/ack" + type: "signals" + description: "Grants permissions to acknowledge Signals Alerting watches" + + +### Accounts + +SGS_SIGNALS_ACCOUNT_READ: + reserved: true + hidden: false + static: true + allowed_actions: + - "cluster:admin:searchguard:signals:account/get" + - "cluster:admin:searchguard:signals:account/search" + type: "signals" + description: "Grants permissions to read Signals Accounts" + +SGS_SIGNALS_ACCOUNT_MANAGE: + reserved: true + hidden: false + static: true + allowed_actions: + - SGS_SIGNALS_ACCOUNT_READ + - "cluster:admin:searchguard:signals:account/put" + - "cluster:admin:searchguard:signals:account/delete" + type: "signals" + description: "Grants permissions to write Signals Accounts" + diff --git a/src/main/resources/static_config/static_roles.yml b/src/main/resources/static_config/static_roles.yml index 6af0c6ffc9..46a2a97c30 100644 --- a/src/main/resources/static_config/static_roles.yml +++ b/src/main/resources/static_config/static_roles.yml @@ -166,4 +166,241 @@ readall: - "*" allowed_actions: - "read" - + +#todo copied roles, remove it: +SGS_ALL_ACCESS: + reserved: true + hidden: false + static: true + description: "Allow full access to all indices and all cluster APIs" + cluster_permissions: + - "*" + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "*" + tenant_permissions: + - tenant_patterns: + - "*" + allowed_actions: + - "*" + +SGS_KIBANA_USER: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for a kibana user" + cluster_permissions: + - "SGS_CLUSTER_COMPOSITE_OPS" + index_permissions: + - index_patterns: + - ".kibana" + - ".kibana-6" + - '/\.kibana_[0-9]+/' + - ".kibana_task_manager" + - '/\.kibana_task_manager_[0-9]+/' + allowed_actions: + - "SGS_READ" + - "SGS_DELETE" + - "SGS_MANAGE" + - "SGS_INDEX" + - index_patterns: + - ".tasks" + - ".management-beats" + - "*:.tasks" + - "*:.management-beats" + allowed_actions: + - "SGS_INDICES_ALL" + +SGS_OWN_INDEX: + reserved: true + hidden: false + static: true + description: "Allow all for indices named like the current user" + cluster_permissions: + - "SGS_CLUSTER_COMPOSITE_OPS" + index_permissions: + - index_patterns: + - "${user_name}" + allowed_actions: + - "SGS_INDICES_ALL" + +SGS_XP_MONITORING: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for x-pack monitoring" + cluster_permissions: + - "SGS_CLUSTER_MONITOR" + - "cluster:monitor/xpack/info" + - "cluster:monitor/main" + - "cluster:admin/xpack/monitoring/bulk" + index_permissions: + - index_patterns: + - ".monitor*" + - "*:.monitor*" + allowed_actions: + - "SGS_INDICES_ALL" + + +SGS_MANAGE_SNAPSHOTS: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for managing snapshots" + cluster_permissions: + - "SGS_MANAGE_SNAPSHOTS" + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "indices:data/write/index" + - "indices:admin/create" + +SGS_XP_ALERTING: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for x-pack alerting" + cluster_permissions: + - "SGS_CLUSTER_MONITOR" + - "indices:data/read/scroll" + - "cluster:admin/xpack/watcher*" + - "cluster:monitor/xpack/watcher*" + index_permissions: + - index_patterns: + - ".watches*" + - ".watcher-history-*" + - ".triggered_watches" + - "*:.watches*" + - "*:.watcher-history-*" + - "*:.triggered_watches" + allowed_actions: + - "SGS_INDICES_ALL" + - index_patterns: + - "*" + allowed_actions: + - "SGS_READ" + - "indices:admin/aliases/get" + +SGS_XP_MACHINE_LEARNING: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for x-pack machine learning" + cluster_permissions: + - "SGS_CLUSTER_MONITOR" + - "cluster:admin/persistent*" + - "cluster:internal/xpack/ml*" + - "indices:data/read/scroll*" + - "cluster:admin/xpack/ml*" + - "cluster:monitor/xpack/ml*" + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "SGS_READ" + - "indices:admin/get*" + - index_patterns: + - ".ml-*" + - "*:.ml-*" + allowed_actions: + - "*" + +SGS_KIBANA_SERVER: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for the Kibana server" + cluster_permissions: + - "SGS_CLUSTER_MONITOR" + - "SGS_CLUSTER_COMPOSITE_OPS" + - "cluster:admin/xpack/monitoring*" + - "indices:admin/template*" + - "cluster:admin/component_template*" + - "indices:admin/index_template*" + - "indices:data/read/scroll*" + - "SGS_CLUSTER_MANAGE_ILM" + - "cluster:admin:searchguard:authtoken/info" + index_permissions: + - index_patterns: + - ".kibana" + - ".kibana-6" + - ".kibana_*" + - ".reporting*" + - "*:.reporting*" + - ".monitoring*" + - "*:.monitoring*" + - ".tasks" + - ".management-beats*" + - "*:.tasks" + - "*:.management-beats*" + - ".apm-*" + - "*:.apm-*" + - ".kibana-event-log-*" + - "*:.kibana-event-log-*" + allowed_actions: + - "SGS_INDICES_ALL" + - index_patterns: + - "*" + allowed_actions: + - "indices:admin/aliases*" + - "indices:data/read/close_point_in_time" + - "indices:admin/mappings/get" + - "indices:monitor/settings/get" + - "indices:monitor/stats" + - "indices:data/read/field_caps*" + +SGS_LOGSTASH: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for logstash and beats" + cluster_permissions: + - "SGS_CLUSTER_MONITOR" + - "SGS_CLUSTER_COMPOSITE_OPS" + - "SGS_CLUSTER_MANAGE_INDEX_TEMPLATES" + - "SGS_CLUSTER_MANAGE_ILM" + - "SGS_CLUSTER_MANAGE_PIPELINES" + - "cluster:admin/xpack/monitoring*" + index_permissions: + - index_patterns: + - "logstash-*" + - "*beat*" + allowed_actions: + - "SGS_CRUD" + - "SGS_CREATE_INDEX" + - "SGS_MANAGE" + - index_patterns: + - "*" + allowed_actions: + - "indices:admin/aliases/get" + +SGS_READALL_AND_MONITOR: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for to readall indices and monitor the cluster" + cluster_permissions: + - "SGS_CLUSTER_MONITOR" + - "SGS_CLUSTER_COMPOSITE_OPS_RO" + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "SGS_READ" + +SGS_READALL: + reserved: true + hidden: false + static: true + description: "Provide the minimum permissions for to readall indices" + cluster_permissions: + - "SGS_CLUSTER_COMPOSITE_OPS_RO" + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "SGS_READ" + diff --git a/src/test/java/org/opensearch/security/privileges/PrivilegesEvaluatorNewStyleTest.java b/src/test/java/org/opensearch/security/privileges/PrivilegesEvaluatorNewStyleTest.java new file mode 100644 index 0000000000..d20f35ecbe --- /dev/null +++ b/src/test/java/org/opensearch/security/privileges/PrivilegesEvaluatorNewStyleTest.java @@ -0,0 +1,635 @@ +package org.opensearch.security.privileges; + +import com.google.common.collect.ImmutableMap; +import org.hamcrest.MatcherAssert; +import org.junit.*; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest.RefreshPolicy; +import org.opensearch.client.Client; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.ResizeRequest; +import org.opensearch.client.indices.ResizeResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.rest.RestStatus; +import org.opensearch.script.ScriptType; +import org.opensearch.script.mustache.SearchTemplateRequest; +import org.opensearch.script.mustache.SearchTemplateResponse; +import org.opensearch.security.test.helper.cluster.newstyle.JavaSecurityTestSetup; +import org.opensearch.security.test.helper.cluster.newstyle.LocalCluster; +import org.opensearch.security.test.helper.cluster.newstyle.TestSgConfig; +import org.opensearch.security.test.helper.cluster.newstyle.TestSgConfig.Role; +import org.opensearch.security.test.helper.rest.GenericRestClient; +import org.opensearch.security.test.helper.rest.GenericRestClient.HttpResponse; + +import static org.hamcrest.Matchers.*; +import static org.opensearch.security.test.RestMatchers.*; + +public class PrivilegesEvaluatorNewStyleTest { + + private static TestSgConfig.User RESIZE_USER_WITHOUT_CREATE_INDEX_PRIV = new TestSgConfig.User("resize_user_without_create_index_priv") + .roles(new Role("resize_role").clusterPermissions("*").indexPermissions("indices:admin/resize", "indices:monitor/stats") + .on("resize_test_source")); + + private static TestSgConfig.User RESIZE_USER = new TestSgConfig.User("resize_user") + .roles(new Role("resize_role").clusterPermissions("*").indexPermissions("indices:admin/resize", "indices:monitor/stats") + .on("resize_test_source").indexPermissions("SGS_CREATE_INDEX").on("resize_test_target")); + + private static TestSgConfig.User SEARCH_TEMPLATE_USER = new TestSgConfig.User("search_template_user").roles(new Role("search_template_role") + .clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS", "SGS_SEARCH_TEMPLATES").indexPermissions("SGS_READ").on("resolve_test_*")); + + private static TestSgConfig.User SEARCH_NO_TEMPLATE_USER = new TestSgConfig.User("search_no_template_user").roles( + new Role("search_no_template_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS").indexPermissions("SGS_READ").on("resolve_test_*")); + + private static TestSgConfig.User NEG_LOOKAHEAD_USER = new TestSgConfig.User("neg_lookahead_user").roles( + new Role("neg_lookahead_user_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS").indexPermissions("SGS_READ").on("/^(?!t.*).*/")); + + private static TestSgConfig.User REGEX_USER = new TestSgConfig.User("regex_user") + .roles(new Role("neg_lookahead_user_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS").indexPermissions("SGS_READ").on("/^[a-z].*/")); + + private static TestSgConfig.User SEARCH_TEMPLATE_LEGACY_USER = new TestSgConfig.User("search_template_legacy_user") + .roles(new Role("search_template_legacy_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS").indexPermissions("SGS_READ") + .on("resolve_test_*").indexPermissions("indices:data/read/search/template").on("*")); + + private static TestSgConfig.User HIDDEN_TEST_USER = new TestSgConfig.User("hidden_test_user").roles( + new Role("hidden_test_user_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS").indexPermissions("*").on("hidden_test_not_hidden")); + + @ClassRule + public static JavaSecurityTestSetup javaSecurity = new JavaSecurityTestSetup(); + + @ClassRule + public static LocalCluster anotherCluster = new LocalCluster.Builder().singleNode().sslEnabled() + .setInSgConfig("config.dynamic.do_not_fail_on_forbidden", "true") + .user("resolve_test_user", "secret", new Role("resolve_test_user_role").indexPermissions("*").on("resolve_test_allow_*"))// + .build(); + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().singleNode().sslEnabled().remote("my_remote", anotherCluster) + .setInSgConfig("config.dynamic.do_not_fail_on_forbidden", "true") + .user("resolve_test_user", "secret", + new Role("resolve_test_user_role").indexPermissions("*").on("resolve_test_allow_*").indexPermissions("*") + .on("/alias_resolve_test_index_allow_.*/")) // + .user("exclusion_test_user_basic", "secret", + new Role("exclusion_test_user_role").clusterPermissions("*").indexPermissions("*").on("exclude_test_*") + .excludeIndexPermissions("*").on("exclude_test_disallow_*"))// + .user("exclusion_test_user_basic_no_pattern", "secret", + new Role("exclusion_test_user_basic_no_pattern_role").clusterPermissions("*").indexPermissions("*").on("exclude_test_*") + .excludeIndexPermissions("*").on("exclude_test_disallow_2"))// + .user("exclusion_test_user_write", "secret", + new Role("exclusion_test_user_action_exclusion_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS")// + .indexPermissions("*").on("write_exclude_test_*")// + .excludeIndexPermissions("SGS_WRITE").on("write_exclude_test_disallow_*"))// + .user("exclusion_test_user_write_no_pattern", "secret", + new Role("exclusion_test_user_write_no_pattern_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS")// + .indexPermissions("*").on("write_exclude_test_*")// + .excludeIndexPermissions("SGS_WRITE").on("write_exclude_test_disallow_2"))// + .user("exclusion_test_user_cluster_permission", "secret", + new Role("exclusion_test_user_cluster_permission_role").clusterPermissions("*") + .excludeClusterPermissions("indices:data/read/msearch").indexPermissions("*").on("exclude_test_*") + .excludeIndexPermissions("*").on("exclude_test_disallow_*"))// + .user("admin", "admin", new Role("admin_role").clusterPermissions("*"))// + .user("permssion_rest_api_user", "secret", new Role("permssion_rest_api_user_role").clusterPermissions("indices:data/read/mtv"))// + .users(SEARCH_TEMPLATE_USER, SEARCH_NO_TEMPLATE_USER, SEARCH_TEMPLATE_LEGACY_USER).build(); + + @ClassRule + public static LocalCluster clusterFof = new LocalCluster.Builder().singleNode().sslEnabled().remote("my_remote", anotherCluster) + .setInSgConfig("config.dynamic.do_not_fail_on_forbidden", "false") + .user("resolve_test_user", "secret", + new Role("resolve_test_user_role").indexPermissions("*").on("resolve_test_allow_*").indexPermissions("*") + .on("/alias_resolve_test_index_allow_.*/")) // + .user("exclusion_test_user_basic", "secret", + new Role("exclusion_test_user_role").clusterPermissions("*").indexPermissions("*").on("exclude_test_*") + .excludeIndexPermissions("*").on("exclude_test_disallow_*"))// + .user("exclusion_test_user_basic_no_pattern", "secret", + new Role("exclusion_test_user_basic_no_pattern_role").clusterPermissions("*").indexPermissions("*").on("exclude_test_*") + .excludeIndexPermissions("*").on("exclude_test_disallow_2"))// + .user("exclusion_test_user_write", "secret", + new Role("exclusion_test_user_action_exclusion_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS")// + .indexPermissions("*").on("write_exclude_test_*")// + .excludeIndexPermissions("SGS_WRITE").on("write_exclude_test_disallow_*"))// + .user("exclusion_test_user_write_no_pattern", "secret", + new Role("exclusion_test_user_write_no_pattern_role").clusterPermissions("SGS_CLUSTER_COMPOSITE_OPS")// + .indexPermissions("*").on("write_exclude_test_*")// + .excludeIndexPermissions("SGS_WRITE").on("write_exclude_test_disallow_2"))// + .user("exclusion_test_user_cluster_permission", "secret", + new Role("exclusion_test_user_cluster_permission_role").clusterPermissions("*") + .excludeClusterPermissions("indices:data/read/msearch").indexPermissions("*").on("exclude_test_*") + .excludeIndexPermissions("*").on("exclude_test_disallow_*"))// + .users(RESIZE_USER, RESIZE_USER_WITHOUT_CREATE_INDEX_PRIV, NEG_LOOKAHEAD_USER, REGEX_USER, HIDDEN_TEST_USER)// + .build(); + + public static Role[] copiedRoles() { + Role SGS_CREATE_INDEX = new Role("SGS_CREATE_INDEX").indexPermissions("indices:admin/create", "indices:admin/mapping/put", "indices:admin/mapping/auto_put", "indices:admin/auto_create").on("*"); + Role SGS_CLUSTER_COMPOSITE_OPS = new Role("SGS_CLUSTER_COMPOSITE_OPS").clusterPermissions("indices:data/write/bulk", "indices:admin/aliases*", "indices:data/write/reindex", "SGS_CLUSTER_COMPOSITE_OPS_RO"); + Role SGS_CLUSTER_COMPOSITE_OPS_RO = new Role("SGS_CLUSTER_COMPOSITE_OPS_RO").clusterPermissions("indices:data/read/mget", "indices:data/read/msearch", "indices:data/read/mtv" , "indices:data/read/sql", "indices:data/read/sql/translate", "indices:data/read/sql/close_cursor", "indices:admin/aliases/exists*", "indices:admin/aliases/get*", "indices:data/read/scroll*", "indices:data/read/async_search/*"); + Role SGS_SEARCH_TEMPLATES = new Role("SGS_SEARCH_TEMPLATES").clusterPermissions("indices:data/read/search/template", "indices:data/read/msearch/template"); + Role SGS_READ = new Role("SGS_READ").indexPermissions("indices:data/read*", "indices:admin/mappings/fields/get*", "indices:admin/resolve/index").on("*"); + Role SGS_WRITE = new Role("SGS_WRITE").indexPermissions("indices:data/write*", "indices:admin/mapping/put", "indices:admin/mapping/auto_put").on("*"); + return new Role[] { + SGS_CREATE_INDEX, SGS_CLUSTER_COMPOSITE_OPS, SGS_CLUSTER_COMPOSITE_OPS_RO, SGS_SEARCH_TEMPLATES, SGS_READ, SGS_WRITE + }; + } + + @BeforeClass + public static void setupTestData() { + + try (Client client = cluster.getInternalNodeClient()) { + client.index(new IndexRequest("resolve_test_allow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "resolve_test_allow_1", "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_allow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "resolve_test_allow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_disallow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "resolve_test_disallow_1", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_disallow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "resolve_test_disallow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + + client.index(new IndexRequest("alias_resolve_test_index_allow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, + "index", "alias_resolve_test_index_allow_1", "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("alias_resolve_test_index_allow_aliased_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source(XContentType.JSON, "index", "alias_resolve_test_index_allow_aliased_1", "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("alias_resolve_test_index_allow_aliased_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source(XContentType.JSON, "index", "alias_resolve_test_index_allow_aliased_2", "b", "y", "date", "1985/01/01")).actionGet(); + client.admin().indices().aliases( + new IndicesAliasesRequest().addAliasAction(AliasActions.add().alias("alias_resolve_test_alias_1").index("alias_resolve_test_*"))) + .actionGet(); + + client.index(new IndexRequest("exclude_test_allow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "exclude_test_allow_1", "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("exclude_test_allow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "exclude_test_allow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("exclude_test_disallow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "exclude_test_disallow_1", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("exclude_test_disallow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "exclude_test_disallow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + } + + try (Client client = clusterFof.getInternalNodeClient()) { + client.index(new IndexRequest("resolve_test_allow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "resolve_test_allow_1", "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_allow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "resolve_test_allow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_disallow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "resolve_test_disallow_1", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_disallow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "resolve_test_disallow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + + client.admin().indices() + .aliases(new IndicesAliasesRequest() + .addAliasAction(new AliasActions(AliasActions.Type.ADD).alias("resolve_test_allow_alias").indices("resolve_test_*"))) + .actionGet(); + + client.index(new IndexRequest("hidden_test_not_hidden").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "hidden_test_not_hidden", "b", "y", "date", "1985/01/01")).actionGet(); + + client.admin().indices().create(new CreateIndexRequest(".hidden_test_actually_hidden").settings(ImmutableMap.of("index.hidden", true))).actionGet(); + client.index(new IndexRequest(".hidden_test_actually_hidden").id("test").source("a", "b").setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); + + client.index(new IndexRequest("exclude_test_allow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "exclude_test_allow_1", "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("exclude_test_allow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "exclude_test_allow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("exclude_test_disallow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "exclude_test_disallow_1", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("exclude_test_disallow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "exclude_test_disallow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + + client.index(new IndexRequest("tttexclude_test_allow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "tttexclude_test_allow_1", "b", "y", "date", "1985/01/01")).actionGet(); + } + + try (Client client = anotherCluster.getInternalNodeClient()) { + client.index(new IndexRequest("resolve_test_allow_remote_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "a", "x", + "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_allow_remote_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "a", + "xx", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_disallow_remote_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "a", + "xx", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("resolve_test_disallow_remote_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "a", + "xx", "b", "yy", "date", "1985/01/01")).actionGet(); + } + } + + @Test + public void resolveTestLocal() throws Exception { + + try (GenericRestClient restClient = cluster.getRestClient("resolve_test_user", "secret")) { + HttpResponse httpResponse = restClient.get("/_resolve/index/resolve_test_*"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("indices[*].name", contains("resolve_test_allow_1", "resolve_test_allow_2")))); + } + } + + @Test + public void resolveTestRemote() throws Exception { + try (GenericRestClient restClient = cluster.getRestClient("resolve_test_user", "secret")) { + + HttpResponse httpResponse = restClient.get("/_resolve/index/my_remote:resolve_test_*"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, + json(nodeAt("indices[*].name", contains("my_remote:resolve_test_allow_remote_1", "my_remote:resolve_test_allow_remote_2")))); + } + } + + @Test + public void resolveTestLocalRemoteMixed() throws Exception { + try (GenericRestClient restClient = cluster.getRestClient("resolve_test_user", "secret")) { + + HttpResponse httpResponse = restClient.get("/_resolve/index/resolve_test_*,my_remote:resolve_test_*_remote_*"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("indices[*].name", contains("resolve_test_allow_1", "resolve_test_allow_2", + "my_remote:resolve_test_allow_remote_1", "my_remote:resolve_test_allow_remote_2")))); + } + } + + @Test + public void resolveTestAliasAndIndexMixed() throws Exception { + try (GenericRestClient restClient = cluster.getRestClient("resolve_test_user", "secret")) { + + HttpResponse httpResponse = restClient.get("/_resolve/index/alias_resolve_test_*"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("indices[*].name", containsInAnyOrder("alias_resolve_test_index_allow_aliased_1", + "alias_resolve_test_index_allow_aliased_2", "alias_resolve_test_index_allow_1")))); + } + } + + @Test + public void readAliasAndIndexMixed() throws Exception { + try (GenericRestClient restClient = cluster.getRestClient("resolve_test_user", "secret")) { + + HttpResponse httpResponse = restClient.get("/alias_resolve_test_*/_search"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("hits.hits[*]._source.index", containsInAnyOrder("alias_resolve_test_index_allow_aliased_1", + "alias_resolve_test_index_allow_aliased_2", "alias_resolve_test_index_allow_1")))); + } + } + + @Test + @Ignore //todo exclusions are not supported? + public void excludeBasic() throws Exception { + + try (GenericRestClient restClient = cluster.getRestClient("exclusion_test_user_basic", "secret")) { + + HttpResponse httpResponse = restClient.get("/exclude_test_*/_search"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, + json(nodeAt("hits.hits[*]._source.index", containsInAnyOrder("exclude_test_allow_1", "exclude_test_allow_2")))); + } + } + + @Test + @Ignore //todo exclusions are not supported? + public void excludeBasicNoPattern() throws Exception { + + try (GenericRestClient restClient = cluster.getRestClient("exclusion_test_user_basic_no_pattern", "secret")) { + + HttpResponse httpResponse = restClient.get("/exclude_test_*/_search"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("hits.hits[*]._source.index", + containsInAnyOrder("exclude_test_allow_1", "exclude_test_allow_2", "exclude_test_disallow_1")))); + } + } + + @Test + @Ignore //todo exclusions are not supported? + public void excludeWrite() throws Exception { + try (Client client = cluster.getInternalNodeClient()) { + client.index(new IndexRequest("write_exclude_test_allow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "write_exclude_test_allow_1", "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("write_exclude_test_allow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "write_exclude_test_allow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("write_exclude_test_disallow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, + "index", "write_exclude_test_disallow_1", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("write_exclude_test_disallow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, + "index", "write_exclude_test_disallow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + } + try (GenericRestClient restClient = cluster.getRestClient("exclusion_test_user_write", "secret"); + RestHighLevelClient client = cluster.getRestHighLevelClient("exclusion_test_user_write", "secret")) { + + HttpResponse httpResponse = restClient.get("/write_exclude_test_*/_search"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("hits.hits[*]._source.index", containsInAnyOrder("write_exclude_test_allow_1", + "write_exclude_test_allow_2", "write_exclude_test_disallow_1", "write_exclude_test_disallow_2")))); + + IndexResponse indexResponse = client.index(new IndexRequest("write_exclude_test_allow_1").source("a", "b"), RequestOptions.DEFAULT); + + Assert.assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); + + try { + client.index(new IndexRequest("write_exclude_test_disallow_1").source("a", "b"), RequestOptions.DEFAULT); + + Assert.fail(); + } catch (OpenSearchStatusException e) { + Assert.assertEquals(RestStatus.FORBIDDEN, e.status()); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("no permissions for [indices:data/write/index]")); + } + + } + } + + @Test + @Ignore //todo exclusions are not supported? + public void excludeBasicFof() throws Exception { + + try (GenericRestClient restClient = clusterFof.getRestClient("exclusion_test_user_basic", "secret")) { + + HttpResponse httpResponse = restClient.get("/exclude_test_*/_search"); + MatcherAssert.assertThat(httpResponse, isForbidden()); + + httpResponse = restClient.get("/exclude_test_allow_*/_search"); + MatcherAssert.assertThat(httpResponse, isOk()); + + MatcherAssert.assertThat(httpResponse, + json(nodeAt("hits.hits[*]._source.index", containsInAnyOrder("exclude_test_allow_1", "exclude_test_allow_2")))); + + httpResponse = restClient.get("/exclude_test_disallow_1/_search"); + MatcherAssert.assertThat(httpResponse, isForbidden()); + } + } + + @Test + @Ignore //todo exclusions are not supported? + public void excludeBasicFofNoPattern() throws Exception { + + try (GenericRestClient restClient = clusterFof.getRestClient("exclusion_test_user_basic_no_pattern", "secret")) { + + HttpResponse httpResponse = restClient.get("/exclude_test_*/_search"); + MatcherAssert.assertThat(httpResponse, isForbidden()); + + httpResponse = restClient.get("/exclude_test_allow_*/_search"); + MatcherAssert.assertThat(httpResponse, isOk()); + + MatcherAssert.assertThat(httpResponse, + json(nodeAt("hits.hits[*]._source.index", containsInAnyOrder("exclude_test_allow_1", "exclude_test_allow_2")))); + + httpResponse = restClient.get("/exclude_test_disallow_1/_search"); + MatcherAssert.assertThat(httpResponse, isOk()); + + httpResponse = restClient.get("/exclude_test_disallow_2/_search"); + MatcherAssert.assertThat(httpResponse, isForbidden()); + } + } + + @Test + @Ignore //todo exclusions are not supported? + public void excludeWriteFof() throws Exception { + try (Client client = clusterFof.getInternalNodeClient()) { + client.index(new IndexRequest("write_exclude_test_allow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "write_exclude_test_allow_1", "b", "y", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("write_exclude_test_allow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", + "write_exclude_test_allow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("write_exclude_test_disallow_1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, + "index", "write_exclude_test_disallow_1", "b", "yy", "date", "1985/01/01")).actionGet(); + client.index(new IndexRequest("write_exclude_test_disallow_2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, + "index", "write_exclude_test_disallow_2", "b", "yy", "date", "1985/01/01")).actionGet(); + } + + try (GenericRestClient restClient = cluster.getRestClient("exclusion_test_user_write", "secret"); + RestHighLevelClient client = clusterFof.getRestHighLevelClient("exclusion_test_user_write", "secret")) { + + HttpResponse httpResponse = restClient.get("/write_exclude_test_*/_search"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("hits.hits[*]._source.index", containsInAnyOrder("write_exclude_test_allow_1", + "write_exclude_test_allow_2", "write_exclude_test_disallow_1", "write_exclude_test_disallow_2")))); + + IndexResponse indexResponse = client.index(new IndexRequest("write_exclude_test_allow_1").source("a", "b"), RequestOptions.DEFAULT); + + Assert.assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); + + try { + client.index(new IndexRequest("write_exclude_test_disallow_1").source("a", "b"), RequestOptions.DEFAULT); + + Assert.fail(); + } catch (OpenSearchStatusException e) { + Assert.assertEquals(RestStatus.FORBIDDEN, e.status()); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("no permissions for [indices:data/write/index]")); + } + } + } + + @Test + @Ignore //todo exclusions are not supported? + public void excludeClusterPermission() throws Exception { + try (GenericRestClient basicCestClient = cluster.getRestClient("exclusion_test_user_basic", "secret"); + GenericRestClient clusterPermissionCestClient = cluster.getRestClient("exclusion_test_user_cluster_permission", "secret")) { + + HttpResponse httpResponse = basicCestClient.get("/exclude_test_*/_search"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, + json(nodeAt("hits.hits[*]._source.index", containsInAnyOrder("exclude_test_allow_1", "exclude_test_allow_2")))); + + httpResponse = clusterPermissionCestClient.get("/exclude_test_*/_search"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, + json(nodeAt("hits.hits[*]._source.index", containsInAnyOrder("exclude_test_allow_1", "exclude_test_allow_2")))); + + httpResponse = basicCestClient.postJson("/exclude_test_*/_msearch", "{}\n{\"query\": {\"match_all\": {}}}\n"); + MatcherAssert.assertThat(httpResponse, isOk()); + + MatcherAssert.assertThat(httpResponse, + json(nodeAt("responses[0].hits.hits[*]._source.index", containsInAnyOrder("exclude_test_allow_1", "exclude_test_allow_2")))); + + httpResponse = clusterPermissionCestClient.postJson("/exclude_test_*/_msearch", "{}\n{\"query\": {\"match_all\": {}}}\n"); + MatcherAssert.assertThat(httpResponse, isForbidden()); + } + } + + @Test + @Ignore //todo there is no such endpoint? + public void evaluateClusterAndTenantPrivileges() throws Exception { + try (GenericRestClient adminRestClient = cluster.getRestClient("admin", "admin"); + GenericRestClient permissionRestClient = cluster.getRestClient("permssion_rest_api_user", "secret")) { + HttpResponse httpResponse = adminRestClient.get("/_searchguard/permission?permissions=indices:data/read/mtv,indices:data/read/viva"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("permissions['indices:data/read/mtv']", equalTo(true)))); + MatcherAssert.assertThat(httpResponse, json(nodeAt("permissions['indices:data/read/viva']", equalTo(true)))); + + httpResponse = permissionRestClient.get("/_searchguard/permission?permissions=indices:data/read/mtv,indices:data/read/viva"); + + MatcherAssert.assertThat(httpResponse, isOk()); + MatcherAssert.assertThat(httpResponse, json(nodeAt("permissions['indices:data/read/mtv']", equalTo(true)))); + MatcherAssert.assertThat(httpResponse, json(nodeAt("permissions['indices:data/read/viva']", equalTo(false)))); + } + + } + + @Test + public void testResizeAction() throws Exception { + String sourceIndex = "resize_test_source"; + String targetIndex = "resize_test_target"; + + try (Client client = clusterFof.getInternalNodeClient()) { + client.index(new IndexRequest(sourceIndex).setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(XContentType.JSON, "index", "a", "b", "y", + "date", "1985/01/01")).actionGet(); + + client.admin().indices() + .updateSettings(new UpdateSettingsRequest(sourceIndex).settings(Settings.builder().put("index.blocks.write", true).build())) + .actionGet(); + } + + Thread.sleep(300); + + try (RestHighLevelClient client = clusterFof.getRestHighLevelClient(RESIZE_USER_WITHOUT_CREATE_INDEX_PRIV)) { + client.indices().shrink(new ResizeRequest(targetIndex, "whatever"), RequestOptions.DEFAULT); + Assert.fail(); + } catch (OpenSearchStatusException e) { + // Expected + Assert.assertTrue(e.toString(), + //todo changed code +// e.getMessage().contains("no permissions for [indices:/adminresize] and User [name=resize_user_without_create_index_priv")); + e.getMessage().contains("no permissions for [indices:admin/resize] and User [name=resize_user_without_create_index_priv")); + } + + try (RestHighLevelClient client = clusterFof.getRestHighLevelClient(RESIZE_USER_WITHOUT_CREATE_INDEX_PRIV)) { + client.indices().shrink(new ResizeRequest(targetIndex, sourceIndex), RequestOptions.DEFAULT); + Assert.fail(); + } catch (OpenSearchStatusException e) { + // Expected + Assert.assertTrue(e.toString(), + //todo changed code +// e.getMessage().contains("no permissions for [indices:admin/create] and User resize_user_without_create_index_priv")); + e.getMessage().contains("no permissions for [indices:admin/resize] and User [name=resize_user_without_create_index_priv")); + } + + try (RestHighLevelClient client = clusterFof.getRestHighLevelClient(RESIZE_USER)) { + client.indices().shrink(new ResizeRequest(targetIndex, "whatever"), RequestOptions.DEFAULT); + Assert.fail(); + } catch (OpenSearchStatusException e) { + // Expected + //todo changed code +// Assert.assertTrue(e.toString(), e.getMessage().contains("no permissions for [indices:admin/resize] and User resize_user")); + Assert.assertTrue(e.toString(), e.getMessage().contains("no permissions for [indices:admin/resize] and User [name=resize_user")); + } + + //todo it fails + try (RestHighLevelClient client = clusterFof.getRestHighLevelClient(RESIZE_USER)) { + ResizeResponse resizeResponse = client.indices().shrink(new ResizeRequest(targetIndex, sourceIndex), RequestOptions.DEFAULT); + Assert.assertTrue(resizeResponse.toString(), resizeResponse.isAcknowledged()); + } + + try (Client client = clusterFof.getInternalNodeClient()) { + IndicesExistsResponse response = client.admin().indices().exists(new IndicesExistsRequest(targetIndex)).actionGet(); + Assert.assertTrue(response.toString(), response.isExists()); + } + } + + @Test + //todo it fails + public void searchTemplate() throws Exception { + + SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(new SearchRequest("resolve_test_allow_*")); + searchTemplateRequest.setScriptType(ScriptType.INLINE); + searchTemplateRequest.setScript("{\"query\": {\"term\": {\"b\": \"{{x}}\" } } }"); + searchTemplateRequest.setScriptParams(ImmutableMap.of("x", "yy")); + + try (RestHighLevelClient client = cluster.getRestHighLevelClient(SEARCH_TEMPLATE_USER)) { + SearchTemplateResponse searchTemplateResponse = client.searchTemplate(searchTemplateRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = searchTemplateResponse.getResponse(); + + Assert.assertEquals(searchResponse.toString(), 1, searchResponse.getHits().getTotalHits().value); + } + + try (RestHighLevelClient client = cluster.getRestHighLevelClient(SEARCH_NO_TEMPLATE_USER)) { + SearchTemplateResponse searchTemplateResponse = client.searchTemplate(searchTemplateRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = searchTemplateResponse.getResponse(); + + Assert.fail(searchResponse.toString()); + } catch (OpenSearchStatusException e) { + Assert.assertEquals(e.toString(), RestStatus.FORBIDDEN, e.status()); + } + } + + @Test + public void searchTemplateLegacy() throws Exception { + + SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(new SearchRequest("resolve_test_allow_*")); + searchTemplateRequest.setScriptType(ScriptType.INLINE); + searchTemplateRequest.setScript("{\"query\": {\"term\": {\"b\": \"{{x}}\" } } }"); + searchTemplateRequest.setScriptParams(ImmutableMap.of("x", "yy")); + + try (RestHighLevelClient client = cluster.getRestHighLevelClient(SEARCH_TEMPLATE_LEGACY_USER)) { + SearchTemplateResponse searchTemplateResponse = client.searchTemplate(searchTemplateRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = searchTemplateResponse.getResponse(); + + Assert.assertEquals(searchResponse.toString(), 1, searchResponse.getHits().getTotalHits().value); + } + + try (RestHighLevelClient client = cluster.getRestHighLevelClient(SEARCH_NO_TEMPLATE_USER)) { + SearchTemplateResponse searchTemplateResponse = client.searchTemplate(searchTemplateRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = searchTemplateResponse.getResponse(); + + Assert.fail(searchResponse.toString()); + } catch (OpenSearchStatusException e) { + Assert.assertEquals(e.toString(), RestStatus.FORBIDDEN, e.status()); + } + } + + @Test + public void negativeLookaheadPattern() throws Exception { + + try (GenericRestClient restClient = clusterFof.getRestClient(NEG_LOOKAHEAD_USER)) { + + HttpResponse httpResponse = restClient.get("*/_search"); + + Assert.assertEquals(httpResponse.getBody(), 403, httpResponse.getStatusCode()); + + httpResponse = restClient.get("r*/_search"); + + Assert.assertEquals(httpResponse.getBody(), 200, httpResponse.getStatusCode()); + } + } + + @Test + public void regexPattern() throws Exception { + + try (GenericRestClient restClient = clusterFof.getRestClient(REGEX_USER)) { + + HttpResponse httpResponse = restClient.get("*/_search"); + + Assert.assertEquals(httpResponse.getBody(), 403, httpResponse.getStatusCode()); + + httpResponse = restClient.get("r*/_search"); + + Assert.assertEquals(httpResponse.getBody(), 200, httpResponse.getStatusCode()); + } + } + + @Test + //todo it fails + public void resolveTestHidden() throws Exception { + + try (GenericRestClient restClient = clusterFof.getRestClient(HIDDEN_TEST_USER)) { + HttpResponse httpResponse = restClient.get("/*hidden_test*/_search?expand_wildcards=all&pretty=true"); + Assert.assertEquals(httpResponse.getBody(), 403, httpResponse.getStatusCode()); + + httpResponse = restClient.get("/*hidden_test*/_search?pretty=true"); + Assert.assertEquals(httpResponse.getBody(), 200, httpResponse.getStatusCode()); + Assert.assertFalse(httpResponse.getBody(), httpResponse.getBody().contains("hidden_test_actually_hidden")); + } + + } +} diff --git a/src/test/java/org/opensearch/security/test/RestMatchers.java b/src/test/java/org/opensearch/security/test/RestMatchers.java new file mode 100644 index 0000000000..f528b5ac49 --- /dev/null +++ b/src/test/java/org/opensearch/security/test/RestMatchers.java @@ -0,0 +1,167 @@ +package org.opensearch.security.test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.DiagnosingMatcher; +import org.hamcrest.Matcher; +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.test.helper.rest.GenericRestClient.HttpResponse; + +import java.io.IOException; + +public class RestMatchers { + public static DiagnosingMatcher isOk() { + return new DiagnosingMatcher() { + + @Override + public void describeTo(Description description) { + description.appendText("Response has status 200 OK"); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (!(item instanceof HttpResponse)) { + mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); + return false; + } + + HttpResponse response = (HttpResponse) item; + + if (response.getStatusCode() == 200) { + return true; + } else { + mismatchDescription.appendText("Status is not 200 OK: ").appendValue(item); + return false; + } + + } + + }; + } + + public static DiagnosingMatcher isForbidden() { + return new DiagnosingMatcher() { + + @Override + public void describeTo(Description description) { + description.appendText("Response has status 403 Forbidden"); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (!(item instanceof HttpResponse)) { + mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); + return false; + } + + HttpResponse response = (HttpResponse) item; + + if (response.getStatusCode() == 403) { + return true; + } else { + mismatchDescription.appendText("Status is not 403 Forbidden: ").appendValue(item); + return false; + } + + } + + }; + } + + public static DiagnosingMatcher json(BaseMatcher... subMatchers) { + return new DiagnosingMatcher() { + + @Override + public void describeTo(Description description) { + description.appendText("Content type of response body is application/json"); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (!(item instanceof HttpResponse)) { + mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); + return false; + } + + HttpResponse response = (HttpResponse) item; + + String contentType = response.getContentType() != null ? response.getContentType().toLowerCase() : ""; + + if (!(contentType.startsWith("application/json"))) { + mismatchDescription.appendText("Response does not have the content type application/json: ") + .appendValue(response.getContentType() + "; " + response.getHeaders()); + return false; + } + + try { + JsonNode jsonNode = DefaultObjectMapper.objectMapper.readTree(response.getBody()); + boolean ok = true; + + for (BaseMatcher subMatcher : subMatchers) { + if (subMatcher.matches(jsonNode)) { + } else { + subMatcher.describeMismatch(jsonNode, mismatchDescription); + ok = false; + } + } + + return ok; + + } catch (IOException e) { + mismatchDescription.appendText("Response cannot be parsed as JSON: " + e.toString()).appendValue(response.getBody()); + return false; + } + } + + }; + } + + public static DiagnosingMatcher nodeAt(String jsonPath, Matcher subMatcher) { + return new DiagnosingMatcher() { + + @Override + public void describeTo(Description description) { + description.appendText("JSON element at ").appendValue(jsonPath).appendText(" matches ").appendDescriptionOf(subMatcher); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (!(item instanceof JsonNode)) { + mismatchDescription.appendValue(item).appendText(" is not a JsonNode"); + return false; + } + + Configuration config = Configuration.builder().options(Option.SUPPRESS_EXCEPTIONS).jsonProvider(new JacksonJsonNodeJsonProvider()) + .mappingProvider(new JacksonMappingProvider()).build(); + + Object value = JsonPath.using(config).parse(item).read(jsonPath); + + if (value == null) { + mismatchDescription.appendText("No value at " + jsonPath + " ").appendValue(item); + return false; + } + + if (value instanceof JsonNode) { + value = new ObjectMapper().convertValue(value, Object.class); + } + + if (subMatcher.matches(value)) { + return true; + } else { + mismatchDescription.appendText("at " + jsonPath + ": ").appendValue(value).appendText("\n"); + subMatcher.describeMismatch(value, mismatchDescription); + return false; + } + } + + }; + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/ClusterConfiguration.java b/src/test/java/org/opensearch/security/test/helper/cluster/ClusterConfiguration.java index bc3ec819ff..a3fa63fd86 100644 --- a/src/test/java/org/opensearch/security/test/helper/cluster/ClusterConfiguration.java +++ b/src/test/java/org/opensearch/security/test/helper/cluster/ClusterConfiguration.java @@ -30,6 +30,7 @@ package org.opensearch.security.test.helper.cluster; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; @@ -134,5 +135,16 @@ public NodeSettings removePluginIfPresent(Class pluginToRemove public Class[] getPlugins() { return plugins.toArray(new Class[0] ); } + + @SuppressWarnings("unchecked") + public Class[] getPlugins(List> additionalPlugins) { + List> plugins = new ArrayList<>(this.plugins); + + if (additionalPlugins != null) { + plugins.addAll(additionalPlugins); + } + + return plugins.toArray(new Class[0]); + } } } diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/EsClientProvider.java b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/EsClientProvider.java new file mode 100644 index 0000000000..cf5002b3ec --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/EsClientProvider.java @@ -0,0 +1,160 @@ +/* + * Copyright 2020 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.opensearch.security.test.helper.cluster.newstyle; + +import org.apache.http.Header; +import org.apache.http.HttpException; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.message.BasicHeader; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.protocol.HttpContext; +import org.opensearch.client.Client; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.RestClientBuilder.HttpClientConfigCallback; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.security.test.helper.rest.GenericRestClient; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +//todo rename class +public interface EsClientProvider { + default GenericRestClient getRestClient(TestSgConfig.User user) { + return getRestClient(user.getName(), user.getPassword()); + } + + default GenericRestClient getRestClient(TestSgConfig.User user, Header... headers) { + return getRestClient(user.getName(), user.getPassword(), headers); + } + + default public GenericRestClient getRestClient(String user, String password, String tenant) { + BasicHeader basicAuthHeader = new BasicHeader("Authorization", + "Basic " + Base64.getEncoder().encodeToString((user + ":" + Objects.requireNonNull(password)).getBytes(StandardCharsets.UTF_8))); + + return new GenericRestClient(getHttpAddress(), Arrays.asList(basicAuthHeader, new BasicHeader("sgtenant", tenant)), getResourceFolder()); + } + + default public GenericRestClient getRestClient(String user, String password) { + BasicHeader basicAuthHeader = new BasicHeader("Authorization", + "Basic " + Base64.getEncoder().encodeToString((user + ":" + Objects.requireNonNull(password)).getBytes(StandardCharsets.UTF_8))); + + return new GenericRestClient(getHttpAddress(), Collections.singletonList(basicAuthHeader), getResourceFolder()); + } + + default public GenericRestClient getRestClient(String user, String password, Header... headers) { + BasicHeader basicAuthHeader = new BasicHeader("Authorization", + "Basic " + Base64.getEncoder().encodeToString((user + ":" + Objects.requireNonNull(password)).getBytes(StandardCharsets.UTF_8))); + + List
headersList = new ArrayList<>(); + headersList.add(basicAuthHeader); + if (headers != null && headers.length > 0) { + headersList.addAll(Arrays.asList(headers)); + } + + return new GenericRestClient(getHttpAddress(), headersList, getResourceFolder()); + } + + default public GenericRestClient getRestClient(Header... headers) { + return new GenericRestClient(getHttpAddress(), Arrays.asList(headers), getResourceFolder()); + } + + default public GenericRestClient getAdminCertRestClient() { + GenericRestClient result = new GenericRestClient(getHttpAddress(), Collections.emptyList(), getResourceFolder()); + + result.setKeystore("kirk-keystore.jks"); + result.setSendHTTPClientCertificate(true); + + return result; + } + + default RestHighLevelClient getRestHighLevelClient(TestSgConfig.User user) { + return getRestHighLevelClient(user.getName(), user.getPassword()); + } + + default RestHighLevelClient getRestHighLevelClient(String user, String password) { + return getRestHighLevelClient(user, password, null); + } + + default RestHighLevelClient getRestHighLevelClient(String user, String password, String tenant) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password)); + + HttpClientConfigCallback configCallback = new HttpClientConfigCallback() { + + @Override + public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + httpClientBuilder = httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setSSLStrategy(getSSLIOSessionStrategy()); + + if (tenant != null) { + httpClientBuilder = httpClientBuilder.addInterceptorLast(new HttpRequestInterceptor() { + + @Override + public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { + request.setHeader("sgtenant", tenant); + + } + + }); + } + + return httpClientBuilder; + } + }; + + RestClientBuilder builder = RestClient.builder(new HttpHost(getHttpAddress().getHostString(), getHttpAddress().getPort(), "https")) + .setHttpClientConfigCallback(configCallback); + + return new RestHighLevelClient(builder); + } + + default RestHighLevelClient getRestHighLevelClient(Header... headers) { + RestClientBuilder builder = RestClient.builder(new HttpHost(getHttpAddress().getHostString(), getHttpAddress().getPort(), "https")) + .setDefaultHeaders(headers) + .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setSSLStrategy(getSSLIOSessionStrategy())); + + return new RestHighLevelClient(builder); + } + + Client getInternalNodeClient(); + + InetSocketAddress getHttpAddress(); + + InetSocketAddress getTransportAddress(); + + String getClusterName(); + + // XXX fixme + String getResourceFolder(); + + SSLIOSessionStrategy getSSLIOSessionStrategy(); +} diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/EsJavaSecurity.java b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/EsJavaSecurity.java new file mode 100644 index 0000000000..85c8e916d7 --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/EsJavaSecurity.java @@ -0,0 +1,206 @@ +/* + * Includes code from the following Apache 2 licensed files from Elasticsearch 7.10.2: + * + * - /server/src/main/java/org/elasticsearch/bootstrap/Security.java: + * https://github.com/elastic/elasticsearch/blob/7.10/server/src/main/java/org/elasticsearch/bootstrap/Security.java + * - /server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java: + * https://github.com/elastic/elasticsearch/blob/7.10/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java + * - /test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java: + * https://github.com/elastic/elasticsearch/blob/7.10/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java + * + * Original license notice for all of the noted files: + * + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * + */ + +package org.opensearch.security.test.helper.cluster.newstyle; + +import org.opensearch.bootstrap.BootstrapInfo; +import org.opensearch.bootstrap.JarHell; +import org.opensearch.cli.SuppressForbidden; +import org.opensearch.common.io.PathUtils; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; +import java.security.Permissions; +import java.security.Policy; +import java.security.URIParameter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.opensearch.bootstrap.FilePermissionUtils.addDirectoryPath; +import static org.opensearch.bootstrap.FilePermissionUtils.addSingleFilePath; + +//todo rename class +public class EsJavaSecurity { + static Policy getBaseEsSecurityPolicy() { + return readPolicy(BootstrapInfo.class.getResource("security.policy"), getCodebases()); + } + + static Policy getSgPluginSecurityPolicy() { + return readPolicy(EsJavaSecurity.class.getResource("/plugin-security.policy"), getCodebases()); + } + + /** + * Return a map from codebase name to codebase url of jar codebases used by ES core. + */ + @SuppressForbidden(reason = "find URL path") + static Map getCodebaseJarMap(Set urls) { + Map codebases = new LinkedHashMap<>(); // maintain order + for (URL url : urls) { + try { + String fileName = PathUtils.get(url.toURI()).getFileName().toString(); + if (fileName.endsWith(".jar") == false) { + // tests :( + continue; + } + codebases.put(fileName, url); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + return codebases; + } + + /** + * Reads and returns the specified {@code policyFile}. + *

+ * Jar files listed in {@code codebases} location will be provided to the policy file via + * a system property of the short name: e.g. ${codebase.joda-convert-1.2.jar} + * would map to full URL. + */ + @SuppressForbidden(reason = "accesses fully qualified URLs to configure security") + static Policy readPolicy(URL policyFile, Map codebases) { + try { + List propertiesSet = new ArrayList<>(); + try { + // set codebase properties + for (Map.Entry codebase : codebases.entrySet()) { + String name = codebase.getKey(); + URL url = codebase.getValue(); + + // We attempt to use a versionless identifier for each codebase. This assumes a specific version + // format in the jar filename. While we cannot ensure all jars in all plugins use this format, nonconformity + // only means policy grants would need to include the entire jar filename as they always have before. + String property = "codebase." + name; + String aliasProperty = "codebase." + name.replaceFirst("-\\d+\\.\\d+.*\\.jar", ""); + if (aliasProperty.equals(property) == false) { + propertiesSet.add(aliasProperty); + System.setProperty(aliasProperty, url.toString()); + } + propertiesSet.add(property); + System.setProperty(property, url.toString()); + } + return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI())); + } finally { + // clear codebase properties + for (String property : propertiesSet) { + System.clearProperty(property); + } + } + } catch (NoSuchAlgorithmException | URISyntaxException e) { + throw new IllegalArgumentException("unable to parse policy file `" + policyFile + "`", e); + } + } + + /** Adds access to classpath jars/classes for jar hell scan, etc */ + @SuppressForbidden(reason = "accesses fully qualified URLs to configure security") + static Permissions getClasspathPermissions() { + try { + Permissions perms = new Permissions(); + + // add permissions to everything in classpath + // really it should be covered by lib/, but there could be e.g. agents or similar configured) + for (URL url : JarHell.parseClassPath()) { + Path path; + try { + path = PathUtils.get(url.toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + // resource itself + if (Files.isDirectory(path)) { + addDirectoryPath(perms, "class.path", path, "read,readlink", false); + } else { + addSingleFilePath(perms, path, "read,readlink"); + } + } + + return perms; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static Permissions getMiscPermissions() { + + try { + Permissions perms = new Permissions(); + Path javaTmpDir = PathUtils.get(Objects.requireNonNull(System.getProperty("java.io.tmpdir"), "please set ${java.io.tmpdir} in pom.xml")); + addDirectoryPath(perms, "java.io.tmpdir", javaTmpDir, "read,readlink,write,delete", false); + perms.add(new RuntimePermission("getStackWalkerWithClassReference")); + return perms; + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + static Map getCodebases() { + // read test-framework permissions + Map codebases = getCodebaseJarMap(JarHell.parseClassPath()); + // when testing server, the main elasticsearch code is not yet in a jar, so we need to manually add it + addClassCodebase(codebases, "elasticsearch", "org.elasticsearch.plugins.PluginsService"); + if (System.getProperty("tests.gradle") == null) { + // intellij and eclipse don't package our internal libs, so we need to set the codebases for them manually + addClassCodebase(codebases, "plugin-classloader", "org.elasticsearch.plugins.ExtendedPluginsClassLoader"); + addClassCodebase(codebases, "elasticsearch-nio", "org.elasticsearch.nio.ChannelFactory"); + addClassCodebase(codebases, "elasticsearch-secure-sm", "org.elasticsearch.secure_sm.SecureSM"); + addClassCodebase(codebases, "elasticsearch-rest-client", "org.elasticsearch.client.RestClient"); + } + + return codebases; + } + + /** Add the codebase url of the given classname to the codebases map, if the class exists. */ + private static void addClassCodebase(Map codebases, String name, String classname) { + try { + Class clazz = EsJavaSecurity.class.getClassLoader().loadClass(classname); + URL location = clazz.getProtectionDomain().getCodeSource().getLocation(); + if (location.toString().endsWith(".jar") == false) { + if (codebases.put(name, location) != null) { + throw new IllegalStateException("Already added " + name + " codebase for testing"); + } + } + } catch (ClassNotFoundException e) { + // no class, fall through to not add. this can happen for any tests that do not include + // the given class. eg only core tests include plugin-classloader + } + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/JavaSecurityTestSetup.java b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/JavaSecurityTestSetup.java new file mode 100644 index 0000000000..99c61a3e11 --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/JavaSecurityTestSetup.java @@ -0,0 +1,230 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.opensearch.security.test.helper.cluster.newstyle; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.rules.ExternalResource; +import org.opensearch.bootstrap.BootstrapInfo; +import org.opensearch.monitor.jvm.JvmInfo; +import org.opensearch.secure_sm.SecureSM; + +import java.io.FilePermission; +import java.net.SocketPermission; +import java.security.*; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Provides a simplified Java Security Manager environment for JUnit tests. + * + * This is far from perfect, but good enough to notice most missing doPriviledged() blocks, etc. + * + * In order to run tests without this, set -Dsg.test-java-security.enabled=false + * + * Helpful VM args for privilege debugging: + * + * - -Djava.security.debug=access,failure,domain + * - -Djava.security.debug=access,domain,permission=java.net.RuntimePermission + * + * Note: When using java.security.debug=access,failure, don't be confused by file access control failures caused by Lucene. + * During startup, Lucene seems to do a very liberal scan of your whole root dir and just ignores dirs it cannot access. + * + * Note: This does not work with multi threaded unit tests. If you want to have concurrency, use forking instead. + * + * TODO: + * + * - This needs a copy of plugin-security.policy in test/resources. We should find a way to avoid this redunancy. + * - Lots of more polishing. In parts, this is quite messy. + */ +public class JavaSecurityTestSetup extends ExternalResource { + private static final Logger log = LogManager.getLogger(JavaSecurityTestSetup.class); + private static ReentrantLock lock = new ReentrantLock(); + private static Policy baseSystemPolicy = Policy.getPolicy(); + private static Policy baseEsPolicy = EsJavaSecurity.getBaseEsSecurityPolicy(); + private static Policy sgPluginPolicy = EsJavaSecurity.getSgPluginSecurityPolicy(); + private static Permissions classPathPermissions = EsJavaSecurity.getClasspathPermissions(); + private static Permissions miscPermissions = EsJavaSecurity.getMiscPermissions(); + private boolean enabled = System.getProperty("sg.test-java-security.enabled", "true").equals("true"); + + static { + try { + JvmInfo.jvmInfo(); + } catch (AccessControlException e) { + // If we get this, we are already properly initialized + } + try { + BootstrapInfo.init(); + } catch (AccessControlException e) { + // If we get this, we are already properly initialized + } + try { + BootstrapInfo.isNativesAvailable(); + } catch (AccessControlException e) { + // If we get this, we are already properly initialized + } + } + + public JavaSecurityTestSetup() { + if (enabled) { + if (lock.isLocked()) { + try { + if (!lock.tryLock(10, TimeUnit.SECONDS)) { + log.warn("***** Multithreaded use of TestJavaSecurityManagement is not possible. Waiting for current owner to finish: " + + lock); + lock.lock(); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + Policy.setPolicy(new TestPolicy()); + //todo is it ok? +// System.setSecurityManager(SecureSM.createTestSecureSM()); + System.setSecurityManager(SecureSM.createTestSecureSM(Set.of())); + log.info("JavaSecurityTestSetup has been installed"); + } + } + + @Override + protected void after() { + try { + if (enabled) { + System.setSecurityManager(null); + Policy.setPolicy(baseSystemPolicy); + } + } finally { + if (lock.isHeldByCurrentThread()) { + lock.unlock(); + } + } + } + + static class TestPolicy extends Policy { + + private static final Pattern JAR_PATTERN = Pattern.compile(".*/(.*?)(-[0-9]+\\.[0-9]+(\\.[0-9]+)?(\\.[^\\.]+)?)?\\.jar$"); + + @Override + public boolean implies(ProtectionDomain protectionDomain, Permission permission) { + if (permission instanceof FilePermission) { + FilePermission filePermission = ((FilePermission) permission); + + if (filePermission.getName().contains("data")) { + // Special case for ES data access + return true; + } + + if (filePermission.getName().contains("/target/test-classes/") || filePermission.getName().contains("/config") + || filePermission.getName().contains("/sgconfig")) { + // Special case for reading cobfig files from test data; we might want to clean this up a bit + return true; + } + + if (filePermission.getName().contains("/search-guard-suite")) { + // In some cases the local cluster seems to start using just the project dir as cwd. Allow this for now, but we should fix the local cluster to use a temp dir + return true; + } + + if (filePermission.getName().contains("/modules") || filePermission.getName().contains("/plugins")) { + return true; + } + } + + if (baseEsPolicy.implies(protectionDomain, permission)) { + return true; + } + + if (classPathPermissions.implies(permission)) { + return true; + } + + if (miscPermissions.implies(permission)) { + return true; + } + + if (permission instanceof SocketPermission) { + // TODO make finer + return true; + } + + if (baseSystemPolicy != null && baseSystemPolicy.implies(protectionDomain, permission)) { + return true; + } + + String protectionDomainKey = getProtectionDomainKey(protectionDomain); + + if (permission instanceof SocketPermission && log.isTraceEnabled()) { + log.trace(permission + " " + protectionDomainKey + " " + protectionDomain.getCodeSource().getLocation() + " " + + protectionDomain.getClassLoader()); + } + + if ("search-guard-plugin".equals(protectionDomainKey)) { + if (sgPluginPolicy.implies(protectionDomain, permission)) { + return true; + } else { + return false; + } + } else { + // We trust trust all other libraries. The most important point in this code is that the Search Guard code is properly trust checked. + // As the final result depends on the intersection of all stack frames, this should be sufficient for most cases. + return true; + } + } + + private String getProtectionDomainKey(ProtectionDomain protectionDomain) { + String uri = protectionDomain.getCodeSource().getLocation().toExternalForm(); + + if (uri.contains("/org/elasticsearch/") && uri.endsWith(".jar")) { + return "es"; + } + + if (uri.contains("/org/apache/lucene/") && uri.endsWith(".jar")) { + return "lucene"; + } + + if (uri.contains("/io/netty/") && uri.endsWith(".jar")) { + return "netty"; + } + + Matcher jarPatternMatcher = JAR_PATTERN.matcher(uri); + + if (jarPatternMatcher.matches()) { + return jarPatternMatcher.group(1) + ".jar"; + } + + if (uri.endsWith("/target/test-classes/")) { + return "test-classes"; + } + + if (uri.endsWith("/target/classes/")) { + return "search-guard-plugin"; + } + + if (uri.contains("eclipse/configuration")) { + return "test-runner"; + } + + return uri; + } + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/LocalCluster.java b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/LocalCluster.java new file mode 100644 index 0000000000..ebba3b17e0 --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/LocalCluster.java @@ -0,0 +1,462 @@ +/* + * Copyright 2015-2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.opensearch.security.test.helper.cluster.newstyle; + +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.rules.ExternalResource; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.WriteRequest.RefreshPolicy; +import org.opensearch.client.Client; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.settings.Settings; +import org.opensearch.node.PluginAwareNode; +import org.opensearch.plugins.Plugin; +import org.opensearch.security.action.configupdate.ConfigUpdateAction; +import org.opensearch.security.action.configupdate.ConfigUpdateRequest; +import org.opensearch.security.action.configupdate.ConfigUpdateResponse; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.ssl.util.SSLConfigConstants; +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.security.test.NodeSettingsSupplier; +import org.opensearch.security.test.helper.cluster.ClusterConfiguration; +import org.opensearch.security.test.helper.cluster.newstyle.TestSgConfig.Role; +import org.opensearch.security.test.helper.file.FileHelper; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +public class LocalCluster extends ExternalResource implements AutoCloseable, EsClientProvider { + private static final Logger log = LogManager.getLogger(LocalCluster.class); + + static { + System.setProperty("sg.default_init.dir", new File("./sgconfig").getAbsolutePath()); + } + + protected static final AtomicLong num = new AtomicLong(); + + protected final String resourceFolder; + private final List> plugins; + private final ClusterConfiguration clusterConfiguration; + private final TestSgConfig testSgConfig; + private final Settings nodeOverride; + private final String clusterName; + private LocalEsCluster localCluster; + + public LocalCluster(String clusterName, String resourceFolder, TestSgConfig testSgConfig, Settings nodeOverride, ClusterConfiguration clusterConfiguration, + List> plugins) { + this.resourceFolder = resourceFolder; + this.plugins = plugins; + this.clusterConfiguration = clusterConfiguration; + this.testSgConfig = testSgConfig; + this.nodeOverride = nodeOverride; + this.clusterName = clusterName; + + painlessWhitelistKludge(); + + start(); + } + + @Override + protected void before() throws Throwable { + if (localCluster == null) { + start(); + } + } + + @Override + protected void after() { + if (localCluster != null && localCluster.isStarted()) { + try { + Thread.sleep(1234); + localCluster.destroy(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + localCluster = null; + } + } + } + + @Override + public void close() throws Exception { + if (localCluster != null && localCluster.isStarted()) { + try { + Thread.sleep(100); + localCluster.destroy(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + localCluster = null; + } + } + } + + public X getInjectable(Class clazz) { + return this.localCluster.masterNode().getInjectable(clazz); + } + + public PluginAwareNode node() { + return this.localCluster.masterNode().esNode(); + } + + public List nodes() { + return this.localCluster.allNodes(); + } + + public LocalEsCluster.Node getNodeByName(String name) { + return this.localCluster.getNodeByName(name); + } + + public void updateSgConfig(CType configType, String key, Map value) { + //todo changed code +// try (Client client = getAdminCertClient()) { + try (Client client = getInternalNodeClient()) { + log.info("Updating config " + configType + "." + key + ": " + value); + + GetResponse getResponse = client.get(new GetRequest(testSgConfig.getIndexName(), configType.toLCString())).actionGet(); + String jsonDoc = new String(Base64.getDecoder().decode(String.valueOf(getResponse.getSource().get(configType.toLCString())))); + NestedValueMap config = NestedValueMap.fromJsonString(jsonDoc); + + config.put(key, value); + + if (log.isTraceEnabled()) { + log.trace("Updated config: " + config); + } + + IndexResponse response = client + .index(new IndexRequest(testSgConfig.getIndexName()).id(configType.toLCString()).setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source(configType.toLCString(), BytesReference.fromByteBuffer(ByteBuffer.wrap(config.toJsonString().getBytes("utf-8"))))) + .actionGet(); + + if (response.getResult() != DocWriteResponse.Result.UPDATED) { + throw new RuntimeException("Updated failed " + response); + } + + ConfigUpdateResponse configUpdateResponse = client + .execute(ConfigUpdateAction.INSTANCE, new ConfigUpdateRequest(CType.lcStringValues().toArray(new String[0]))).actionGet(); + + if (configUpdateResponse.hasFailures()) { + throw new RuntimeException("ConfigUpdateResponse produced failures: " + configUpdateResponse.failures()); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void start() { + try { + this.localCluster = new LocalEsCluster(clusterName, clusterConfiguration, minimumSearchGuardSettings(ccs(nodeOverride)), resourceFolder, + plugins); + localCluster.start(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + + if (testSgConfig != null) { + initSearchGuardIndex(testSgConfig); + } + } + + private void painlessWhitelistKludge() { + try { + // TODO make this optional + + /* + + + final ClassLoader classLoader = getClass().getClassLoader(); + + try (PainlessPlugin p = new PainlessPlugin()) { + p.loadExtensions(new ExtensionLoader() { + + @SuppressWarnings("unchecked") + @Override + public List loadExtensions(Class extensionPointType) { + if (extensionPointType.equals(PainlessExtension.class)) { + List result = StreamSupport.stream(ServiceLoader.load(PainlessExtension.class, classLoader).spliterator(), false) + .collect(Collectors.toList()); + + return (List) result; + } else { + return Collections.emptyList(); + } + } + }); + } catch (Exception e) { + e.printStackTrace(); + } + */ + } catch (NoClassDefFoundError e) { + + } + } + + protected void initSearchGuardIndex(TestSgConfig testSgConfig) { + + log.info("Initializing Search Guard index"); + + try (Client client = getInternalNodeClient()) { + + testSgConfig.initIndex(client); + //todo commented code +// Assert.assertTrue(client.get(new GetRequest(testSgConfig.getIndexName(), "config")).actionGet().isExists()); +// Assert.assertTrue(client.get(new GetRequest(testSgConfig.getIndexName(), "internalusers")).actionGet().isExists()); +// Assert.assertTrue(client.get(new GetRequest(testSgConfig.getIndexName(), "roles")).actionGet().isExists()); +// Assert.assertTrue(client.get(new GetRequest(testSgConfig.getIndexName(), "rolesmapping")).actionGet().isExists()); +// Assert.assertTrue(client.get(new GetRequest(testSgConfig.getIndexName(), "actiongroups")).actionGet().isExists()); +// Assert.assertFalse(client.get(new GetRequest(testSgConfig.getIndexName(), "rolesmapping_xcvdnghtu165759i99465")).actionGet().isExists()); +// Assert.assertTrue(client.get(new GetRequest(testSgConfig.getIndexName(), "config")).actionGet().isExists()); + } + } + + private Settings ccs(Settings nodeOverride) { + + return nodeOverride; + } + + protected Settings.Builder minimumSearchGuardSettingsBuilder(int node, boolean sslOnly) { + final String prefix = getResourceFolder() == null ? "" : getResourceFolder() + "/"; + + Settings.Builder builder = Settings.builder() + //.put("searchguard.ssl.transport.enabled", true) + //.put("searchguard.no_default_init", true) + .put(SSLConfigConstants.SECURITY_SSL_HTTP_ENABLE_OPENSSL_IF_AVAILABLE, false) + .put(SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENABLE_OPENSSL_IF_AVAILABLE, false) + .put("plugins.security.ssl.transport.keystore_alias", "node-0") + .put("plugins.security.ssl.transport.keystore_filepath", FileHelper.getAbsoluteFilePathFromClassPath(prefix + "node-0-keystore.jks")) + .put("plugins.security.ssl.transport.truststore_filepath", FileHelper.getAbsoluteFilePathFromClassPath(prefix + "truststore.jks")) + .put("plugins.security.ssl.transport.enforce_hostname_verification", false); + + if (!sslOnly) { + builder.putList("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,l=tEst, C=De"); + builder.put(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false); + } + + return builder; + } + + protected NodeSettingsSupplier minimumSearchGuardSettings(Settings other) { + return new NodeSettingsSupplier() { + @Override + public Settings get(int i) { + return minimumSearchGuardSettingsBuilder(i, false).put(other).build(); + } + }; + } + + protected NodeSettingsSupplier minimumSearchGuardSettingsSslOnly(Settings other) { + return new NodeSettingsSupplier() { + @Override + public Settings get(int i) { + return minimumSearchGuardSettingsBuilder(i, true).put(other).build(); + } + }; + } + + public String getResourceFolder() { + return resourceFolder; + } + + public static class Builder { + private boolean sslEnabled; + private String httpKeystoreFilepath = "node-0-keystore.jks"; + private String httpTruststoreFilepath = "truststore.jks"; + private String resourceFolder; + private ClusterConfiguration clusterConfiguration = ClusterConfiguration.DEFAULT; + private Settings.Builder nodeOverrideSettingsBuilder = Settings.builder(); + private List disabledModules = new ArrayList<>(); + private List> plugins = new ArrayList<>(); + private TestSgConfig testSgConfig = new TestSgConfig().resources("/"); + private String clusterName = "local_cluster"; + + public Builder sslEnabled() { + this.sslEnabled = true; + return this; + } + + public Builder dependsOn(Object object) { + // We just want to make sure that the object is already done + if (object == null) { + throw new IllegalStateException("Dependency not fulfilled"); + } + return this; + } + + public Builder resources(String resourceFolder) { + this.resourceFolder = resourceFolder; + testSgConfig.resources(resourceFolder); + return this; + } + + public Builder clusterConfiguration(ClusterConfiguration clusterConfiguration) { + this.clusterConfiguration = clusterConfiguration; + return this; + } + + public Builder singleNode() { + this.clusterConfiguration = ClusterConfiguration.SINGLENODE; + return this; + } + + public Builder sgConfig(TestSgConfig testSgConfig) { + this.testSgConfig = testSgConfig; + return this; + } + + public Builder setInSgConfig(String keyPath, Object value, Object... more) { + testSgConfig.sgConfigSettings(keyPath, value, more); + return this; + } + + public Builder nodeSettings(Object... settings) { + + for (int i = 0; i < settings.length - 1; i += 2) { + String key = String.valueOf(settings[i]); + Object value = settings[i + 1]; + + nodeOverrideSettingsBuilder.put(key, String.valueOf(value)); + } + + return this; + } + + //todo commented code +// public Builder disableModule(Class> moduleClass) { +// this.disabledModules.add(moduleClass.getName()); +// +// return this; +// } + + public Builder plugin(Class plugin) { + this.plugins.add(plugin); + + return this; + } + + public Builder remote(String name, LocalCluster anotherCluster) { + InetSocketAddress transportAddress = anotherCluster.localCluster.masterNode().getTransportAddress(); + + nodeOverrideSettingsBuilder.putList("cluster.remote." + name + ".seeds", + transportAddress.getHostString() + ":" + transportAddress.getPort()); + + return this; + } + + public Builder users(TestSgConfig.User... users) { + for (TestSgConfig.User user : users) { + testSgConfig.user(user); + } + return this; + } + + public Builder user(TestSgConfig.User user) { + testSgConfig.user(user); + return this; + } + + public Builder user(String name, String password, String... sgRoles) { + testSgConfig.user(name, password, sgRoles); + return this; + } + + public Builder user(String name, String password, Role... sgRoles) { + testSgConfig.user(name, password, sgRoles); + return this; + } + + public Builder roles(Role... roles) { + testSgConfig.roles(roles); + return this; + } + + public Builder clusterName(String clusterName) { + this.clusterName = clusterName; + return this; + } + + public LocalCluster build() { + try { + + if (sslEnabled) { + nodeOverrideSettingsBuilder.put("plugins.security.ssl.http.enabled", true) + .put("plugins.security.ssl.http.keystore_filepath", + FileHelper.getAbsoluteFilePathFromClassPath( + resourceFolder != null ? (resourceFolder + "/" + httpKeystoreFilepath) : httpKeystoreFilepath)) + .put("plugins.security.ssl.http.truststore_filepath", FileHelper.getAbsoluteFilePathFromClassPath( + resourceFolder != null ? (resourceFolder + "/" + httpTruststoreFilepath) : httpTruststoreFilepath)); + } + + //todo commented code +// if (this.disabledModules.size() > 0) { +// nodeOverrideSettingsBuilder.putList(SearchGuardModulesRegistry.DISABLED_MODULES.getKey(), this.disabledModules); +// } + + clusterName += "_" + num.incrementAndGet(); + + return new LocalCluster(clusterName, resourceFolder, testSgConfig, nodeOverrideSettingsBuilder.build(), clusterConfiguration, plugins); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + } + + @Override + public InetSocketAddress getHttpAddress() { + return localCluster.clientNode().getHttpAddress(); + } + + @Override + public InetSocketAddress getTransportAddress() { + return localCluster.clientNode().getTransportAddress(); + } + + @Override + public String getClusterName() { + return localCluster.getClusterName(); + } + + @Override + public SSLIOSessionStrategy getSSLIOSessionStrategy() { + return localCluster.getSSLIOSessionStrategy(); + } + + @Override + public Client getInternalNodeClient() { + return localCluster.clientNode().getInternalNodeClient(); + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/LocalEsCluster.java b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/LocalEsCluster.java new file mode 100644 index 0000000000..8cc17a0deb --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/LocalEsCluster.java @@ -0,0 +1,642 @@ +/* + * Copyright 2015-2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.opensearch.security.test.helper.cluster.newstyle; + +import com.google.common.net.InetAddresses; +import org.apache.commons.io.FileUtils; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchTimeoutException; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.action.admin.cluster.node.info.NodeInfo; +import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.common.Strings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.TransportAddress; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.http.BindHttpException; +import org.opensearch.http.HttpInfo; +import org.opensearch.node.PluginAwareNode; +import org.opensearch.plugins.Plugin; +import org.opensearch.security.test.NodeSettingsSupplier; +import org.opensearch.security.test.helper.cluster.ClusterConfiguration; +import org.opensearch.security.test.helper.cluster.ClusterConfiguration.NodeSettings; +import org.opensearch.security.test.helper.cluster.ClusterInfo; +import org.opensearch.security.test.helper.file.FileHelper; +import org.opensearch.security.test.helper.network.PortAllocator; +import org.opensearch.transport.BindTransportException; +import org.opensearch.transport.TransportInfo; + +import javax.net.ssl.SSLContext; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.security.KeyStore; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +/** + * This is the SG-agnostic and ES-specific part of LocalCluster + */ +public class LocalEsCluster { + + static { + System.setProperty("es.enforce.bootstrap.checks", "true"); + } + + private static final Logger log = LogManager.getLogger(LocalEsCluster.class); + + private final String clusterName; + private final ClusterConfiguration clusterConfiguration; + private final NodeSettingsSupplier nodeSettingsSupplier; + private final List> additionalPlugins; + private File clusterHomeDir; + private final List allNodes = new ArrayList<>(); + private final List masterNodes = new ArrayList<>(); + private final List dataNodes = new ArrayList<>(); + private final List clientNodes = new ArrayList<>(); + + // TODO replace by proper TLS config + private String resourcesFolder; + + private List seedHosts; + private List initialMasterHosts; + private TimeValue timeout = TimeValue.timeValueSeconds(10); + private int retry = 0; + private boolean started; + + public LocalEsCluster(String clustername, ClusterConfiguration clusterConfiguration, NodeSettingsSupplier nodeSettingsSupplier, + String resourcesFolder, List> additionalPlugins) { + super(); + this.clusterName = clustername; + this.clusterConfiguration = clusterConfiguration; + this.nodeSettingsSupplier = nodeSettingsSupplier; + this.additionalPlugins = additionalPlugins; + this.resourcesFolder = resourcesFolder; + try { + this.clusterHomeDir = Files.createTempDirectory("sg_local_cluster_" + clustername).toFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void start() throws Exception { + + if (log.isDebugEnabled()) { + log.debug("Starting " + clusterName); + } + + int forkNumber = getUnitTestForkNumber(); + int masterNodeCount = clusterConfiguration.getMasterNodes(); + int nonMasterNodeCount = clusterConfiguration.getDataNodes() + clusterConfiguration.getClientNodes(); + + SortedSet masterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, Math.max(masterNodeCount, 4), + 5000 + forkNumber * 1000 + 300); + SortedSet masterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, masterNodeCount, 5000 + forkNumber * 1000 + 200); + + this.seedHosts = toHostList(masterNodeTransportPorts); + this.initialMasterHosts = toHostList(masterNodeTransportPorts.stream().limit(masterNodeCount).collect(Collectors.toSet())); + + started = true; + + CompletableFuture masterNodeFuture = startNodes(clusterConfiguration.getMasterNodeSettings(), masterNodeTransportPorts, + masterNodeHttpPorts); + + SortedSet nonMasterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + forkNumber * 1000 + 310); + SortedSet nonMasterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + forkNumber * 1000 + 210); + + CompletableFuture otherNodeFuture = startNodes(clusterConfiguration.getNonMasterNodeSettings(), nonMasterNodeTransportPorts, + nonMasterNodeHttpPorts); + + CompletableFuture.allOf(masterNodeFuture, otherNodeFuture).join(); + + if (isNodeFailedWithPortCollision()) { + log.info("Detected port collision for master node. Retrying."); + + retry(); + return; + } + + if (log.isDebugEnabled()) { + log.debug("Startup finished. Waiting for GREEN"); + } + + waitForCluster(ClusterHealthStatus.GREEN, timeout, allNodes.size()); + putDefaultTemplate(); + + if (log.isInfoEnabled()) { + log.info("Started: " + this); + } + + } + + private void putDefaultTemplate() { + String defaultTemplate = "{\n" // + + " \"index_patterns\": [\"*\"],\n" // + + " \"order\": -1,\n" + " \"settings\": {\n" // + + " \"number_of_shards\": \"5\",\n" // + + " \"number_of_replicas\": \"1\"\n"// + + " }\n" // + + " }"; + + AcknowledgedResponse templateAck = clientNode().getInternalNodeClient().admin().indices() + .putTemplate(new PutIndexTemplateRequest("default").source(defaultTemplate, XContentType.JSON)).actionGet(); + + if (!templateAck.isAcknowledged()) { + throw new RuntimeException("Default template could not be created"); + } + } + + private CompletableFuture startNodes(List nodeSettingList, SortedSet transportPorts, SortedSet httpPorts) { + Iterator transportPortIterator = transportPorts.iterator(); + Iterator httpPortIterator = httpPorts.iterator(); + List> futures = new ArrayList<>(); + + for (NodeSettings nodeSettings : nodeSettingList) { + Node node = new Node(nodeSettings, transportPortIterator.next(), httpPortIterator.next()); + futures.add(node.start()); + } + + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + + private int getUnitTestForkNumber() { + + String forkno = System.getProperty("forkno"); + + if (forkno != null && forkno.length() > 0) { + return Integer.parseInt(forkno.split("_")[1]); + } else { + return 42; + } + } + + public void stop() { + + for (Node node : clientNodes) { + node.stop(); + } + + for (Node node : dataNodes) { + node.stop(); + } + + for (Node node : masterNodes) { + node.stop(); + } + } + + public void destroy() { + stop(); + clientNodes.clear(); + dataNodes.clear(); + masterNodes.clear(); + + try { + FileUtils.deleteDirectory(clusterHomeDir); + } catch (IOException e) { + log.warn("Error while deleting " + clusterHomeDir, e); + } + } + + public Node clientNode() { + return findRunningNode(clientNodes, dataNodes, masterNodes); + } + + public Node masterNode() { + return findRunningNode(masterNodes); + } + + private boolean isNodeFailedWithPortCollision() { + for (Node node : allNodes) { + if (node.isPortCollision()) { + return true; + } + } + + return false; + } + + private void retry() throws Exception { + retry++; + + if (retry > 10) { + throw new RuntimeException("Detected port collisions for master node. Giving up."); + } + + stop(); + + this.allNodes.clear(); + this.masterNodes.clear(); + this.dataNodes.clear(); + this.clientNodes.clear(); + this.seedHosts = null; + this.initialMasterHosts = null; + this.clusterHomeDir = Files.createTempDirectory("sg_local_cluster_" + clusterName + "_retry_" + retry).toFile(); + + start(); + } + + @SafeVarargs + private static final Node findRunningNode(List nodes, List... moreNodes) { + for (Node node : nodes) { + if (node.isRunning()) { + return node; + } + } + + if (moreNodes != null && moreNodes.length > 0) { + for (List nodesList : moreNodes) { + for (Node node : nodesList) { + if (node.isRunning()) { + return node; + } + } + } + } + + return null; + } + + public List allNodes() { + return Collections.unmodifiableList(allNodes); + } + + public Node getNodeByName(String name) { + for (Node node : allNodes) { + if (node.getNodeName().equals(name)) { + return node; + } + } + + throw new RuntimeException( + "No such node name: " + name + "; available: " + allNodes.stream().map((n) -> n.getNodeName()).collect(Collectors.toList())); + } + + public ClusterInfo waitForCluster(ClusterHealthStatus status, TimeValue timeout, int expectedNodeCount) throws IOException { + + ClusterInfo clusterInfo = new ClusterInfo(); + Client client = clientNode().getInternalNodeClient(); + + try { + log.debug("waiting for cluster state {} and {} nodes", status.name(), expectedNodeCount); + final ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForStatus(status).setTimeout(timeout) + .setMasterNodeTimeout(timeout).setWaitForNodes("" + expectedNodeCount).execute().actionGet(); + + if (log.isDebugEnabled()) { + log.debug("Current ClusterState:\n" + Strings.toString(healthResponse)); + } + + if (healthResponse.isTimedOut()) { + throw new IOException( + "cluster state is " + healthResponse.getStatus().name() + " with " + healthResponse.getNumberOfNodes() + " nodes"); + } else { + log.debug("... cluster state ok " + healthResponse.getStatus().name() + " with " + healthResponse.getNumberOfNodes() + " nodes"); + } + + org.junit.Assert.assertEquals(expectedNodeCount, healthResponse.getNumberOfNodes()); + + final NodesInfoResponse res = client.admin().cluster().nodesInfo(new NodesInfoRequest()).actionGet(); + + final List nodes = res.getNodes(); + + final List masterNodes = nodes.stream().filter(n -> n.getNode().getRoles().contains(DiscoveryNodeRole.MASTER_ROLE)) + .collect(Collectors.toList()); + final List dataNodes = nodes.stream().filter(n -> n.getNode().getRoles().contains(DiscoveryNodeRole.DATA_ROLE) + && !n.getNode().getRoles().contains(DiscoveryNodeRole.MASTER_ROLE)).collect(Collectors.toList()); + final List clientNodes = nodes.stream().filter(n -> !n.getNode().getRoles().contains(DiscoveryNodeRole.MASTER_ROLE) + && !n.getNode().getRoles().contains(DiscoveryNodeRole.DATA_ROLE)).collect(Collectors.toList()); + + for (NodeInfo nodeInfo : masterNodes) { + final TransportAddress is = nodeInfo.getInfo(TransportInfo.class).getAddress().publishAddress(); + clusterInfo.nodePort = is.getPort(); + clusterInfo.nodeHost = is.getAddress(); + } + + if (!clientNodes.isEmpty()) { + NodeInfo nodeInfo = clientNodes.get(0); + if (nodeInfo.getInfo(HttpInfo.class) != null && nodeInfo.getInfo(HttpInfo.class).address() != null) { + final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); + clusterInfo.httpPort = his.getPort(); + clusterInfo.httpHost = his.getAddress(); + } else { + throw new RuntimeException("no http host/port for client node"); + } + } else if (!dataNodes.isEmpty()) { + + for (NodeInfo nodeInfo : dataNodes) { + if (nodeInfo.getInfo(HttpInfo.class) != null && nodeInfo.getInfo(HttpInfo.class).address() != null) { + final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); + clusterInfo.httpPort = his.getPort(); + clusterInfo.httpHost = his.getAddress(); + break; + } + } + } else { + + for (NodeInfo nodeInfo : nodes) { + if (nodeInfo.getInfo(HttpInfo.class) != null && nodeInfo.getInfo(HttpInfo.class).address() != null) { + final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); + clusterInfo.httpPort = his.getPort(); + clusterInfo.httpHost = his.getAddress(); + break; + } + } + } + + for (NodeInfo nodeInfo : nodes) { + clusterInfo.httpAdresses.add(nodeInfo.getInfo(HttpInfo.class).address().publishAddress()); + } + } catch (final OpenSearchTimeoutException e) { + throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); + } + return clusterInfo; + } + + @Override + public String toString() { + return "\nES Cluster " + clusterName + "\nmaster nodes: " + masterNodes + "\n data nodes: " + dataNodes + "\nclient nodes: " + clientNodes + + "\n"; + } + + private SSLContext getSSLContext() { + try { + String truststoreType = "JKS"; + String truststorePassword = "changeit"; + String prefix = resourcesFolder == null ? "" : resourcesFolder + "/"; + + KeyStore trustStore = KeyStore.getInstance(truststoreType); + try (InputStream in = Files.newInputStream(FileHelper.getAbsoluteFilePathFromClassPath(prefix + "truststore.jks"))) { + trustStore.load(in, (truststorePassword == null || truststorePassword.length() == 0) ? null : truststorePassword.toCharArray()); + } + + SSLContextBuilder sslContextBuilder = SSLContexts.custom().loadTrustMaterial(trustStore, null); + return sslContextBuilder.build(); + + //return new OverlyTrustfulSSLContextBuilder().build(); + } catch (Exception e) { + throw new RuntimeException("Error while building SSLContext", e); + } + } + + public SSLIOSessionStrategy getSSLIOSessionStrategy() { + return new SSLIOSessionStrategy(getSSLContext(), null, null, NoopHostnameVerifier.INSTANCE); + } + + private static List toHostList(Collection ports) { + return ports.stream().map(s -> "127.0.0.1:" + s).collect(Collectors.toList()); + } + + public class Node implements EsClientProvider { + private final String nodeName; + private final NodeSettings nodeSettings; + private final File nodeHomeDir; + private final File dataDir; + private final File logsDir; + private final int transportPort; + private final int httpPort; + private final InetAddress hostAddress; + private final InetSocketAddress httpAddress; + private final InetSocketAddress transportAddress; + private PluginAwareNode node; + private boolean running = false; + private boolean portCollision = false; + + Node(NodeSettings nodeSettings, int transportPort, int httpPort) { + this.nodeName = createNextNodeName(nodeSettings); + this.nodeSettings = nodeSettings; + this.nodeHomeDir = new File(clusterHomeDir, nodeName); + this.dataDir = new File(this.nodeHomeDir, "data"); + this.logsDir = new File(this.nodeHomeDir, "logs"); + this.transportPort = transportPort; + this.httpPort = httpPort; + this.hostAddress = InetAddresses.forString("127.0.0.1"); + this.httpAddress = new InetSocketAddress(this.hostAddress, httpPort); + this.transportAddress = new InetSocketAddress(this.hostAddress, transportPort); + + if (nodeSettings.masterNode) { + masterNodes.add(this); + } else if (nodeSettings.dataNode) { + dataNodes.add(this); + } else { + clientNodes.add(this); + } + + allNodes.add(this); + + } + + CompletableFuture start() { + CompletableFuture completableFuture = new CompletableFuture<>(); + + this.node = new PluginAwareNode(nodeSettings.masterNode, getEsSettings(), nodeSettings.getPlugins(additionalPlugins)); + + new Thread(new Runnable() { + + @Override + public void run() { + try { + node.start(); + running = true; + completableFuture.complete("initialized"); + } catch (BindTransportException | BindHttpException e) { + log.warn("Port collision detected for " + this, e); + portCollision = true; + try { + node.close(); + } catch (IOException e2) { + e2.printStackTrace(); + } + + node = null; + PortAllocator.TCP.blacklist(transportPort, httpPort); + + completableFuture.complete("retry"); + + } catch (Throwable e) { + log.error("Unable to start " + this, e); + node = null; + completableFuture.completeExceptionally(e); + } + } + }).start(); + + return completableFuture; + } + + Settings getEsSettings() { + Settings settings = getMinimalEsSettings(); + + if (nodeSettingsSupplier != null) { + // TODO node number + settings = Settings.builder().put(settings).put(nodeSettingsSupplier.get(0)).build(); + } + + return settings; + } + + Settings getMinimalEsSettings() { + + return Settings.builder().put("node.name", nodeName)// + .put("node.data", nodeSettings.dataNode)// + .put("node.master", nodeSettings.masterNode)// + .put("cluster.name", clusterName)// + .put("path.home", nodeHomeDir.toPath())// + .put("path.data", dataDir.toPath())// + .put("path.logs", logsDir.toPath())// + .putList("cluster.initial_master_nodes", initialMasterHosts)// + .put("discovery.initial_state_timeout", "8s")// + .putList("discovery.seed_hosts", seedHosts)// + .put("transport.tcp.port", transportPort)// + .put("http.port", httpPort)// + .put("cluster.routing.allocation.disk.threshold_enabled", false)// + .put("discovery.probe.connect_timeout", "10s") + .put("discovery.probe.handshake_timeout", "10s") + .put("http.cors.enabled", true).build(); + } + + @Override + public Client getInternalNodeClient() { + return node.client(); + } + + public PluginAwareNode esNode() { + return node; + } + + public boolean isRunning() { + return running; + } + + public X getInjectable(Class clazz) { + return node.injector().getInstance(clazz); + } + + public void stop() { + try { + log.info("Stopping " + this); + + running = false; + + if (node != null) { + node.close(); + node = null; + Thread.sleep(10); + } + + } catch (Throwable e) { + log.warn("Error while stopping " + this, e); + } + } + + @Override + public String toString() { + String state = running ? "RUNNING" : node != null ? "INITIALIZING" : "STOPPED"; + + return nodeName + " " + state + " [" + transportPort + ", " + httpPort + "]"; + } + + public boolean isPortCollision() { + return portCollision; + } + + public String getNodeName() { + return nodeName; + } + + public int getTransportPort() { + return transportPort; + } + + public int getHttpPort() { + return httpPort; + } + + public String getHost() { + return "127.0.0.1"; + } + + @Override + public InetSocketAddress getHttpAddress() { + return httpAddress; + } + + @Override + public InetSocketAddress getTransportAddress() { + return transportAddress; + } + + @Override + public String getResourceFolder() { + return resourcesFolder; + } + + @Override + public SSLIOSessionStrategy getSSLIOSessionStrategy() { + return LocalEsCluster.this.getSSLIOSessionStrategy(); + } + + @Override + public String getClusterName() { + return LocalEsCluster.this.clusterName; + } + + } + + private String createNextNodeName(NodeSettings nodeSettings) { + List nodes; + String nodeType; + + if (nodeSettings.masterNode) { + nodes = this.masterNodes; + nodeType = "master"; + } else if (nodeSettings.dataNode) { + nodes = this.dataNodes; + nodeType = "data"; + } else { + nodes = this.clientNodes; + nodeType = "client"; + } + + return nodeType + "_" + nodes.size(); + } + + public String getClusterName() { + return clusterName; + } + + public boolean isStarted() { + return started; + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/NestedValueMap.java b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/NestedValueMap.java new file mode 100644 index 0000000000..507af0151f --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/NestedValueMap.java @@ -0,0 +1,499 @@ +package org.opensearch.security.test.helper.cluster.newstyle; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.MapMaker; +import org.opensearch.security.DefaultObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Array; +import java.net.URI; +import java.net.URL; +import java.util.*; + +public class NestedValueMap extends HashMap { + + private static final long serialVersionUID = 2953312818482932741L; + + private Map originalToCloneMap; + private final boolean cloneWhilePut; + private boolean writable = true; + + public NestedValueMap() { + originalToCloneMap = new MapMaker().weakKeys().makeMap(); + cloneWhilePut = true; + } + + public NestedValueMap(int initialCapacity) { + super(initialCapacity); + originalToCloneMap = new MapMaker().weakKeys().makeMap(); + cloneWhilePut = true; + } + + NestedValueMap(Map originalToCloneMap, boolean cloneWhilePut) { + this.originalToCloneMap = originalToCloneMap; + this.cloneWhilePut = cloneWhilePut; + } + + NestedValueMap(int initialCapacity, Map originalToCloneMap, boolean cloneWhilePut) { + super(initialCapacity); + this.originalToCloneMap = originalToCloneMap; + this.cloneWhilePut = cloneWhilePut; + } + + @Override + public NestedValueMap clone() { + NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), + this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); + + result.putAll(this); + + return result; + } + + public NestedValueMap without(String... keys) { + NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), + this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); + + Set withoutKeySet = new HashSet<>(Arrays.asList(keys)); + + for (Entry entry : this.entrySet()) { + if (!withoutKeySet.contains(entry.getKey())) { + result.put(entry.getKey(), entry.getValue()); + } + } + + return result; + } + + public static NestedValueMap copy(Map data) { + NestedValueMap result = new NestedValueMap(data.size()); + + result.putAllFromAnyMap(data); + + return result; + } + + public static NestedValueMap copy(Object data) { + if (data instanceof Map) { + return copy((Map) data); + } else { + NestedValueMap result = new NestedValueMap(); + result.put("_value", data); + return result; + } + } + + public static NestedValueMap createNonCloningMap() { + return new NestedValueMap(null, false); + } + + public static NestedValueMap createUnmodifieableMap(Map data) { + NestedValueMap result = new NestedValueMap(data.size()); + + result.putAllFromAnyMap(data); + result.seal(); + + return result; + } + + public static NestedValueMap fromJsonString(String jsonString) throws IOException { + return NestedValueMap.copy(DefaultObjectMapper.readValue(jsonString, Map.class)); + } + + public static NestedValueMap fromYaml(String yamlString) throws IOException { + return NestedValueMap.copy(DefaultObjectMapper.YAML_MAPPER.readValue(yamlString, Object.class)); + } + + public static NestedValueMap fromYaml(InputStream inputSteam) throws IOException { + return NestedValueMap.copy(DefaultObjectMapper.YAML_MAPPER.readValue(inputSteam, Object.class)); + } + + public static NestedValueMap fromJsonArrayString(String jsonString) throws IOException { + return NestedValueMap.copy(DefaultObjectMapper.readValue(jsonString, List.class)); + } + + public static NestedValueMap of(String key1, Object value1) { + NestedValueMap result = new NestedValueMap(1); + result.put(key1, value1); + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2) { + NestedValueMap result = new NestedValueMap(2); + result.put(key1, value1); + result.put(key2, value2); + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3) { + NestedValueMap result = new NestedValueMap(3); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3, Object... furtherEntries) { + NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + for (int i = 0; i < furtherEntries.length - 1; i += 2) { + result.put(String.valueOf(furtherEntries[i]), furtherEntries[i + 1]); + } + + return result; + } + + public static NestedValueMap of(Path key1, Object value1) { + NestedValueMap result = new NestedValueMap(1); + result.put(key1, value1); + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2) { + NestedValueMap result = new NestedValueMap(2); + result.put(key1, value1); + result.put(key2, value2); + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3) { + NestedValueMap result = new NestedValueMap(3); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3, Object... furtherEntries) { + NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + for (int i = 0; i < furtherEntries.length - 1; i += 2) { + result.put(Path.parse(String.valueOf(furtherEntries[i])), furtherEntries[i + 1]); + } + + return result; + } + + public Object put(String key, Map data) { + checkWritable(); + + Object result = this.get(key); + NestedValueMap subMap = this.getOrCreateSubMapAt(key, data.size()); + + subMap.putAllFromAnyMap(data); + return result; + } + + public void putAll(Map map) { + checkWritable(); + + for (Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + put(key, entry.getValue()); + } + } + + public void putAllFromAnyMap(Map map) { + checkWritable(); + + for (Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + put(key, entry.getValue()); + } + } + + public void overrideLeafs(NestedValueMap map) { + checkWritable(); + + for (Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + + if (entry.getValue() instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) entry.getValue(); + + getOrCreateSubMapAt(key, subMap.size()).overrideLeafs(subMap); + } else { + put(key, entry.getValue()); + } + } + } + + public Object put(String key, Object object) { + checkWritable(); + + if (object instanceof Map) { + return put(key, (Map) object); + } + + return super.put(key, deepCloneObject(object)); + } + + public void put(Path path, Object object) { + checkWritable(); + + if (path.isEmpty()) { + if (object instanceof Map) { + putAllFromAnyMap((Map) object); + } else { + throw new IllegalArgumentException("put([], " + object + "): If an empty path is given, the object must be of type map"); + } + + } else { + NestedValueMap subMap = getOrCreateSubMapAtPath(path.withoutLast()); + subMap.put(path.getLast(), object); + } + } + + public Object get(Path path) { + if (path.isEmpty()) { + return this; + } else if (path.length() == 1) { + return this.get(path.getFirst()); + } else { + Object subObject = this.get(path.getFirst()); + + if (subObject instanceof NestedValueMap) { + return ((NestedValueMap) subObject).get(path.withoutFirst()); + } else { + return null; + } + } + } + + public void seal() { + if (!this.writable) { + return; + } + + this.writable = false; + this.originalToCloneMap = null; + + for (Object value : this.values()) { + if (value instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) value; + subMap.seal(); + } else if (value instanceof Iterable) { + for (Object subValue : ((Iterable) value)) { + if (subValue instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) subValue; + subMap.seal(); + } + } + } + } + } + + public String toJsonString() { + try { + return DefaultObjectMapper.objectMapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public String toYamlString() { + try { + return DefaultObjectMapper.YAML_MAPPER.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Object deepCloneObject(Object object) { + if (!cloneWhilePut || object == null || isImmutable(object)) { + return object; + } + + Object clone = this.originalToCloneMap.get(object); + + if (clone != null) { + return clone; + } + + if (object instanceof Set) { + Set set = (Set) object; + Set copy = new HashSet<>(set.size()); + this.originalToCloneMap.put(object, copy); + + for (Object element : set) { + copy.add(deepCloneObject(element)); + } + + return copy; + } else if (object instanceof Map) { + Map map = (Map) object; + NestedValueMap copy = new NestedValueMap(map.size(), this.originalToCloneMap, this.cloneWhilePut); + this.originalToCloneMap.put(object, copy); + + for (Entry entry : map.entrySet()) { + copy.put((String) deepCloneObject(String.valueOf(entry.getKey())), deepCloneObject(entry.getValue())); + } + + return copy; + } else if (object instanceof Collection) { + Collection collection = (Collection) object; + ArrayList copy = new ArrayList<>(collection.size()); + this.originalToCloneMap.put(object, copy); + + for (Object element : collection) { + copy.add(deepCloneObject(element)); + } + + return copy; + } else if (object.getClass().isArray()) { + int length = Array.getLength(object); + Object copy = Array.newInstance(object.getClass().getComponentType(), length); + this.originalToCloneMap.put(object, copy); + + for (int i = 0; i < length; i++) { + Array.set(copy, i, deepCloneObject(Array.get(object, i))); + } + + return copy; + } else { + // Hope the best + + return object; + } + } + + private boolean isImmutable(Object object) { + return object instanceof String || object instanceof Number || object instanceof Boolean || object instanceof Void || object instanceof Class + || object instanceof Character || object instanceof Enum || object instanceof File || object instanceof UUID || object instanceof URL + || object instanceof URI; + } + + private NestedValueMap getOrCreateSubMapAt(String key, int capacity) { + Object value = this.get(key); + + if (value instanceof NestedValueMap) { + return (NestedValueMap) value; + } else { + if (value instanceof Map) { + capacity = Math.max(capacity, ((Map) value).size()); + } + + NestedValueMap mapValue = new NestedValueMap(capacity, this.originalToCloneMap, this.cloneWhilePut); + + if (value instanceof Map) { + mapValue.putAllFromAnyMap((Map) value); + } + + super.put(key, mapValue); + return mapValue; + } + + } + + private NestedValueMap getOrCreateSubMapAtPath(Path path) { + if (path.isEmpty()) { + return this; + } + + String pathElement = path.getFirst(); + Path remainingPath = path.withoutFirst(); + + Object value = this.get(pathElement); + + if (value instanceof NestedValueMap) { + NestedValueMap mapValue = (NestedValueMap) value; + if (remainingPath.isEmpty()) { + return mapValue; + } else { + return mapValue.getOrCreateSubMapAtPath(remainingPath); + } + } else { + NestedValueMap mapValue = new NestedValueMap(this.originalToCloneMap, this.cloneWhilePut); + super.put(pathElement, mapValue); + + if (remainingPath.isEmpty()) { + return mapValue; + } else { + return mapValue.getOrCreateSubMapAtPath(remainingPath); + } + } + } + + private void checkWritable() { + if (!writable) { + throw new UnsupportedOperationException("Map is not writable"); + } + } + + public static class Path { + private String[] elements; + private int start; + private int end; + + public Path(String... elements) { + this.elements = elements; + this.start = 0; + this.end = elements.length; + } + + private Path(String[] elements, int start, int end) { + this.elements = elements; + this.start = start; + this.end = end; + } + + public String getFirst() { + if (this.start >= this.end) { + return null; + } + + return this.elements[start]; + } + + public String getLast() { + if (this.start >= this.end) { + return null; + } + + return this.elements[end - 1]; + } + + public Path withoutFirst() { + if (this.start >= this.end - 1) { + return new Path(null, 0, 0); + } + + return new Path(elements, start + 1, end); + } + + public Path withoutLast() { + if (this.start >= this.end - 1) { + return new Path(null, 0, 0); + } + + return new Path(elements, start, end - 1); + } + + public int length() { + return this.end - this.start; + } + + public boolean isEmpty() { + return this.start == this.end; + } + + public static Path parse(String path) { + if (path.length() == 0) { + return new Path(new String [0]); + } else { + return new Path(path.split("\\.")); + } + } + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/TestSgConfig.java b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/TestSgConfig.java new file mode 100644 index 0000000000..0ec90a6f3b --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/cluster/newstyle/TestSgConfig.java @@ -0,0 +1,612 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.opensearch.security.test.helper.cluster.newstyle; + +import com.google.common.collect.ImmutableMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.WriteRequest.RefreshPolicy; +import org.opensearch.client.Client; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.security.action.configupdate.ConfigUpdateAction; +import org.opensearch.security.action.configupdate.ConfigUpdateRequest; +import org.opensearch.security.action.configupdate.ConfigUpdateResponse; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.test.helper.cluster.newstyle.NestedValueMap.Path; +import org.opensearch.security.test.helper.file.FileHelper; +import org.opensearch.security.tools.Hasher; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.stream.Collectors; + +public class TestSgConfig { + private static final Logger log = LogManager.getLogger(TestSgConfig.class); + + private String resourceFolder = null; + private NestedValueMap overrideSgConfigSettings; + private NestedValueMap overrideUserSettings; + private NestedValueMap overrideRoleSettings; + private String indexName = ".opendistro_security"; + + public TestSgConfig() { + + } + + public TestSgConfig resources(String resourceFolder) { + this.resourceFolder = resourceFolder; + return this; + } + + public TestSgConfig sgConfigSettings(String keyPath, Object value, Object... more) { + if (overrideSgConfigSettings == null) { + overrideSgConfigSettings = new NestedValueMap(); + } + + overrideSgConfigSettings.put(Path.parse(keyPath), value); + + for (int i = 0; i < more.length - 1; i += 2) { + overrideSgConfigSettings.put(Path.parse(String.valueOf(more[i])), more[i + 1]); + } + + return this; + } + + public String getIndexName() { + return indexName; + } + + public TestSgConfig authc(AuthcDomain authcDomain) { + if (overrideSgConfigSettings == null) { + overrideSgConfigSettings = new NestedValueMap(); + } + + overrideSgConfigSettings.put(new Path("config", "dynamic", "authc"), authcDomain.toMap()); + + return this; + } + + public TestSgConfig xff(String proxies) { + if (overrideSgConfigSettings == null) { + overrideSgConfigSettings = new NestedValueMap(); + } + + overrideSgConfigSettings.put(new Path("config", "dynamic", "http", "xff"), + NestedValueMap.of("enabled", true, "internalProxies", proxies)); + + return this; + } + + public TestSgConfig user(User user) { + if (user.roleNames != null) { + return this.user(user.name, user.password, user.attributes, user.roleNames); + } else { + return this.user(user.name, user.password, user.attributes, user.roles); + } + } + + public TestSgConfig user(String name, String password, String... sgRoles) { + return user(name, password, null, sgRoles); + } + + public TestSgConfig user(String name, String password, Map attributes, String... sgRoles) { + if (overrideUserSettings == null) { + overrideUserSettings = new NestedValueMap(); + } + + overrideUserSettings.put(new Path(name, "hash"), Hasher.hash(password.toCharArray())); + + if (sgRoles != null && sgRoles.length > 0) { + overrideUserSettings.put(new Path(name, "opendistro_security_roles"), sgRoles); + } + + if (attributes != null && attributes.size() != 0) { + for (Map.Entry attr : attributes.entrySet()) { + overrideUserSettings.put(new Path(name, "attributes", attr.getKey()), attr.getValue()); + } + } + + return this; + } + + public TestSgConfig user(String name, String password, Role... sgRoles) { + return user(name, password, null, sgRoles); + } + + public TestSgConfig user(String name, String password, Map attributes, Role... sgRoles) { + if (overrideUserSettings == null) { + overrideUserSettings = new NestedValueMap(); + } + + overrideUserSettings.put(new Path(name, "hash"), Hasher.hash(password.toCharArray())); + + if (sgRoles != null && sgRoles.length > 0) { + String roleNamePrefix = "user_" + name + "__"; + + overrideUserSettings.put(new Path(name, "opendistro_security_roles"), + Arrays.asList(sgRoles).stream().map((r) -> roleNamePrefix + r.name).collect(Collectors.toList())); + roles(roleNamePrefix, sgRoles); + } + + if (attributes != null && attributes.size() != 0) { + for (Map.Entry attr : attributes.entrySet()) { + overrideUserSettings.put(new Path(name, "attributes", attr.getKey()), attr.getValue()); + } + } + + return this; + } + + public TestSgConfig roles(Role... roles) { + return roles("", roles); + } + + public TestSgConfig roles(String roleNamePrefix, Role... roles) { + if (overrideRoleSettings == null) { + overrideRoleSettings = new NestedValueMap(); + } + + for (Role role : roles) { + + String name = roleNamePrefix + role.name; + + if (role.clusterPermissions.size() > 0) { + overrideRoleSettings.put(new Path(name, "cluster_permissions"), role.clusterPermissions); + } + + if (role.indexPermissions.size() > 0) { + overrideRoleSettings.put(new Path(name, "index_permissions"), + role.indexPermissions.stream().map((p) -> p.toJsonMap()).collect(Collectors.toList())); + } + + //todo commented code - because exclude_cluster_permissions was unrecognized +// if (role.excludedClusterPermissions.size() > 0) { +// overrideRoleSettings.put(new Path(name, "exclude_cluster_permissions"), role.excludedClusterPermissions); +// } + + //todo commented code - because exclude_index_permissions was unrecognized +// if (role.excludedIndexPermissions.size() > 0) { +// overrideRoleSettings.put(new Path(name, "exclude_index_permissions"), role.excludedIndexPermissions.stream() +// .map((p) -> NestedValueMap.of("index_patterns", p.indexPatterns, "actions", p.actions)).collect(Collectors.toList())); +// } + } + + return this; + } + + public TestSgConfig authFailureListener(AuthFailureListener authFailureListener) { + if (overrideSgConfigSettings == null) { + overrideSgConfigSettings = new NestedValueMap(); + } + + overrideSgConfigSettings.put(new Path("config", "dynamic", "auth_failure_listeners"), authFailureListener.toMap()); + + return this; + } + + public TestSgConfig clone() { + TestSgConfig result = new TestSgConfig(); + + result.resourceFolder = resourceFolder; + result.indexName = indexName; + result.overrideRoleSettings = overrideRoleSettings != null ? overrideRoleSettings.clone() : null; + result.overrideSgConfigSettings = overrideSgConfigSettings != null ? overrideSgConfigSettings.clone() : null; + result.overrideUserSettings = overrideUserSettings != null ? overrideUserSettings.clone() : null; + + return result; + } + + void initIndex(Client client) { + client.admin().indices().create(new CreateIndexRequest(indexName)).actionGet(); + + writeConfigToIndex(client, CType.CONFIG, "config.yml", overrideSgConfigSettings); + writeConfigToIndex(client, CType.ROLES, "roles.yml", overrideRoleSettings); + writeConfigToIndex(client, CType.INTERNALUSERS, "internal_users.yml", overrideUserSettings); + writeConfigToIndex(client, CType.ROLESMAPPING, "roles_mapping.yml", null); + writeConfigToIndex(client, CType.ACTIONGROUPS, "action_groups.yml", null); + writeConfigToIndex(client, CType.TENANTS, "roles_tenants.yml", null); + //todo commented code +// writeConfigToIndex(client, CType.BLOCKS, "sg_blocks.yml", null); + + ConfigUpdateResponse configUpdateResponse = client + .execute(ConfigUpdateAction.INSTANCE, new ConfigUpdateRequest(CType.lcStringValues().toArray(new String[0]))).actionGet(); + + if (configUpdateResponse.hasFailures()) { + throw new RuntimeException("ConfigUpdateResponse produced failures: " + configUpdateResponse.failures()); + } + } + + private void writeConfigToIndex(Client client, CType configType, String file, NestedValueMap overrides) { + try { + NestedValueMap config; + + if (resourceFolder != null) { + config = NestedValueMap.fromYaml(openFile(file)); + } else { + config = NestedValueMap.of(new Path("_meta", "type"), configType.toLCString(), + new Path("_meta", "config_version"), 2); + } + + if (overrides != null) { + config.overrideLeafs(overrides); + } + + log.info("Writing " + configType + "\n:" + config.toJsonString()); + + client.index(new IndexRequest(indexName).id(configType.toLCString()).setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source(configType.toLCString(), BytesReference.fromByteBuffer(ByteBuffer.wrap(config.toJsonString().getBytes("utf-8"))))) + .actionGet(); + + } catch (Exception e) { + throw new RuntimeException("Error while initializing config for " + indexName, e); + } + } + + private InputStream openFile(String file) throws IOException { + + String path; + + if (resourceFolder == null || resourceFolder.length() == 0 || resourceFolder.equals("/")) { + path = "/" + file; + } else { + path = "/" + resourceFolder + "/" + file; + } + + InputStream is = FileHelper.class.getResourceAsStream(path); + + if (is == null) { + throw new FileNotFoundException("Could not find resource in class path: " + path); + } + + return is; + } + + public static NestedValueMap fromYaml(String yamlString) { + try { + return NestedValueMap.fromYaml(yamlString); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static class User { + private String name; + private String password; + private Role[] roles; + private String[] roleNames; + private Map attributes = new HashMap<>(); + + public User(String name) { + this.name = name; + this.password = "secret"; + } + + public User password(String password) { + this.password = password; + return this; + } + + public User roles(Role... roles) { + this.roles = roles; + return this; + } + + public User roles(String... roles) { + this.roleNames = roles; + return this; + } + + public User attr(String key, Object value) { + this.attributes.put(key, value); + return this; + } + + public String getName() { + return name; + } + + public String getPassword() { + return password; + } + + } + + public static class Role { + private String name; + private List clusterPermissions = new ArrayList<>(); + private List excludedClusterPermissions = new ArrayList<>(); + + private List indexPermissions = new ArrayList<>(); + private List excludedIndexPermissions = new ArrayList<>(); + + public Role(String name) { + this.name = name; + } + + public Role clusterPermissions(String... clusterPermissions) { + this.clusterPermissions.addAll(Arrays.asList(clusterPermissions)); + return this; + } + + public Role excludeClusterPermissions(String... clusterPermissions) { + this.excludedClusterPermissions.addAll(Arrays.asList(clusterPermissions)); + return this; + } + + public IndexPermission indexPermissions(String... indexPermissions) { + return new IndexPermission(this, indexPermissions); + } + + public ExcludedIndexPermission excludeIndexPermissions(String... indexPermissions) { + return new ExcludedIndexPermission(this, indexPermissions); + } + + } + + public static class IndexPermission { + private List allowedActions; + private List indexPatterns; + private Role role; + private String dlsQuery; + private List fls; + private List maskedFields; + + IndexPermission(Role role, String... allowedActions) { + this.allowedActions = Arrays.asList(allowedActions); + this.role = role; + } + + public IndexPermission dls(String dlsQuery) { + this.dlsQuery = dlsQuery; + return this; + } + + public IndexPermission fls(String... fls) { + this.fls = Arrays.asList(fls); + return this; + } + + public IndexPermission maskedFields(String... maskedFields) { + this.maskedFields = Arrays.asList(maskedFields); + return this; + } + + public Role on(String... indexPatterns) { + this.indexPatterns = Arrays.asList(indexPatterns); + this.role.indexPermissions.add(this); + return this.role; + } + + public NestedValueMap toJsonMap() { + NestedValueMap result = new NestedValueMap(); + + result.put("index_patterns", indexPatterns); + result.put("allowed_actions", allowedActions); + + if (dlsQuery != null) { + result.put("dls", dlsQuery); + } + + if (fls != null) { + result.put("fls", fls); + } + + if (maskedFields != null) { + result.put("masked_fields", maskedFields); + } + + return result; + } + + } + + public static class ExcludedIndexPermission { + private List actions; + private List indexPatterns; + private Role role; + + ExcludedIndexPermission(Role role, String... actions) { + this.actions = Arrays.asList(actions); + this.role = role; + } + + public Role on(String... indexPatterns) { + this.indexPatterns = Arrays.asList(indexPatterns); + this.role.excludedIndexPermissions.add(this); + return this.role; + } + + } + + public static class AuthcDomain { + + private final String id; + private boolean enabled = true; + private boolean transportEnabled = true; + private int order; + private List skipUsers = new ArrayList<>(); + private List enabledOnlyForIps = null; + private HttpAuthenticator httpAuthenticator; + private AuthenticationBackend authenticationBackend; + + public AuthcDomain(String id, int order) { + this.id = id; + this.order = order; + } + + public AuthcDomain httpAuthenticator(String type) { + this.httpAuthenticator = new HttpAuthenticator(type); + return this; + } + + public AuthcDomain challengingAuthenticator(String type) { + this.httpAuthenticator = new HttpAuthenticator(type).challenge(true); + return this; + } + + public AuthcDomain httpAuthenticator(HttpAuthenticator httpAuthenticator) { + this.httpAuthenticator = httpAuthenticator; + return this; + } + + public AuthcDomain backend(String type) { + this.authenticationBackend = new AuthenticationBackend(type); + return this; + } + + public AuthcDomain backend(AuthenticationBackend authenticationBackend) { + this.authenticationBackend = authenticationBackend; + return this; + } + + public AuthcDomain skipUsers(String... users) { + this.skipUsers.addAll(Arrays.asList(users)); + return this; + } + + public AuthcDomain enabledOnlyForIps(String... ips) { + if (enabledOnlyForIps == null) { + enabledOnlyForIps = new ArrayList<>(); + } + + enabledOnlyForIps.addAll(Arrays.asList(ips)); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put(new Path(id, "http_enabled"), enabled); + result.put(new Path(id, "transport_enabled"), transportEnabled); + result.put(new Path(id, "order"), order); + + if (httpAuthenticator != null) { + result.put(new Path(id, "http_authenticator"), httpAuthenticator.toMap()); + } + + if (authenticationBackend != null) { + result.put(new Path(id, "authentication_backend"), authenticationBackend.toMap()); + } + + if (enabledOnlyForIps != null) { + result.put(new Path(id, "enabled_only_for_ips"), enabledOnlyForIps); + } + + if (skipUsers != null && skipUsers.size() > 0) { + result.put(new Path(id, "skip_users"), skipUsers); + } + + return result; + } + + public static class HttpAuthenticator { + private final String type; + private boolean challenge; + private NestedValueMap config = new NestedValueMap(); + + public HttpAuthenticator(String type) { + this.type = type; + } + + public HttpAuthenticator challenge(boolean challenge) { + this.challenge = challenge; + return this; + } + + public HttpAuthenticator config(Map config) { + this.config.putAllFromAnyMap(config); + return this; + } + + public HttpAuthenticator config(String key, Object value) { + this.config.put(Path.parse(key), value); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put("type", type); + result.put("challenge", challenge); + result.put("config", config); + return result; + } + } + + public static class AuthenticationBackend { + private final String type; + private NestedValueMap config = new NestedValueMap(); + + public AuthenticationBackend(String type) { + this.type = type; + } + + public AuthenticationBackend config(Map config) { + this.config.putAllFromAnyMap(config); + return this; + } + + public AuthenticationBackend config(String key, Object value) { + this.config.put(Path.parse(key), value); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put("type", type); + result.put("config", config); + return result; + } + } + } + + public static class AuthFailureListener { + private final String id; + private final String type; + private int allowedTries; + private int timeWindowSeconds = 3600; + private int blockExpirySeconds = 600; + + public AuthFailureListener(String id, String type) { + this.id = id; + this.type = type; + this.allowedTries = 3; + } + + public AuthFailureListener(String id, String type, int allowedTries) { + this.id = id; + this.type = type; + this.allowedTries = allowedTries; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put("type", type); + result.put("allowed_tries", allowedTries); + result.put("time_window_seconds", timeWindowSeconds); + result.put("block_expiry_seconds", blockExpirySeconds); + + return NestedValueMap.of(id, result); + } + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/network/PortAllocator.java b/src/test/java/org/opensearch/security/test/helper/network/PortAllocator.java new file mode 100644 index 0000000000..0221e683b9 --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/network/PortAllocator.java @@ -0,0 +1,145 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.opensearch.security.test.helper.network; + +import org.opensearch.security.test.helper.network.SocketUtils.SocketType; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; + +public class PortAllocator { + + public static final PortAllocator TCP = new PortAllocator(SocketType.TCP, Duration.ofSeconds(100)); + public static final PortAllocator UDP = new PortAllocator(SocketType.UDP, Duration.ofSeconds(100)); + + private final SocketType socketType; + private final Duration timeoutDuration; + private final Map allocatedPorts = new HashMap<>(); + + PortAllocator(SocketType socketType, Duration timeoutDuration) { + this.socketType = socketType; + this.timeoutDuration = timeoutDuration; + } + + public SortedSet allocate(String clientName, int numRequested, int minPort) { + + int startPort = minPort; + + while (!isAvailable(startPort)) { + startPort += 10; + } + + SortedSet foundPorts = new TreeSet<>(); + + for (int currentPort = startPort; foundPorts.size() < numRequested && currentPort < SocketUtils.PORT_RANGE_MAX + && (currentPort - startPort) < 10000; currentPort++) { + if (allocate(clientName, currentPort)) { + foundPorts.add(currentPort); + } + } + + if (foundPorts.size() < numRequested) { + throw new IllegalStateException("Could not find " + numRequested + " free ports starting at " + minPort + " for " + clientName); + } + + return foundPorts; + } + + public int allocateSingle(String clientName, int minPort) { + + int startPort = minPort; + + for (int currentPort = startPort; currentPort < SocketUtils.PORT_RANGE_MAX && (currentPort - startPort) < 10000; currentPort++) { + if (allocate(clientName, currentPort)) { + return currentPort; + } + } + + throw new IllegalStateException("Could not find free port starting at " + minPort + " for " + clientName); + + } + + public void blacklist(int... ports) { + + for (int port : ports) { + allocate("blacklisted", port); + } + } + + private boolean isInUse(int port) { + boolean result = !this.socketType.isPortAvailable(port); + + if (result) { + synchronized (this) { + allocatedPorts.put(port, new AllocatedPort("external")); + } + } + + return result; + } + + private boolean isAvailable(int port) { + return !isAllocated(port) && !isInUse(port); + } + + private synchronized boolean isAllocated(int port) { + AllocatedPort allocatedPort = this.allocatedPorts.get(port); + + return allocatedPort != null && !allocatedPort.isTimedOut(); + } + + private synchronized boolean allocate(String clientName, int port) { + + AllocatedPort allocatedPort = allocatedPorts.get(port); + + if (allocatedPort != null && allocatedPort.isTimedOut()) { + allocatedPort = null; + allocatedPorts.remove(port); + } + + if (allocatedPort == null && !isInUse(port)) { + allocatedPorts.put(port, new AllocatedPort(clientName)); + return true; + } else { + return false; + } + } + + private class AllocatedPort { + final String client; + final Instant allocatedAt; + + AllocatedPort(String client) { + this.client = client; + this.allocatedAt = Instant.now(); + } + + boolean isTimedOut() { + return allocatedAt.plus(timeoutDuration).isBefore(Instant.now()); + } + + @Override + public String toString() { + return "AllocatedPort [client=" + client + ", allocatedAt=" + allocatedAt + "]"; + } + } +} diff --git a/src/test/java/org/opensearch/security/test/helper/network/SocketUtils.java b/src/test/java/org/opensearch/security/test/helper/network/SocketUtils.java index c4d344bfd3..35d07a9937 100644 --- a/src/test/java/org/opensearch/security/test/helper/network/SocketUtils.java +++ b/src/test/java/org/opensearch/security/test/helper/network/SocketUtils.java @@ -223,7 +223,7 @@ public static SortedSet findAvailableUdpPorts(int numRequested, int min } - private enum SocketType { + public enum SocketType { TCP { @Override diff --git a/src/test/java/org/opensearch/security/test/helper/rest/ClientAuthCredentials.java b/src/test/java/org/opensearch/security/test/helper/rest/ClientAuthCredentials.java new file mode 100644 index 0000000000..934389ae2e --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/rest/ClientAuthCredentials.java @@ -0,0 +1,159 @@ +package org.opensearch.security.test.helper.rest; + +import org.opensearch.security.support.PemKeyReader; + +import javax.crypto.NoSuchPaddingException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.security.InvalidAlgorithmParameterException; +import java.security.KeyException; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.security.spec.InvalidKeySpecException; + +//todo this class in SG project is not located next to tests +public class ClientAuthCredentials { + public static Builder from() { + return new Builder(); + } + + private KeyStore keyStore; + private char[] keyPassword; + private String keyAlias; + + + public KeyStore getKeyStore() { + return keyStore; + } + + public char[] getKeyPassword() { + return keyPassword; + } + + public String getKeyAlias() { + return keyAlias; + } + + public static class Builder { + private X509Certificate[] authenticationCertificate; + private PrivateKey authenticationKey; + private KeyStore keyStore; + private String keyAlias; + private String keyPassword; + + public Builder certPem(File file) throws GenericSSLConfigException { + try (FileInputStream in = new FileInputStream(file)) { + return certPem(in); + } catch (FileNotFoundException e) { + throw new GenericSSLConfigException("Could not find certificate file " + file, e); + } catch (Exception e) { + throw new GenericSSLConfigException("Error while reading certificate file " + file, e); + } + } + + public Builder certPem(Path path) throws GenericSSLConfigException { + return certPem(path.toFile()); + } + + public Builder certPem(InputStream inputStream) throws Exception { + authenticationCertificate = PemKeyReader.loadCertificatesFromStream(inputStream); + return this; + } + + public Builder certKeyPem(File file, String password) throws GenericSSLConfigException { + try (FileInputStream in = new FileInputStream(file)) { + return certKeyPem(in, password); + } catch (FileNotFoundException e) { + throw new GenericSSLConfigException("Could not find certificate key file " + file, e); + } catch (IOException e) { + throw new GenericSSLConfigException("Error while reading certificate key file " + file, e); + } + } + + public Builder certKeyPem(Path path, String password) throws GenericSSLConfigException { + return certKeyPem(path.toFile(), password); + } + + public Builder certKeyPem(InputStream inputStream, String password) throws GenericSSLConfigException { + try { + authenticationKey = PemKeyReader.toPrivateKey(inputStream, password); + } catch (IOException | InvalidAlgorithmParameterException | KeyException | NoSuchPaddingException | NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new GenericSSLConfigException("Could not load private key", e); + } + + return this; + } + + public Builder jks(File file, String alias, String password) throws GenericSSLConfigException { + return keyStore(file, alias, password, "JKS"); + } + + public Builder pkcs12(File file, String alias, String password) throws GenericSSLConfigException { + return keyStore(file, alias, password, "PKCS12"); + } + + public Builder keyStore(File file, String alias, String password) throws GenericSSLConfigException { + return keyStore(file, alias, password, null); + } + + public Builder keyStore(File file, String alias, String password, String type) throws GenericSSLConfigException { + + try { + if (type == null) { + String fileName = file.getName(); + + if (fileName.endsWith(".jks")) { + type = "JKS"; + } else if (fileName.endsWith(".pfx") || fileName.endsWith(".p12")) { + type = "PKCS12"; + } else { + throw new IllegalArgumentException("Unknwon file type: " + fileName); + } + } + + keyStore = KeyStore.getInstance(type.toUpperCase()); + keyStore.load(new FileInputStream(file), password == null ? null : password.toCharArray()); + keyAlias = alias; + keyPassword = password; + + return this; + + } catch (Exception e) { + throw new GenericSSLConfigException("Error loading client auth key store from " + file, e); + } + + } + + public ClientAuthCredentials build() throws GenericSSLConfigException { + + try { + ClientAuthCredentials result = new ClientAuthCredentials(); + + if (keyStore != null) { + result.keyStore = keyStore; + result.keyAlias = keyAlias; + result.keyPassword = keyPassword != null ? keyPassword.toCharArray() : null; + } else if (authenticationCertificate != null && authenticationKey != null) { + result.keyPassword = PemKeyReader.randomChars(12); + result.keyAlias = "al"; + result.keyStore = PemKeyReader.toKeystore(result.keyAlias, result.keyPassword, + authenticationCertificate, authenticationKey); + } else { + throw new IllegalStateException("Builder not completely initialized: " + this); + } + + return result; + } catch (Exception e) { + throw new GenericSSLConfigException("Error initializing client auth credentials", e); + } + } + + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/rest/GenericRestClient.java b/src/test/java/org/opensearch/security/test/helper/rest/GenericRestClient.java new file mode 100644 index 0000000000..7d24c13c4a --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/rest/GenericRestClient.java @@ -0,0 +1,409 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.opensearch.security.test.helper.rest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpOptions; +import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.ToXContentObject; +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.test.helper.file.FileHelper; + +import javax.net.ssl.SSLContext; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.security.KeyStore; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class GenericRestClient implements AutoCloseable { + private static final Logger log = LogManager.getLogger(RestHelper.class); + + public boolean enableHTTPClientSSL = true; + public boolean enableHTTPClientSSLv3Only = false; + private boolean sendHTTPClientCertificate = false; + public boolean trustHTTPServerCertificate = true; + private String keystore = "node-0-keystore.jks"; + public final String prefix; + private InetSocketAddress nodeHttpAddress; + private GenericSSLConfig sslConfig; + private RequestConfig requestConfig; + private List
headers = new ArrayList<>(); + private Header CONTENT_TYPE_JSON = new BasicHeader("Content-Type", "application/json"); + private boolean trackResources = false; + + private Set puttedResourcesSet = new HashSet<>(); + private List puttedResourcesList = new ArrayList<>(); + + public GenericRestClient(InetSocketAddress nodeHttpAddress, List
headers, String prefix) { + this.nodeHttpAddress = nodeHttpAddress; + this.headers.addAll(headers); + this.prefix = prefix; + } + + public GenericRestClient(InetSocketAddress nodeHttpAddress, boolean enableHTTPClientSSL, boolean trustHTTPServerCertificate, String prefix) { + this.nodeHttpAddress = nodeHttpAddress; + this.enableHTTPClientSSL = enableHTTPClientSSL; + this.trustHTTPServerCertificate = trustHTTPServerCertificate; + this.prefix = prefix; + } + + public HttpResponse get(String path, Header... headers) throws Exception { + return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse head(String path, Header... headers) throws Exception { + return executeRequest(new HttpHead(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse options(String path, Header... headers) throws Exception { + return executeRequest(new HttpOptions(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse putJson(String path, String body, Header... headers) throws Exception { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + uriRequest.setEntity(new StringEntity(body)); + + HttpResponse response = executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + + if (response.getStatusCode() < 400 && trackResources && !puttedResourcesSet.contains(path)) { + puttedResourcesSet.add(path); + puttedResourcesList.add(path); + } + + return response; + } + + public HttpResponse putJson(String path, ToXContentObject body) throws Exception { + return putJson(path, Strings.toString(body)); + } + + public HttpResponse put(String path) throws Exception { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + HttpResponse response = executeRequest(uriRequest); + + if (response.getStatusCode() < 400 && trackResources && !puttedResourcesSet.contains(path)) { + puttedResourcesSet.add(path); + puttedResourcesList.add(path); + } + + return response; + } + + public HttpResponse delete(String path, Header... headers) throws Exception { + return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse postJson(String path, String body, Header... headers) throws Exception { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + uriRequest.setEntity(new StringEntity(body)); + return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + } + + public HttpResponse postJson(String path, ToXContentObject body) throws Exception { + return postJson(path, Strings.toString(body)); + } + + public HttpResponse post(String path) throws Exception { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + return executeRequest(uriRequest); + } + + public HttpResponse patch(String path, String body) throws Exception { + HttpPatch uriRequest = new HttpPatch(getHttpServerUri() + "/" + path); + uriRequest.setEntity(new StringEntity(body)); + return executeRequest(uriRequest, CONTENT_TYPE_JSON); + } + + public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestSpecificHeaders) throws Exception { + + CloseableHttpClient httpClient = null; + try { + + httpClient = getHTTPClient(); + + if (requestSpecificHeaders != null && requestSpecificHeaders.length > 0) { + for (int i = 0; i < requestSpecificHeaders.length; i++) { + Header h = requestSpecificHeaders[i]; + uriRequest.addHeader(h); + } + } + + for (Header header : headers) { + uriRequest.addHeader(header); + } + + HttpResponse res = new HttpResponse(httpClient.execute(uriRequest)); + log.debug(res.getBody()); + return res; + } finally { + + if (httpClient != null) { + httpClient.close(); + } + } + } + + public GenericRestClient trackResources() { + trackResources = true; + return this; + } + + private void cleanupResources() { + if (puttedResourcesList.size() > 0) { + log.info("Cleaning up " + puttedResourcesList); + + for (String resource : Lists.reverse(puttedResourcesList)) { + try { + delete(resource); + } catch (Exception e) { + log.error("Error cleaning up created resources " + resource, e); + } + } + } + } + + protected final String getHttpServerUri() { + return "http" + (enableHTTPClientSSL ? "s" : "") + "://" + nodeHttpAddress.getHostString() + ":" + nodeHttpAddress.getPort(); + } + + protected final CloseableHttpClient getHTTPClient() throws Exception { + + final HttpClientBuilder hcb = HttpClients.custom(); + + if (sslConfig != null) { + hcb.setSSLSocketFactory(sslConfig.toSSLConnectionSocketFactory()); + } else if (enableHTTPClientSSL) { + + log.debug("Configure HTTP client with SSL"); + + if (prefix != null && !keystore.contains("/")) { + keystore = prefix + "/" + keystore; + } + + final String keyStorePath = FileHelper.getAbsoluteFilePathFromClassPath(keystore).toFile().getParent(); + + final KeyStore myTrustStore = KeyStore.getInstance("JKS"); + myTrustStore.load(new FileInputStream(keyStorePath + "/truststore.jks"), "changeit".toCharArray()); + + final KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(new FileInputStream(FileHelper.getAbsoluteFilePathFromClassPath(keystore).toFile()), "changeit".toCharArray()); + + final SSLContextBuilder sslContextbBuilder = SSLContexts.custom(); + + if (trustHTTPServerCertificate) { + sslContextbBuilder.loadTrustMaterial(myTrustStore, null); + } + + if (sendHTTPClientCertificate) { + sslContextbBuilder.loadKeyMaterial(keyStore, "changeit".toCharArray()); + } + + final SSLContext sslContext = sslContextbBuilder.build(); + + String[] protocols = null; + + if (enableHTTPClientSSLv3Only) { + protocols = new String[] { "SSLv3" }; + } else { + protocols = new String[] { "TLSv1", "TLSv1.1", "TLSv1.2" }; + } + + final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, protocols, null, NoopHostnameVerifier.INSTANCE); + + hcb.setSSLSocketFactory(sslsf); + } + + hcb.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(60 * 1000).build()); + + if (requestConfig != null) { + hcb.setDefaultRequestConfig(requestConfig); + } + + return hcb.build(); + } + + private Header[] mergeHeaders(Header header, Header... headers) { + + if (headers == null || headers.length == 0) { + return new Header[] { header }; + } else { + Header[] result = new Header[headers.length + 1]; + result[0] = header; + System.arraycopy(headers, 0, result, 1, headers.length); + return result; + } + } + + public static class HttpResponse { + private final CloseableHttpResponse inner; + private final String body; + private final Header[] header; + private final int statusCode; + private final String statusReason; + + public HttpResponse(CloseableHttpResponse inner) throws IllegalStateException, IOException { + super(); + this.inner = inner; + final HttpEntity entity = inner.getEntity(); + if (entity == null) { //head request does not have a entity + this.body = ""; + } else { + this.body = IOUtils.toString(entity.getContent(), StandardCharsets.UTF_8); + } + this.header = inner.getAllHeaders(); + this.statusCode = inner.getStatusLine().getStatusCode(); + this.statusReason = inner.getStatusLine().getReasonPhrase(); + inner.close(); + } + + public String getContentType() { + Header h = getInner().getFirstHeader("content-type"); + if (h != null) { + return h.getValue(); + } + return null; + } + + public boolean isJsonContentType() { + String ct = getContentType(); + if (ct == null) { + return false; + } + return ct.contains("application/json"); + } + + public CloseableHttpResponse getInner() { + return inner; + } + + public String getBody() { + return body; + } + + public Header[] getHeader() { + return header; + } + + public int getStatusCode() { + return statusCode; + } + + public String getStatusReason() { + return statusReason; + } + + public List
getHeaders() { + return header == null ? Collections.emptyList() : Arrays.asList(header); + } + + public JsonNode toJsonNode() throws JsonProcessingException, IOException { + return DefaultObjectMapper.objectMapper.readTree(getBody()); + } + + @Override + public String toString() { + return "HttpResponse [inner=" + inner + ", body=" + body + ", header=" + Arrays.toString(header) + ", statusCode=" + statusCode + + ", statusReason=" + statusReason + "]"; + } + + } + + public GenericSSLConfig getSslConfig() { + return sslConfig; + } + + public void setSslConfig(GenericSSLConfig sslConfig) { + this.sslConfig = sslConfig; + } + + @Override + public String toString() { + return "RestHelper [server=" + getHttpServerUri() + ", node=" + nodeHttpAddress + ", sslConfig=" + sslConfig + "]"; + } + + public RequestConfig getRequestConfig() { + return requestConfig; + } + + public void setRequestConfig(RequestConfig requestConfig) { + this.requestConfig = requestConfig; + } + + public void setLocalAddress(InetAddress inetAddress) { + if (requestConfig == null) { + requestConfig = RequestConfig.custom().setLocalAddress(inetAddress).build(); + } else { + requestConfig = RequestConfig.copy(requestConfig).setLocalAddress(inetAddress).build(); + } + } + + @Override + public void close() throws IOException { + cleanupResources(); + } + + public boolean isSendHTTPClientCertificate() { + return sendHTTPClientCertificate; + } + + public void setSendHTTPClientCertificate(boolean sendHTTPClientCertificate) { + this.sendHTTPClientCertificate = sendHTTPClientCertificate; + } + + public String getKeystore() { + return keystore; + } + + public void setKeystore(String keystore) { + this.keystore = keystore; + } +} diff --git a/src/test/java/org/opensearch/security/test/helper/rest/GenericSSLConfig.java b/src/test/java/org/opensearch/security/test/helper/rest/GenericSSLConfig.java new file mode 100644 index 0000000000..a78ca91d35 --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/rest/GenericSSLConfig.java @@ -0,0 +1,258 @@ +package org.opensearch.security.test.helper.rest; + +import com.google.common.collect.ImmutableList; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.ssl.PrivateKeyDetails; +import org.apache.http.ssl.PrivateKeyStrategy; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; + +import javax.net.ssl.*; +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; +import java.security.*; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +//todo this class in SG project is not located next to tests +public class GenericSSLConfig { + private static final List DEFAULT_TLS_PROTOCOLS = ImmutableList.of("TLSv1.2", "TLSv1.1"); + + private String[] enabledProtocols; + private String[] enabledCiphers; + private HostnameVerifier hostnameVerifier; + private boolean hostnameVerificationEnabled; + private boolean trustAll; + private SSLContext sslContext; + + public SSLContext getUnrestrictedSslContext() { + return sslContext; + } + + public RestrictingSSLSocketFactory getRestrictedSSLSocketFactory() { + return new RestrictingSSLSocketFactory(sslContext.getSocketFactory(), enabledProtocols, enabledCiphers); + } + + public SSLIOSessionStrategy toSSLIOSessionStrategy() { + return new SSLIOSessionStrategy(sslContext, enabledProtocols, enabledCiphers, hostnameVerifier); + } + + public SSLConnectionSocketFactory toSSLConnectionSocketFactory() { + return new SSLConnectionSocketFactory(sslContext, enabledProtocols, enabledCiphers, hostnameVerifier); + } + + public static class Builder { + private GenericSSLConfig result = new GenericSSLConfig(); + private ClientAuthCredentials clientAuthCredentials; + private TrustStore trustStore; + private String clientName; + + public Builder clientName(String clientName) { + this.clientName = clientName; + return this; + } + + public Builder verifyHostnames(boolean hostnameVerificationEnabled) { + result.hostnameVerificationEnabled = hostnameVerificationEnabled; + return this; + } + + public Builder trustAll(boolean trustAll) { + result.trustAll = trustAll; + return this; + } + + public Builder useCiphers(String... enabledCiphers) { + result.enabledCiphers = enabledCiphers; + return this; + } + + public Builder useProtocols(String... enabledProtocols) { + result.enabledProtocols = enabledProtocols; + return this; + } + + public Builder useClientAuth(ClientAuthCredentials clientAuthCredentials) { + this.clientAuthCredentials = clientAuthCredentials; + return this; + } + + public Builder useTrustStore(TrustStore trustStore) { + this.trustStore = trustStore; + return this; + } + + public GenericSSLConfig build() throws GenericSSLConfigException { + if (result.hostnameVerificationEnabled) { + result.hostnameVerifier = new DefaultHostnameVerifier(); + } else { + result.hostnameVerifier = NoopHostnameVerifier.INSTANCE; + } + + if (result.enabledProtocols == null) { + result.enabledProtocols = DEFAULT_TLS_PROTOCOLS.toArray(new String[0]); + } + + result.sslContext = buildSSLContext(); + + return result; + } + + public SSLIOSessionStrategy toSSLIOSessionStrategy() throws GenericSSLConfigException { + return build().toSSLIOSessionStrategy(); + } + + public SSLConnectionSocketFactory toSSLConnectionSocketFactory() throws GenericSSLConfigException { + return build().toSSLConnectionSocketFactory(); + } + + SSLContext buildSSLContext() throws GenericSSLConfigException { + try { + SSLContextBuilder sslContextBuilder; + + if (result.trustAll) { + sslContextBuilder = new OverlyTrustfulSSLContextBuilder(); + } else { + sslContextBuilder = SSLContexts.custom(); + } + + if (trustStore != null) { + sslContextBuilder.loadTrustMaterial(trustStore.getKeyStore(), null); + } + + if (clientAuthCredentials != null) { + sslContextBuilder.loadKeyMaterial(clientAuthCredentials.getKeyStore(), clientAuthCredentials.getKeyPassword(), + new PrivateKeySelector(clientAuthCredentials.getKeyAlias())); + + } + + return sslContextBuilder.build(); + + } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException | UnrecoverableKeyException e) { + throw new GenericSSLConfigException("Error while initializing SSL configuration for " + this.clientName, e); + } + } + + } + + private static class OverlyTrustfulSSLContextBuilder extends SSLContextBuilder { + @Override + protected void initSSLContext(SSLContext sslContext, Collection keyManagers, Collection trustManagers, + SecureRandom secureRandom) throws KeyManagementException { + sslContext.init(!keyManagers.isEmpty() ? keyManagers.toArray(new KeyManager[keyManagers.size()]) : null, + new TrustManager[] { new OverlyTrustfulTrustManager() }, secureRandom); + } + } + + private static class OverlyTrustfulTrustManager implements X509TrustManager { + @Override + public void checkClientTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + } + + @Override + public void checkServerTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } + + private static class PrivateKeySelector implements PrivateKeyStrategy { + + private final String effectiveKeyAlias; + + PrivateKeySelector(String effectiveKeyAlias) { + this.effectiveKeyAlias = effectiveKeyAlias; + } + + @Override + public String chooseAlias(Map aliases, Socket socket) { + if (aliases == null || aliases.isEmpty()) { + return effectiveKeyAlias; + } + + if (effectiveKeyAlias == null || effectiveKeyAlias.isEmpty()) { + return aliases.keySet().iterator().next(); + } + + return effectiveKeyAlias; + } + } + + private static class RestrictingSSLSocketFactory extends SSLSocketFactory { + + private final SSLSocketFactory delegate; + private final String[] enabledProtocols; + private final String[] enabledCipherSuites; + + public RestrictingSSLSocketFactory(final SSLSocketFactory delegate, final String[] enabledProtocols, final String[] enabledCipherSuites) { + this.delegate = delegate; + this.enabledProtocols = enabledProtocols; + this.enabledCipherSuites = enabledCipherSuites; + } + + @Override + public String[] getDefaultCipherSuites() { + return enabledCipherSuites == null ? delegate.getDefaultCipherSuites() : enabledCipherSuites; + } + + @Override + public String[] getSupportedCipherSuites() { + return enabledCipherSuites == null ? delegate.getSupportedCipherSuites() : enabledCipherSuites; + } + + @Override + public Socket createSocket() throws IOException { + return enforce(delegate.createSocket()); + } + + @Override + public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException { + return enforce(delegate.createSocket(s, host, port, autoClose)); + } + + @Override + public Socket createSocket(String host, int port) throws IOException, UnknownHostException { + return enforce(delegate.createSocket(host, port)); + } + + @Override + public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException, UnknownHostException { + return enforce(delegate.createSocket(host, port, localHost, localPort)); + } + + @Override + public Socket createSocket(InetAddress host, int port) throws IOException { + return enforce(delegate.createSocket(host, port)); + } + + @Override + public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException { + return enforce(delegate.createSocket(address, port, localAddress, localPort)); + } + + private Socket enforce(Socket socket) { + if (socket != null && (socket instanceof SSLSocket)) { + + if (enabledProtocols != null) { + ((SSLSocket) socket).setEnabledProtocols(enabledProtocols); + } + + if (enabledCipherSuites != null) { + ((SSLSocket) socket).setEnabledCipherSuites(enabledCipherSuites); + } + } + return socket; + } + } +} diff --git a/src/test/java/org/opensearch/security/test/helper/rest/GenericSSLConfigException.java b/src/test/java/org/opensearch/security/test/helper/rest/GenericSSLConfigException.java new file mode 100644 index 0000000000..d9fdf922f6 --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/rest/GenericSSLConfigException.java @@ -0,0 +1,28 @@ +package org.opensearch.security.test.helper.rest; + +//todo this class in SG project is not located next to tests +public class GenericSSLConfigException extends Exception { + + private static final long serialVersionUID = 3774103067927533078L; + + public GenericSSLConfigException() { + super(); + } + + public GenericSSLConfigException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public GenericSSLConfigException(String message, Throwable cause) { + super(message, cause); + } + + public GenericSSLConfigException(String message) { + super(message); + } + + public GenericSSLConfigException(Throwable cause) { + super(cause); + } + +} diff --git a/src/test/java/org/opensearch/security/test/helper/rest/TrustStore.java b/src/test/java/org/opensearch/security/test/helper/rest/TrustStore.java new file mode 100644 index 0000000000..fd78390467 --- /dev/null +++ b/src/test/java/org/opensearch/security/test/helper/rest/TrustStore.java @@ -0,0 +1,113 @@ +package org.opensearch.security.test.helper.rest; + +import org.opensearch.security.support.PemKeyReader; + +import java.io.*; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.cert.X509Certificate; + +//todo this class in SG project is not located next to tests +public class TrustStore { + public static Builder from() { + return new Builder(); + } + + private KeyStore keyStore; + private char[] keyPassword; + private String keyAlias; + + public KeyStore getKeyStore() { + return keyStore; + } + + public char[] getKeyPassword() { + return keyPassword; + } + + public String getKeyAlias() { + return keyAlias; + } + + public static class Builder { + private X509Certificate[] certificates; + private KeyStore keyStore; + + public Builder certPem(File file) throws GenericSSLConfigException { + try (FileInputStream in = new FileInputStream(file)) { + return certPem(in); + } catch (FileNotFoundException e) { + throw new GenericSSLConfigException("Could not find certificate file " + file, e); + } catch (Exception e) { + throw new GenericSSLConfigException("Error while reading certificate file " + file, e); + } + } + + public Builder certPem(Path path) throws GenericSSLConfigException { + return certPem(path.toFile()); + } + + public Builder certPem(InputStream inputStream) throws Exception { + certificates = PemKeyReader.loadCertificatesFromStream(inputStream); + return this; + } + + public Builder jks(File file, String password) throws GenericSSLConfigException { + return keyStore(file, password, "JKS"); + } + + public Builder pkcs12(File file, String password) throws GenericSSLConfigException { + return keyStore(file, password, "PKCS12"); + } + + public Builder keyStore(File file, String password) throws GenericSSLConfigException { + return keyStore(file, password, null); + } + + public Builder keyStore(File file, String password, String type) throws GenericSSLConfigException { + + try { + if (type == null) { + String fileName = file.getName(); + + if (fileName.endsWith(".jks")) { + type = "JKS"; + } else if (fileName.endsWith(".pfx") || fileName.endsWith(".p12")) { + type = "PKCS12"; + } else { + throw new IllegalArgumentException("Unknwon file type: " + fileName); + } + } + + keyStore = KeyStore.getInstance(type.toUpperCase()); + keyStore.load(new FileInputStream(file), password == null ? null : password.toCharArray()); + + return this; + + } catch (Exception e) { + throw new GenericSSLConfigException("Error loading client auth key store from " + file, e); + } + + } + + public TrustStore build() throws GenericSSLConfigException { + + try { + TrustStore result = new TrustStore(); + + if (keyStore != null) { + result.keyStore = keyStore; + } else if (certificates != null) { + result.keyStore = PemKeyReader.toTruststore("al", certificates); + } else { + throw new IllegalStateException("Builder not completely initialized: " + this); + } + + return result; + } catch (Exception e) { + throw new GenericSSLConfigException("Error initializing client auth credentials", e); + } + } + + } +}