diff --git a/docs/reference/rest-api/security/query-api-key.asciidoc b/docs/reference/rest-api/security/query-api-key.asciidoc index 0e5973a010a47..67b0b7bfac58d 100644 --- a/docs/reference/rest-api/security/query-api-key.asciidoc +++ b/docs/reference/rest-api/security/query-api-key.asciidoc @@ -64,6 +64,11 @@ You can query the following public values associated with an API key. `id`:: ID of the API key. Note `id` must be queried with the <> query. +`type`:: +API keys can be of type `rest`, if created via the <> or +the <> APIs, or of type `cross_cluster` if created via +the <> API. + `name`:: Name of the API key. diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java index f79077ae3a550..18d9dcdc822e5 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java @@ -9,8 +9,10 @@ import org.apache.http.HttpHeaders; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.XContentTestUtils; @@ -21,6 +23,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; @@ -43,6 +46,8 @@ public class QueryApiKeyIT extends SecurityInBasicRestTestCase { private static final String API_KEY_ADMIN_AUTH_HEADER = "Basic YXBpX2tleV9hZG1pbjpzZWN1cml0eS10ZXN0LXBhc3N3b3Jk"; private static final String API_KEY_USER_AUTH_HEADER = "Basic YXBpX2tleV91c2VyOnNlY3VyaXR5LXRlc3QtcGFzc3dvcmQ="; private static final String TEST_USER_AUTH_HEADER = "Basic c2VjdXJpdHlfdGVzdF91c2VyOnNlY3VyaXR5LXRlc3QtcGFzc3dvcmQ="; + private static final String SYSTEM_WRITE_ROLE_NAME = "system_write"; + private static final String SUPERUSER_WITH_SYSTEM_WRITE = "superuser_with_system_write"; public void testQuery() throws IOException { createApiKeys(); @@ -297,6 +302,71 @@ public void testPagination() throws IOException, InterruptedException { assertThat(responseMap2.get("count"), equalTo(0)); } + public void testTypeField() throws Exception { + final List allApiKeyIds = new ArrayList<>(7); + for (int i = 0; i < 7; i++) { + allApiKeyIds.add( + createApiKey("typed_key_" + i, Map.of(), randomFrom(API_KEY_ADMIN_AUTH_HEADER, API_KEY_USER_AUTH_HEADER)).v1() + ); + } + List apiKeyIdsSubset = randomSubsetOf(allApiKeyIds); + List apiKeyIdsSubsetDifference = new ArrayList<>(allApiKeyIds); + apiKeyIdsSubsetDifference.removeAll(apiKeyIdsSubset); + + List apiKeyRestTypeQueries = List.of(""" + {"query": {"term": {"type": "rest" }}}""", """ + {"query": {"bool": {"must_not": [{"term": {"type": "cross_cluster"}}, {"term": {"type": "other"}}]}}}""", """ + {"query": {"prefix": {"type": "re" }}}""", """ + {"query": {"wildcard": {"type": "r*t" }}}""", """ + {"query": {"range": {"type": {"gte": "raaa", "lte": "rzzz"}}}}"""); + + for (String query : apiKeyRestTypeQueries) { + assertQuery(API_KEY_ADMIN_AUTH_HEADER, query, apiKeys -> { + assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(allApiKeyIds.toArray(new String[0])) + ); + }); + } + + createSystemWriteRole(SYSTEM_WRITE_ROLE_NAME); + String systemWriteCreds = createUser(SUPERUSER_WITH_SYSTEM_WRITE, new String[] { "superuser", SYSTEM_WRITE_ROLE_NAME }); + + // test keys with no "type" field are still considered of type "rest" + // this is so in order to accommodate pre-8.9 API keys which where all of type "rest" implicitly + updateApiKeys(systemWriteCreds, "ctx._source.remove('type');", apiKeyIdsSubset); + for (String query : apiKeyRestTypeQueries) { + assertQuery(API_KEY_ADMIN_AUTH_HEADER, query, apiKeys -> { + assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(allApiKeyIds.toArray(new String[0])) + ); + }); + } + + // but the same keys with type "other" are NOT of type "rest" + updateApiKeys(systemWriteCreds, "ctx._source['type']='other';", apiKeyIdsSubset); + for (String query : apiKeyRestTypeQueries) { + assertQuery(API_KEY_ADMIN_AUTH_HEADER, query, apiKeys -> { + assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(apiKeyIdsSubsetDifference.toArray(new String[0])) + ); + }); + } + // the complement set is not of type "rest" if it is "cross_cluster" + updateApiKeys(systemWriteCreds, "ctx._source['type']='rest';", apiKeyIdsSubset); + updateApiKeys(systemWriteCreds, "ctx._source['type']='cross_cluster';", apiKeyIdsSubsetDifference); + for (String query : apiKeyRestTypeQueries) { + assertQuery(API_KEY_ADMIN_AUTH_HEADER, query, apiKeys -> { + assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(apiKeyIdsSubset.toArray(new String[0])) + ); + }); + } + } + @SuppressWarnings("unchecked") public void testSort() throws IOException { final String authHeader = randomFrom(API_KEY_ADMIN_AUTH_HEADER, API_KEY_USER_AUTH_HEADER); @@ -598,10 +668,73 @@ private String createAndInvalidateApiKey(String name, String authHeader) throws return tuple.v1(); } - private void createUser(String name) throws IOException { - final Request request = new Request("POST", "/_security/user/" + name); - request.setJsonEntity(""" - {"password":"super-strong-password","roles":[]}"""); - assertOK(adminClient().performRequest(request)); + private String createUser(String username) throws IOException { + return createUser(username, new String[0]); + } + + private String createUser(String username, String[] roles) throws IOException { + final Request request = new Request("POST", "/_security/user/" + username); + Map body = Map.ofEntries(Map.entry("roles", roles), Map.entry("password", "super-strong-password".toString())); + request.setJsonEntity(XContentTestUtils.convertToXContent(body, XContentType.JSON).utf8ToString()); + Response response = adminClient().performRequest(request); + assertOK(response); + return basicAuthHeaderValue(username, new SecureString("super-strong-password".toCharArray())); + } + + private void createSystemWriteRole(String roleName) throws IOException { + final Request addRole = new Request("POST", "/_security/role/" + roleName); + addRole.setJsonEntity(""" + { + "indices": [ + { + "names": [ "*" ], + "privileges": ["all"], + "allow_restricted_indices" : true + } + ] + }"""); + Response response = adminClient().performRequest(addRole); + assertOK(response); + } + + private void expectWarnings(Request request, String... expectedWarnings) { + final Set expected = Set.of(expectedWarnings); + RequestOptions options = request.getOptions().toBuilder().setWarningsHandler(warnings -> { + final Set actual = Set.copyOf(warnings); + // Return true if the warnings aren't what we expected; the client will treat them as a fatal error. + return actual.equals(expected) == false; + }).build(); + request.setOptions(options); + } + + private void updateApiKeys(String creds, String script, Collection ids) throws IOException { + if (ids.isEmpty()) { + return; + } + final Request request = new Request("POST", "/.security/_update_by_query?refresh=true&wait_for_completion=true"); + request.setJsonEntity(Strings.format(""" + { + "script": { + "source": "%s", + "lang": "painless" + }, + "query": { + "bool": { + "must": [ + {"term": {"doc_type": "api_key"}}, + {"ids": {"values": %s}} + ] + } + } + } + """, script, ids.stream().map(id -> "\"" + id + "\"").collect(Collectors.toList()))); + request.setOptions(request.getOptions().toBuilder().addHeader(HttpHeaders.AUTHORIZATION, creds)); + expectWarnings( + request, + "this request accesses system indices: [.security-7]," + + " but in a future major version, direct access to system indices will be prevented by default" + ); + Response response = client().performRequest(request); + assertOK(response); } } diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 6c4aaeada74c7..0d5a757f65084 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -35,8 +35,10 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -54,9 +56,11 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.iterableWithSize; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -703,6 +707,73 @@ public void testRemoteIndicesSupportForApiKeys() throws IOException { } + @SuppressWarnings("unchecked") + public void testQueryCrossClusterApiKeysByType() throws IOException { + final List apiKeyIds = new ArrayList<>(3); + for (int i = 0; i < randomIntBetween(3, 5); i++) { + Request createRequest = new Request("POST", "/_security/cross_cluster/api_key"); + createRequest.setJsonEntity(Strings.format(""" + { + "name": "test-cross-key-query-%d", + "access": { + "search": [ + { + "names": [ "whatever" ] + } + ] + }, + "metadata": { "tag": %d, "label": "rest" } + }""", i, i)); + setUserForRequest(createRequest, MANAGE_SECURITY_USER, END_USER_PASSWORD); + ObjectPath createResponse = assertOKAndCreateObjectPath(client().performRequest(createRequest)); + apiKeyIds.add(createResponse.evaluate("id")); + } + // the "cross_cluster" keys are not "rest" type + for (String restTypeQuery : List.of(""" + {"query": {"term": {"type": "rest" }}}""", """ + {"query": {"bool": {"must_not": {"term": {"type": "cross_cluster"}}}}}""", """ + {"query": {"prefix": {"type": "re" }}}""", """ + {"query": {"wildcard": {"type": "r*t" }}}""", """ + {"query": {"range": {"type": {"gte": "raaa", "lte": "rzzz"}}}}""")) { + Request queryRequest = new Request("GET", "/_security/_query/api_key"); + queryRequest.addParameter("with_limited_by", String.valueOf(randomBoolean())); + queryRequest.setJsonEntity(restTypeQuery); + setUserForRequest(queryRequest, MANAGE_API_KEY_USER, END_USER_PASSWORD); + ObjectPath queryResponse = assertOKAndCreateObjectPath(client().performRequest(queryRequest)); + assertThat(queryResponse.evaluate("total"), is(0)); + assertThat(queryResponse.evaluate("count"), is(0)); + assertThat(queryResponse.evaluate("api_keys"), iterableWithSize(0)); + } + for (String crossClusterTypeQuery : List.of(""" + {"query": {"term": {"type": "cross_cluster" }}}""", """ + {"query": {"bool": {"must_not": {"term": {"type": "rest"}}}}}""", """ + {"query": {"prefix": {"type": "cro" }}}""", """ + {"query": {"wildcard": {"type": "*oss_*er" }}}""", """ + {"query": {"range": {"type": {"gte": "cross", "lte": "zzzz"}}}}""")) { + Request queryRequest = new Request("GET", "/_security/_query/api_key"); + queryRequest.addParameter("with_limited_by", String.valueOf(randomBoolean())); + queryRequest.setJsonEntity(crossClusterTypeQuery); + setUserForRequest(queryRequest, MANAGE_API_KEY_USER, END_USER_PASSWORD); + ObjectPath queryResponse = assertOKAndCreateObjectPath(client().performRequest(queryRequest)); + assertThat(queryResponse.evaluate("total"), is(apiKeyIds.size())); + assertThat(queryResponse.evaluate("count"), is(apiKeyIds.size())); + assertThat(queryResponse.evaluate("api_keys"), iterableWithSize(apiKeyIds.size())); + Iterator apiKeys = ((List) queryResponse.evaluate("api_keys")).iterator(); + while (apiKeys.hasNext()) { + assertThat(apiKeyIds, hasItem((String) ((Map) apiKeys.next()).get("id"))); + } + } + final Request queryRequest = new Request("GET", "/_security/_query/api_key"); + queryRequest.addParameter("with_limited_by", String.valueOf(randomBoolean())); + queryRequest.setJsonEntity(""" + {"query": {"bool": {"must": [{"term": {"type": "cross_cluster" }}, {"term": {"metadata.tag": 2}}]}}}"""); + setUserForRequest(queryRequest, MANAGE_API_KEY_USER, END_USER_PASSWORD); + final ObjectPath queryResponse = assertOKAndCreateObjectPath(client().performRequest(queryRequest)); + assertThat(queryResponse.evaluate("total"), is(1)); + assertThat(queryResponse.evaluate("count"), is(1)); + assertThat(queryResponse.evaluate("api_keys.0.name"), is("test-cross-key-query-2")); + } + public void testCreateCrossClusterApiKey() throws IOException { final Request createRequest = new Request("POST", "/_security/cross_cluster/api_key"); createRequest.setJsonEntity(""" diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java index 4077597a7ef16..b9961e6735c7e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java @@ -27,11 +27,26 @@ import org.elasticsearch.xpack.security.support.ApiKeyFieldNameTranslators; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; public final class TransportQueryApiKeyAction extends HandledTransportAction { + // API keys with no "type" field are implicitly of type "rest" (this is the case for all API Keys created before v8.9). + // The below runtime field ensures that the "type" field can be used by the {@link RestQueryApiKeyAction}, + // while making the implicit "rest" type feature transparent to the caller (hence all keys are either "rest" + // or "cross_cluster", and the "type" is always set). + // This can be improved, to get rid of the runtime performance impact of the runtime field, by reindexing + // the api key docs and setting the "type" to "rest" if empty. But the infrastructure to run such a maintenance + // task on a system index (once the cluster version permits) is not currently available. + public static final String API_KEY_TYPE_RUNTIME_MAPPING_FIELD = "runtime_key_type"; + private static final Map API_KEY_TYPE_RUNTIME_MAPPING = Map.of( + API_KEY_TYPE_RUNTIME_MAPPING_FIELD, + Map.of("type", "keyword", "script", Map.of("source", "emit(field('type').get(\"rest\"));")) + ); + private final ApiKeyService apiKeyService; private final SecurityContext securityContext; @@ -66,12 +81,19 @@ protected void doExecute(Task task, QueryApiKeyRequest request, ActionListener { + if (API_KEY_TYPE_RUNTIME_MAPPING_FIELD.equals(fieldName)) { + accessesApiKeyTypeField.set(true); + } + }, request.isFilterForCurrentUser() ? authentication : null); searchSourceBuilder.query(apiKeyBoolQueryBuilder); + // only add the query-level runtime field to the search request if it's actually referring the "type" field + if (accessesApiKeyTypeField.get()) { + searchSourceBuilder.runtimeMappings(API_KEY_TYPE_RUNTIME_MAPPING); + } + if (request.getFieldSortBuilders() != null) { translateFieldSortBuilders(request.getFieldSortBuilders(), searchSourceBuilder); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java index 28ecd5ffe5b57..5cb6573c8b5dc 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java @@ -28,6 +28,9 @@ import java.io.IOException; import java.util.Set; +import java.util.function.Consumer; + +import static org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD; public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder { @@ -36,10 +39,14 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder { "_id", "doc_type", "name", + "type", + API_KEY_TYPE_RUNTIME_MAPPING_FIELD, "api_key_invalidated", "invalidation_time", "creation_time", - "expiration_time" + "expiration_time", + "creator.principal", + "creator.realm" ); private ApiKeyBoolQueryBuilder() {} @@ -56,17 +63,23 @@ private ApiKeyBoolQueryBuilder() {} * * @param queryBuilder This represents the query parsed directly from the user input. It is validated * and transformed (see above). + * @param fieldNameVisitor This {@code Consumer} is invoked with all the (index-level) field names referred to in the passed-in query. * @param authentication The user's authentication object. If present, it will be used to filter the results * to only include API keys owned by the user. * @return A specialised query builder for API keys that is safe to run on the security index. */ - public static ApiKeyBoolQueryBuilder build(QueryBuilder queryBuilder, @Nullable Authentication authentication) { + public static ApiKeyBoolQueryBuilder build( + QueryBuilder queryBuilder, + Consumer fieldNameVisitor, + @Nullable Authentication authentication + ) { final ApiKeyBoolQueryBuilder finalQuery = new ApiKeyBoolQueryBuilder(); if (queryBuilder != null) { - QueryBuilder processedQuery = doProcess(queryBuilder); + QueryBuilder processedQuery = doProcess(queryBuilder, fieldNameVisitor); finalQuery.must(processedQuery); } finalQuery.filter(QueryBuilders.termQuery("doc_type", "api_key")); + fieldNameVisitor.accept("doc_type"); if (authentication != null) { if (authentication.isApiKey()) { @@ -77,8 +90,10 @@ public static ApiKeyBoolQueryBuilder build(QueryBuilder queryBuilder, @Nullable finalQuery.filter(QueryBuilders.idsQuery().addIds(apiKeyId)); } else { finalQuery.filter(QueryBuilders.termQuery("creator.principal", authentication.getEffectiveSubject().getUser().principal())); + fieldNameVisitor.accept("creator.principal"); final String[] realms = ApiKeyService.getOwnersRealmNames(authentication); final QueryBuilder realmsQuery = ApiKeyService.filterForRealmNames(realms); + fieldNameVisitor.accept("creator.realm"); assert realmsQuery != null; finalQuery.filter(realmsQuery); } @@ -86,15 +101,15 @@ public static ApiKeyBoolQueryBuilder build(QueryBuilder queryBuilder, @Nullable return finalQuery; } - private static QueryBuilder doProcess(QueryBuilder qb) { + private static QueryBuilder doProcess(QueryBuilder qb, Consumer fieldNameVisitor) { if (qb instanceof final BoolQueryBuilder query) { final BoolQueryBuilder newQuery = QueryBuilders.boolQuery() .minimumShouldMatch(query.minimumShouldMatch()) .adjustPureNegative(query.adjustPureNegative()); - query.must().stream().map(ApiKeyBoolQueryBuilder::doProcess).forEach(newQuery::must); - query.should().stream().map(ApiKeyBoolQueryBuilder::doProcess).forEach(newQuery::should); - query.mustNot().stream().map(ApiKeyBoolQueryBuilder::doProcess).forEach(newQuery::mustNot); - query.filter().stream().map(ApiKeyBoolQueryBuilder::doProcess).forEach(newQuery::filter); + query.must().stream().map(q -> ApiKeyBoolQueryBuilder.doProcess(q, fieldNameVisitor)).forEach(newQuery::must); + query.should().stream().map(q -> ApiKeyBoolQueryBuilder.doProcess(q, fieldNameVisitor)).forEach(newQuery::should); + query.mustNot().stream().map(q -> ApiKeyBoolQueryBuilder.doProcess(q, fieldNameVisitor)).forEach(newQuery::mustNot); + query.filter().stream().map(q -> ApiKeyBoolQueryBuilder.doProcess(q, fieldNameVisitor)).forEach(newQuery::filter); return newQuery; } else if (qb instanceof MatchAllQueryBuilder) { return qb; @@ -102,29 +117,35 @@ private static QueryBuilder doProcess(QueryBuilder qb) { return qb; } else if (qb instanceof final TermQueryBuilder query) { final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName()); + fieldNameVisitor.accept(translatedFieldName); return QueryBuilders.termQuery(translatedFieldName, query.value()).caseInsensitive(query.caseInsensitive()); } else if (qb instanceof final ExistsQueryBuilder query) { final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName()); + fieldNameVisitor.accept(translatedFieldName); return QueryBuilders.existsQuery(translatedFieldName); } else if (qb instanceof final TermsQueryBuilder query) { if (query.termsLookup() != null) { throw new IllegalArgumentException("terms query with terms lookup is not supported for API Key query"); } final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName()); + fieldNameVisitor.accept(translatedFieldName); return QueryBuilders.termsQuery(translatedFieldName, query.getValues()); } else if (qb instanceof final PrefixQueryBuilder query) { final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName()); + fieldNameVisitor.accept(translatedFieldName); return QueryBuilders.prefixQuery(translatedFieldName, query.value()).caseInsensitive(query.caseInsensitive()); } else if (qb instanceof final WildcardQueryBuilder query) { final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName()); + fieldNameVisitor.accept(translatedFieldName); return QueryBuilders.wildcardQuery(translatedFieldName, query.value()) .caseInsensitive(query.caseInsensitive()) .rewrite(query.rewrite()); } else if (qb instanceof final RangeQueryBuilder query) { - final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName()); if (query.relation() != null) { throw new IllegalArgumentException("range query with relation is not supported for API Key query"); } + final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName()); + fieldNameVisitor.accept(translatedFieldName); final RangeQueryBuilder newQuery = QueryBuilders.rangeQuery(translatedFieldName); if (query.format() != null) { newQuery.format(query.format()); @@ -159,9 +180,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws } static boolean isIndexFieldNameAllowed(String fieldName) { - return ALLOWED_EXACT_INDEX_FIELD_NAMES.contains(fieldName) - || fieldName.startsWith("metadata_flattened.") - || fieldName.startsWith("creator."); + return ALLOWED_EXACT_INDEX_FIELD_NAMES.contains(fieldName) || fieldName.startsWith("metadata_flattened."); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java index 4d7cc9d978cd4..c204ec031b18c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java @@ -10,6 +10,8 @@ import java.util.List; import java.util.function.Function; +import static org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD; + /** * A class to translate query level field names to index level field names. */ @@ -21,6 +23,7 @@ public class ApiKeyFieldNameTranslators { new ExactFieldNameTranslator(s -> "creator.principal", "username"), new ExactFieldNameTranslator(s -> "creator.realm", "realm_name"), new ExactFieldNameTranslator(Function.identity(), "name"), + new ExactFieldNameTranslator(s -> API_KEY_TYPE_RUNTIME_MAPPING_FIELD, "type"), new ExactFieldNameTranslator(s -> "creation_time", "creation"), new ExactFieldNameTranslator(s -> "expiration_time", "expiration"), new ExactFieldNameTranslator(s -> "api_key_invalidated", "invalidated"), diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java index 477409f22369f..235657a30e11f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java @@ -29,11 +29,13 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationTests; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction; import org.elasticsearch.xpack.security.authc.ApiKeyService; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; @@ -57,7 +59,9 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase { public void testBuildFromSimpleQuery() { final Authentication authentication = randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null; final QueryBuilder q1 = randomSimpleQuery("name"); - final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, authentication); + final List queryFields = new ArrayList<>(); + final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, queryFields::add, authentication); + assertQueryFields(queryFields, q1, authentication); assertCommonFilterQueries(apiKeyQb1, authentication); final List mustQueries = apiKeyQb1.must(); assertThat(mustQueries, hasSize(1)); @@ -69,7 +73,9 @@ public void testBuildFromSimpleQuery() { public void testQueryForDomainAuthentication() { final Authentication authentication = AuthenticationTests.randomAuthentication(null, AuthenticationTests.randomRealmRef(true)); final QueryBuilder query = randomSimpleQuery("name"); - final ApiKeyBoolQueryBuilder apiKeysQuery = ApiKeyBoolQueryBuilder.build(query, authentication); + final List queryFields = new ArrayList<>(); + final ApiKeyBoolQueryBuilder apiKeysQuery = ApiKeyBoolQueryBuilder.build(query, queryFields::add, authentication); + assertQueryFields(queryFields, query, authentication); assertThat(apiKeysQuery.filter().get(0), is(QueryBuilders.termQuery("doc_type", "api_key"))); assertThat( apiKeysQuery.filter().get(1), @@ -102,18 +108,23 @@ public void testQueryForDomainAuthentication() { public void testBuildFromBoolQuery() { final Authentication authentication = randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null; + final List queryFields = new ArrayList<>(); final BoolQueryBuilder bq1 = QueryBuilders.boolQuery(); + boolean accessesNameField = false; if (randomBoolean()) { bq1.must(QueryBuilders.prefixQuery("name", "prod-")); + accessesNameField = true; } if (randomBoolean()) { bq1.should(QueryBuilders.wildcardQuery("name", "*-east-*")); + accessesNameField = true; } if (randomBoolean()) { bq1.filter( QueryBuilders.termsQuery("name", randomArray(3, 8, String[]::new, () -> "prod-" + randomInt() + "-east-" + randomInt())) ); + accessesNameField = true; } if (randomBoolean()) { bq1.mustNot(QueryBuilders.idsQuery().addIds(randomArray(1, 3, String[]::new, () -> randomAlphaOfLength(22)))); @@ -121,9 +132,18 @@ public void testBuildFromBoolQuery() { if (randomBoolean()) { bq1.minimumShouldMatch(randomIntBetween(1, 2)); } - final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(bq1, authentication); + final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(bq1, queryFields::add, authentication); assertCommonFilterQueries(apiKeyQb1, authentication); + assertThat(queryFields, hasItem("doc_type")); + if (accessesNameField) { + assertThat(queryFields, hasItem("name")); + } + if (authentication != null && authentication.isApiKey() == false) { + assertThat(queryFields, hasItem("creator.principal")); + assertThat(queryFields, hasItem("creator.realm")); + } + assertThat(apiKeyQb1.must(), hasSize(1)); assertThat(apiKeyQb1.should(), empty()); assertThat(apiKeyQb1.mustNot(), empty()); @@ -141,35 +161,78 @@ public void testFieldNameTranslation() { final Authentication authentication = randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null; // metadata - final String metadataKey = randomAlphaOfLengthBetween(3, 8); - final TermQueryBuilder q1 = QueryBuilders.termQuery("metadata." + metadataKey, randomAlphaOfLengthBetween(3, 8)); - final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, authentication); - assertCommonFilterQueries(apiKeyQb1, authentication); - assertThat(apiKeyQb1.must().get(0), equalTo(QueryBuilders.termQuery("metadata_flattened." + metadataKey, q1.value()))); + { + final List queryFields = new ArrayList<>(); + final String metadataKey = randomAlphaOfLengthBetween(3, 8); + final TermQueryBuilder q1 = QueryBuilders.termQuery("metadata." + metadataKey, randomAlphaOfLengthBetween(3, 8)); + final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, queryFields::add, authentication); + assertThat(queryFields, hasItem("doc_type")); + assertThat(queryFields, hasItem("metadata_flattened." + metadataKey)); + if (authentication != null && authentication.isApiKey() == false) { + assertThat(queryFields, hasItem("creator.principal")); + assertThat(queryFields, hasItem("creator.realm")); + } + assertCommonFilterQueries(apiKeyQb1, authentication); + assertThat(apiKeyQb1.must().get(0), equalTo(QueryBuilders.termQuery("metadata_flattened." + metadataKey, q1.value()))); + } // username - final PrefixQueryBuilder q2 = QueryBuilders.prefixQuery("username", randomAlphaOfLength(3)); - final ApiKeyBoolQueryBuilder apiKeyQb2 = ApiKeyBoolQueryBuilder.build(q2, authentication); - assertCommonFilterQueries(apiKeyQb2, authentication); - assertThat(apiKeyQb2.must().get(0), equalTo(QueryBuilders.prefixQuery("creator.principal", q2.value()))); + { + final List queryFields = new ArrayList<>(); + final PrefixQueryBuilder q2 = QueryBuilders.prefixQuery("username", randomAlphaOfLength(3)); + final ApiKeyBoolQueryBuilder apiKeyQb2 = ApiKeyBoolQueryBuilder.build(q2, queryFields::add, authentication); + assertThat(queryFields, hasItem("doc_type")); + assertThat(queryFields, hasItem("creator.principal")); + if (authentication != null && authentication.isApiKey() == false) { + assertThat(queryFields, hasItem("creator.realm")); + } + assertCommonFilterQueries(apiKeyQb2, authentication); + assertThat(apiKeyQb2.must().get(0), equalTo(QueryBuilders.prefixQuery("creator.principal", q2.value()))); + } // realm name - final WildcardQueryBuilder q3 = QueryBuilders.wildcardQuery("realm_name", "*" + randomAlphaOfLength(3)); - final ApiKeyBoolQueryBuilder apiKeyQb3 = ApiKeyBoolQueryBuilder.build(q3, authentication); - assertCommonFilterQueries(apiKeyQb3, authentication); - assertThat(apiKeyQb3.must().get(0), equalTo(QueryBuilders.wildcardQuery("creator.realm", q3.value()))); + { + final List queryFields = new ArrayList<>(); + final WildcardQueryBuilder q3 = QueryBuilders.wildcardQuery("realm_name", "*" + randomAlphaOfLength(3)); + final ApiKeyBoolQueryBuilder apiKeyQb3 = ApiKeyBoolQueryBuilder.build(q3, queryFields::add, authentication); + assertThat(queryFields, hasItem("doc_type")); + assertThat(queryFields, hasItem("creator.realm")); + if (authentication != null && authentication.isApiKey() == false) { + assertThat(queryFields, hasItem("creator.principal")); + } + assertCommonFilterQueries(apiKeyQb3, authentication); + assertThat(apiKeyQb3.must().get(0), equalTo(QueryBuilders.wildcardQuery("creator.realm", q3.value()))); + } // creation_time - final TermQueryBuilder q4 = QueryBuilders.termQuery("creation", randomLongBetween(0, Long.MAX_VALUE)); - final ApiKeyBoolQueryBuilder apiKeyQb4 = ApiKeyBoolQueryBuilder.build(q4, authentication); - assertCommonFilterQueries(apiKeyQb4, authentication); - assertThat(apiKeyQb4.must().get(0), equalTo(QueryBuilders.termQuery("creation_time", q4.value()))); + { + final List queryFields = new ArrayList<>(); + final TermQueryBuilder q4 = QueryBuilders.termQuery("creation", randomLongBetween(0, Long.MAX_VALUE)); + final ApiKeyBoolQueryBuilder apiKeyQb4 = ApiKeyBoolQueryBuilder.build(q4, queryFields::add, authentication); + assertThat(queryFields, hasItem("doc_type")); + assertThat(queryFields, hasItem("creation_time")); + if (authentication != null && authentication.isApiKey() == false) { + assertThat(queryFields, hasItem("creator.principal")); + assertThat(queryFields, hasItem("creator.realm")); + } + assertCommonFilterQueries(apiKeyQb4, authentication); + assertThat(apiKeyQb4.must().get(0), equalTo(QueryBuilders.termQuery("creation_time", q4.value()))); + } // expiration_time - final TermQueryBuilder q5 = QueryBuilders.termQuery("expiration", randomLongBetween(0, Long.MAX_VALUE)); - final ApiKeyBoolQueryBuilder apiKeyQb5 = ApiKeyBoolQueryBuilder.build(q5, authentication); - assertCommonFilterQueries(apiKeyQb5, authentication); - assertThat(apiKeyQb5.must().get(0), equalTo(QueryBuilders.termQuery("expiration_time", q5.value()))); + { + final List queryFields = new ArrayList<>(); + final TermQueryBuilder q5 = QueryBuilders.termQuery("expiration", randomLongBetween(0, Long.MAX_VALUE)); + final ApiKeyBoolQueryBuilder apiKeyQb5 = ApiKeyBoolQueryBuilder.build(q5, queryFields::add, authentication); + assertThat(queryFields, hasItem("doc_type")); + assertThat(queryFields, hasItem("expiration_time")); + if (authentication != null && authentication.isApiKey() == false) { + assertThat(queryFields, hasItem("creator.principal")); + assertThat(queryFields, hasItem("creator.realm")); + } + assertCommonFilterQueries(apiKeyQb5, authentication); + assertThat(apiKeyQb5.must().get(0), equalTo(QueryBuilders.termQuery("expiration_time", q5.value()))); + } } public void testAllowListOfFieldNames() { @@ -197,7 +260,7 @@ public void testAllowListOfFieldNames() { ); final IllegalArgumentException e1 = expectThrows( IllegalArgumentException.class, - () -> ApiKeyBoolQueryBuilder.build(q1, authentication) + () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); assertThat(e1.getMessage(), containsString("Field [" + fieldName + "] is not allowed for API Key query")); @@ -208,7 +271,7 @@ public void testTermsLookupIsNotAllowed() { final TermsQueryBuilder q1 = QueryBuilders.termsLookupQuery("name", new TermsLookup("lookup", "1", "names")); final IllegalArgumentException e1 = expectThrows( IllegalArgumentException.class, - () -> ApiKeyBoolQueryBuilder.build(q1, authentication) + () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); assertThat(e1.getMessage(), containsString("terms query with terms lookup is not supported for API Key query")); } @@ -218,7 +281,7 @@ public void testRangeQueryWithRelationIsNotAllowed() { final RangeQueryBuilder q1 = QueryBuilders.rangeQuery("creation").relation("contains"); final IllegalArgumentException e1 = expectThrows( IllegalArgumentException.class, - () -> ApiKeyBoolQueryBuilder.build(q1, authentication) + () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); assertThat(e1.getMessage(), containsString("range query with relation is not supported for API Key query")); } @@ -266,7 +329,7 @@ public void testDisallowedQueryTypes() { final IllegalArgumentException e1 = expectThrows( IllegalArgumentException.class, - () -> ApiKeyBoolQueryBuilder.build(q1, authentication) + () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); assertThat(e1.getMessage(), containsString("Query type [" + q1.getName() + "] is not supported for API Key query")); } @@ -274,6 +337,7 @@ public void testDisallowedQueryTypes() { public void testWillSetAllowedFields() throws IOException { final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build( randomSimpleQuery("name"), + ignored -> {}, randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null ); @@ -305,7 +369,11 @@ public void testWillFilterForApiKeyId() { new User(randomAlphaOfLengthBetween(5, 8)), apiKeyId ); - final ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(randomFrom(randomSimpleQuery("name"), null), authentication); + final ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build( + randomFrom(randomSimpleQuery("name"), null), + ignored -> {}, + authentication + ); assertThat(apiKeyQb.filter(), hasItem(QueryBuilders.termQuery("doc_type", "api_key"))); assertThat(apiKeyQb.filter(), hasItem(QueryBuilders.idsQuery().addIds(apiKeyId))); } @@ -314,11 +382,14 @@ private void testAllowedIndexFieldName(Predicate predicate) { final String allowedField = randomFrom( "doc_type", "name", + "type", + TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD, "api_key_invalidated", "creation_time", "expiration_time", "metadata_flattened." + randomAlphaOfLengthBetween(1, 10), - "creator." + randomAlphaOfLengthBetween(1, 10) + "creator.principal", + "creator.realm" ); assertThat(predicate, trueWith(allowedField)); @@ -362,4 +433,15 @@ private QueryBuilder randomSimpleQuery(String name) { .to(Instant.now().toEpochMilli(), randomBoolean()); }; } + + private void assertQueryFields(List actualQueryFields, QueryBuilder queryBuilder, Authentication authentication) { + assertThat(actualQueryFields, hasItem("doc_type")); + if ((queryBuilder instanceof IdsQueryBuilder || queryBuilder instanceof MatchAllQueryBuilder) == false) { + assertThat(actualQueryFields, hasItem("name")); + } + if (authentication != null && authentication.isApiKey() == false) { + assertThat(actualQueryFields, hasItem("creator.principal")); + assertThat(actualQueryFields, hasItem("creator.realm")); + } + } } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java index 1a37f31bffe79..2bce06543f67c 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -39,19 +40,53 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; public class ApiKeyBackwardsCompatibilityIT extends AbstractUpgradeTestCase { + private static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version")); + private RestClient oldVersionClient = null; private RestClient newVersionClient = null; + public void testQueryRestTypeKeys() throws IOException { + assumeTrue( + "only API keys created pre-8.9 are relevant for the rest-type query bwc case", + UPGRADE_FROM_VERSION.before(Version.V_8_9_0) + ); + switch (CLUSTER_TYPE) { + case OLD -> createOrGrantApiKey(client(), "query-test-rest-key-from-old-cluster", "{}"); + case MIXED -> createOrGrantApiKey(client(), "query-test-rest-key-from-mixed-cluster", "{}"); + case UPGRADED -> { + createOrGrantApiKey(client(), "query-test-rest-key-from-upgraded-cluster", "{}"); + for (String query : List.of(""" + {"query": {"term": {"type": "rest" }}}""", """ + {"query": {"prefix": {"type": "re" }}}""", """ + {"query": {"wildcard": {"type": "r*t" }}}""", """ + {"query": {"range": {"type": {"gte": "raaa", "lte": "rzzz"}}}}""")) { + assertQuery(client(), query, apiKeys -> { + assertThat( + apiKeys.stream().map(k -> (String) k.get("name")).toList(), + hasItems( + "query-test-rest-key-from-old-cluster", + "query-test-rest-key-from-mixed-cluster", + "query-test-rest-key-from-upgraded-cluster" + ) + ); + }); + } + } + } + } + public void testCreatingAndUpdatingApiKeys() throws Exception { assumeTrue( "The remote_indices for API Keys are not supported before transport version " @@ -177,7 +212,10 @@ private Tuple createOrGrantApiKey(String roles) throws IOExcepti } private Tuple createOrGrantApiKey(RestClient client, String roles) throws IOException { - final String name = "test-api-key-" + randomAlphaOfLengthBetween(3, 5); + return createOrGrantApiKey(client, "test-api-key-" + randomAlphaOfLengthBetween(3, 5), roles); + } + + private Tuple createOrGrantApiKey(RestClient client, String name, String roles) throws IOException { final Request createApiKeyRequest; String body = Strings.format(""" { @@ -391,4 +429,15 @@ private static RoleDescriptor randomRoleDescriptor(boolean includeRemoteIndices) null ); } + + private void assertQuery(RestClient restClient, String body, Consumer>> apiKeysVerifier) throws IOException { + final Request request = new Request("GET", "/_security/_query/api_key"); + request.setJsonEntity(body); + final Response response = restClient.performRequest(request); + assertOK(response); + final Map responseMap = responseAsMap(response); + @SuppressWarnings("unchecked") + final List> apiKeys = (List>) responseMap.get("api_keys"); + apiKeysVerifier.accept(apiKeys); + } }