From 685aeaf2f4f2726b401e9181d3957110504eeb0e Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 29 Apr 2024 11:50:40 -0700 Subject: [PATCH] Add skipping indices for all integrations that have sample queries (#1747) * Update avg threshold for cloudfront query Signed-off-by: Simeon Widdis * Add first draft skipping index for cloudfront integration Signed-off-by: Simeon Widdis * Add skipping index for nginx Signed-off-by: Simeon Widdis * Add skipping index for s3 Signed-off-by: Simeon Widdis * Add skipping index for elb Signed-off-by: Simeon Widdis * Update skipping indices to use bloom filters for client IDs Signed-off-by: Simeon Widdis * Add skipping index for apache Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../repository/apache/apache-1.0.0.json | 7 ++++++ .../assets/create_skipping_index-1.0.0.sql | 10 ++++++++ .../assets/create_skipping_index-1.0.0.sql | 10 ++++++++ .../assets/example_queries-1.0.0.ndjson | 2 +- .../aws_cloudfront/aws_cloudfront-1.0.0.json | 23 ++++++++++++------- .../assets/create_skipping_index-1.0.0.sql | 11 +++++++++ .../repository/aws_elb/aws_elb-1.0.0.json | 7 ++++++ .../assets/create_skipping_index-1.0.0.sql | 10 ++++++++ .../repository/aws_s3/aws_s3-1.0.0.json | 7 ++++++ .../assets/create_skipping_index-1.0.0.sql | 10 ++++++++ .../repository/nginx/nginx-1.0.0.json | 7 ++++++ 11 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 server/adaptors/integrations/__data__/repository/apache/assets/create_skipping_index-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_skipping_index-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/aws_elb/assets/create_skipping_index-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/aws_s3/assets/create_skipping_index-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/nginx/assets/create_skipping_index-1.0.0.sql diff --git a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json index 1e9d0c62d..93f62fc38 100644 --- a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json @@ -69,6 +69,13 @@ "extension": "sql", "type": "query" }, + { + "name": "create_skipping_index", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["queries"] + }, { "name": "create_mv", "version": "1.0.0", diff --git a/server/adaptors/integrations/__data__/repository/apache/assets/create_skipping_index-1.0.0.sql b/server/adaptors/integrations/__data__/repository/apache/assets/create_skipping_index-1.0.0.sql new file mode 100644 index 000000000..60fda56fa --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/apache/assets/create_skipping_index-1.0.0.sql @@ -0,0 +1,10 @@ +CREATE SKIPPING INDEX ON {table_name} ( + remote_addr BLOOM_FILTER, + `status` VALUE_SET, + body_bytes_sent MIN_MAX +) WITH ( + auto_refresh = true, + refresh_interval = '15 Minutes', + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute' +) diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_skipping_index-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_skipping_index-1.0.0.sql new file mode 100644 index 000000000..6d7a13b27 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_skipping_index-1.0.0.sql @@ -0,0 +1,10 @@ +CREATE SKIPPING INDEX ON {table_name} ( + c_ip BLOOM_FILTER, + sc_status VALUE_SET, + time_to_first_byte MIN_MAX +) WITH ( + auto_refresh = true, + refresh_interval = '15 Minutes', + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute' +) diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/example_queries-1.0.0.ndjson b/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/example_queries-1.0.0.ndjson index 91fc20255..8ed7ede7a 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/example_queries-1.0.0.ndjson +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/example_queries-1.0.0.ndjson @@ -3,6 +3,6 @@ {"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Number of Requests","query":"SELECT COUNT(*) AS request_count FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Number of Requests","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="} {"attributes":{"createdTimeMs":1713293161193,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Total Bytes Served","query":"SELECT SUM(sc_bytes) AS total_bytes_served FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Total Bytes Served","version":1},"id":"92398eb0-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:46:01.242Z","version":"WzI4MzEsMV0="} {"attributes":{"createdTimeMs":1713293269224,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Average Time Taken","query":"SELECT AVG(time_taken) AS average_time_taken FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Average Time Taken","version":1},"id":"d2a038a0-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:47:49.290Z","version":"WzI4MzIsMV0="} -{"attributes":{"createdTimeMs":1713293425335,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Slow Requests from Average Time threshold","query":"WITH avg_time AS (SELECT AVG(time_to_first_byte) AS avg_time FROM {table_name}) SELECT * FROM {table_name} CROSS JOIN avg_time WHERE time_to_first_byte > 1 * avg_time LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Slow Requests from Average Time threshold","version":1},"id":"2fac4250-fc22-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:59:34.785Z","version":"WzI4MzQsMV0="} +{"attributes":{"createdTimeMs":1713293425335,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Slow Requests from Average Time threshold","query":"WITH avg_time AS (SELECT AVG(time_to_first_byte) AS avg_time FROM {table_name}) SELECT * FROM {table_name} CROSS JOIN avg_time WHERE time_to_first_byte > 2 * avg_time LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Slow Requests from Average Time threshold","version":1},"id":"2fac4250-fc22-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:59:34.785Z","version":"WzI4MzQsMV0="} {"attributes":{"createdTimeMs":1713294061574,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Requests by User Agent","query":"SELECT * FROM {table_name} WHERE cs_user_agent LIKE '%Chrome%' LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Requests by User Agent","version":1},"id":"aae73c80-fc23-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T19:01:01.640Z","version":"WzI4MzUsMV0="} {"exportedCount":7,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json index 6e206f7aa..fb0d4a808 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json @@ -54,17 +54,24 @@ ], "assets": [ { - "name": "aws_cloudfront", + "name": "create_table", "version": "1.0.0", - "extension": "ndjson", - "type": "savedObjectBundle", - "workflows": ["dashboards"] + "extension": "sql", + "type": "query" }, { - "name": "create_table", + "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", - "type": "query" + "type": "query", + "workflows": ["queries"] + }, + { + "name": "example_queries", + "version": "1.0.0", + "extension": "ndjson", + "type": "savedObjectBundle", + "workflows": ["queries"] }, { "name": "create_mv", @@ -74,11 +81,11 @@ "workflows": ["dashboards"] }, { - "name": "example_queries", + "name": "aws_cloudfront", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", - "workflows": ["queries"] + "workflows": ["dashboards"] } ], "sampleData": { diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_skipping_index-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_skipping_index-1.0.0.sql new file mode 100644 index 000000000..8f84e7984 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_skipping_index-1.0.0.sql @@ -0,0 +1,11 @@ +CREATE SKIPPING INDEX ON {table_name} ( + client_ip BLOOM_FILTER, + elb_status_code VALUE_SET, + request_processing_time MIN_MAX, + sent_bytes MIN_MAX +) WITH ( + auto_refresh = true, + refresh_interval = '15 Minutes', + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute' +) diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index a62242b5b..be3b7322c 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -74,6 +74,13 @@ "extension": "sql", "type": "query" }, + { + "name": "create_skipping_index", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["queries"] + }, { "name": "create_mv", "version": "1.0.0", diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_skipping_index-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_skipping_index-1.0.0.sql new file mode 100644 index 000000000..afdf2d407 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_skipping_index-1.0.0.sql @@ -0,0 +1,10 @@ +CREATE SKIPPING INDEX ON {table_name} ( + requester BLOOM_FILTER, + http_status VALUE_SET, + request_time MIN_MAX +) WITH ( + auto_refresh = true, + refresh_interval = '15 Minutes', + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute' +) diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json index df67dd628..d1d8c1307 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json @@ -69,6 +69,13 @@ "extension": "sql", "type": "query" }, + { + "name": "create_skipping_index", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["queries"] + }, { "name": "create_mv", "version": "1.0.0", diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_skipping_index-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_skipping_index-1.0.0.sql new file mode 100644 index 000000000..60fda56fa --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/create_skipping_index-1.0.0.sql @@ -0,0 +1,10 @@ +CREATE SKIPPING INDEX ON {table_name} ( + remote_addr BLOOM_FILTER, + `status` VALUE_SET, + body_bytes_sent MIN_MAX +) WITH ( + auto_refresh = true, + refresh_interval = '15 Minutes', + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute' +) diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index 9b0958503..f59e4c562 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -66,6 +66,13 @@ "extension": "sql", "type": "query" }, + { + "name": "create_skipping_index", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["queries"] + }, { "name": "create_mv", "version": "1.0.0",