forked from opensearch-project/dashboards-observability
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Raw Vpc schema integration (1.0.0 parquet ) (opensearch-project#1853)
* revert default vpc flow logs integration into standard vpc schema Signed-off-by: YANGDB <[email protected]> * update sample queries --------- Signed-off-by: YANGDB <[email protected]>
- Loading branch information
Showing
17 changed files
with
109 additions
and
361 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
...ations/__data__/repository/amazon_vpc_flow/assets/aws_vpc_live_stream_mv_schema-1.0.0.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| CREATE MATERIALIZED VIEW {table_name}__live_mview AS | ||
| SELECT | ||
| CAST(IFNULL(srcport, 0) AS LONG) AS `aws.vpc.srcport`, | ||
| CAST(IFNULL(pkt_srcaddr, 'Unknown') AS STRING) AS `aws.vpc.pkt-src-aws-service`, | ||
| CAST(IFNULL(srcaddr, '0.0.0.0') AS STRING) AS `aws.vpc.srcaddr`, | ||
| CAST(IFNULL(interface_id, 'Unknown') AS STRING) AS `aws.vpc.src-interface_uid`, | ||
| CAST(IFNULL(vpc_id, 'Unknown') AS STRING) AS `aws.vpc.src-vpc_uid`, | ||
| CAST(IFNULL(instance_id, 'Unknown') AS STRING) AS `aws.vpc.src-instance_uid`, | ||
| CAST(IFNULL(subnet_id, 'Unknown') AS STRING) AS `aws.vpc.src-subnet_uid`, | ||
| CAST(IFNULL(dstport, 0) AS LONG) AS `aws.vpc.dstport`, | ||
| CAST(IFNULL(pkt_dstaddr, 'Unknown') AS STRING) AS `aws.vpc.pkt-dst-aws-service`, | ||
| CAST(IFNULL(dstaddr, '0.0.0.0') AS STRING) AS `aws.vpc.dstaddr`, | ||
| CAST(IFNULL(flow_direction, 'Unknown') AS STRING) AS `aws.vpc.flow-direction`, | ||
| CAST(IFNULL(tcp_flags, '0') AS STRING) AS `aws.vpc.connection.tcp_flags`, | ||
| CAST(IFNULL(packets, 0) AS LONG) AS `aws.vpc.packets`, | ||
| CAST(IFNULL(bytes, 0) AS LONG) AS `aws.vpc.bytes`, | ||
| CAST(FROM_UNIXTIME(start ) AS TIMESTAMP) AS `@timestamp`, | ||
| CAST(FROM_UNIXTIME(start ) AS TIMESTAMP) AS `start_time`, | ||
| CAST(FROM_UNIXTIME(start ) AS TIMESTAMP) AS `interval_start_time`, | ||
| CAST(FROM_UNIXTIME(`end` ) AS TIMESTAMP) AS `end_time`, | ||
| CAST(IFNULL(log_status, 'Unknown') AS STRING) AS `aws.vpc.status_code`, | ||
| CAST(IFNULL(version, 0) AS LONG) AS `aws.vpc.version`, | ||
| CAST(IFNULL(type, 'Unknown') AS STRING) AS `aws.vpc.type_name`, | ||
| CAST(IFNULL(traffic_path, 0) AS LONG) AS `aws.vpc.traffic_path`, | ||
| CAST(IFNULL(az_id, 'Unknown') AS STRING) AS `aws.vpc.az_id`, | ||
| CAST(IFNULL(action, 'Unknown') AS STRING) AS `aws.vpc.action`, | ||
| CAST(IFNULL(region, 'Unknown') AS STRING) AS `aws.vpc.region`, | ||
| CAST(IFNULL(account_id, 'Unknown') AS STRING) AS `aws.vpc.account-id`, | ||
| CAST(IFNULL(sublocation_type, 'Unknown') AS STRING) AS `aws.vpc.sublocation_type`, | ||
| CAST(IFNULL(sublocation_id, 'Unknown') AS STRING) AS `aws.vpc.sublocation_id` | ||
|
|
||
| FROM | ||
| {table_name} | ||
| WITH ( | ||
| auto_refresh = true, | ||
| refresh_interval = '15 Minute', | ||
| checkpoint_location = '{s3_checkpoint_location}', | ||
| watermark_delay = '1 Minute', | ||
| extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' | ||
| ) |
14 changes: 14 additions & 0 deletions
14
...s/integrations/__data__/repository/amazon_vpc_flow/assets/create_skipping_index-1.0.0.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| CREATE SKIPPING INDEX ON {table_name} ( | ||
| account_id BLOOM_FILTER, | ||
| region VALUE_SET, | ||
| srcaddr BLOOM_FILTER, | ||
| dstaddr BLOOM_FILTER, | ||
| pkt_src_aws_service VALUE_SET, | ||
| pkt_dst_aws_service VALUE_SET, | ||
| bytes MIN_MAX | ||
| ) WITH ( | ||
| auto_refresh = true, | ||
| refresh_interval = '15 Minutes', | ||
| checkpoint_location = '{s3_checkpoint_location}', | ||
| watermark_delay = '1 Minute' | ||
| ) |
20 changes: 0 additions & 20 deletions
20
...aptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc-1.0.0.sql
This file was deleted.
Oops, something went wrong.
32 changes: 32 additions & 0 deletions
32
...integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc_schema-1.0.0.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( | ||
| version int, | ||
| account_id string, | ||
| interface_id string, | ||
| srcaddr string, | ||
| dstaddr string, | ||
| srcport int, | ||
| dstport int, | ||
| protocol bigint, | ||
| packets bigint, | ||
| bytes bigint, | ||
| start bigint, | ||
| `end` bigint, | ||
| action string, | ||
| log_status string, | ||
| vpc_id string, | ||
| subnet_id string, | ||
| instance_id string, | ||
| tcp_flags int, | ||
| type string, | ||
| pkt_srcaddr string, | ||
| pkt_dstaddr string, | ||
| region string, | ||
| az_id string, | ||
| sublocation_type string, | ||
| sublocation_id string, | ||
| pkt_src_aws_service string, | ||
| pkt_dst_aws_service string, | ||
| flow_direction string, | ||
| traffic_path int | ||
| ) USING parquet | ||
| LOCATION '{s3_bucket_location}' |
4 changes: 4 additions & 0 deletions
4
...tors/integrations/__data__/repository/amazon_vpc_flow/assets/example_queries-1.0.0.ndjson
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| {"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"TopCommonErrorServicesQuery","query":"SELECT pkt_src_aws_service AS source_service, pkt_dst_aws_service AS destination_service, COUNT(*) AS error_count FROM {table_name} where log_status IN ('SKIPDATA', 'RETIREDDATA') GROUP BY pkt_src_aws_service, pkt_dst_aws_service ORDER BY error_count DESC LIMIT 10","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Top 10 pairs of errored source / destination services","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="} | ||
| {"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"HourAggRequestsAndBytes","query":"SELECT date_trunc('hour', FROM_UNIXTIME(CAST(IFNULL(start, 0) AS LONG))) AS interval_start_time, CAST(IFNULL(pkt_srcaddr, 'Unknown') AS STRING) AS `aws.vpc.pkt-src-aws-service`, CAST(IFNULL(srcaddr, '0.0.0.0') AS STRING) AS `aws.vpc.srcaddr`, CAST(IFNULL(pkt_dstaddr, 'Unknown') AS STRING) AS `aws.vpc.pkt-dst-aws-service`, CAST(IFNULL(dstaddr, '0.0.0.0') AS STRING) AS `aws.vpc.dstaddr`, CAST(IFNULL(action, 'Unknown') AS STRING) AS `aws.vpc.action`, CAST(IFNULL(region, 'Unknown') AS STRING) AS `aws.vpc.region`, CAST(IFNULL(account_id, 'Unknown') AS STRING) AS `aws.vpc.account-id`, CAST(IFNULL(log_status, 'Unknown') AS STRING) AS `aws.vpc.status_code`, CAST(IFNULL(flow_direction, 'Unknown') AS STRING) AS `aws.vpc.connection.direction`, COUNT(*) AS total_connections, SUM(CAST(IFNULL(bytes, 0) AS LONG)) AS total_bytes, SUM(CAST(IFNULL(packets, 0) AS LONG)) AS total_packets FROM `zero_etl_walkthrough`.`default`.`amazon_vpc_flow` GROUP BY date_trunc('hour', FROM_UNIXTIME(CAST(IFNULL(start, 0) AS LONG))), pkt_srcaddr, srcaddr, pkt_dstaddr, dstaddr, action, region, account_id, log_status, flow_direction","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Hour aggregation by requests and bytes sum ","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd9992c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="} | ||
| {"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"HourWindowTopIpByCardinality","query":" WITH hourly_buckets AS (SELECT date_trunc('hour', FROM_UNIXTIME(CAST(IFNULL(start, 0) AS LONG))) AS interval_start_time, CAST(IFNULL(dstaddr, '0.0.0.0') AS STRING) AS dstaddr, SUM(CAST(IFNULL(bytes, 0) AS LONG)) AS total_bytes FROM {table_name} GROUP BY interval_start_time, dstaddr), ranked_addresses AS (SELECT CAST(interval_start_time AS TIMESTAMP), dstaddr, total_bytes, RANK() OVER (PARTITION BY interval_start_time ORDER BY total_bytes DESC) AS bytes_rank FROM hourly_buckets) SELECT CAST(interval_start_time AS TIMESTAMP), dstaddr, total_bytes FROM ranked_addresses WHERE bytes_rank <= 50 ORDER BY interval_start_time ASC, bytes_rank ASC","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Hour window of top IP by cardinality ","version":1},"id":"9e6add40-fc1a-11ee-99c9-43e5dbd9992c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="} | ||
| {"exportedCount":7,"missingRefCount":0,"missingReferences":[]} |
32 changes: 0 additions & 32 deletions
32
...rations/__data__/repository/amazon_vpc_flow/assets/vpc_agg_30min_connections_mv-1.0.0.sql
This file was deleted.
Oops, something went wrong.
1 change: 0 additions & 1 deletion
1
...ns/__data__/repository/amazon_vpc_flow/assets/vpc_agg_30min_connections_refresh-1.0.0.sql
This file was deleted.
Oops, something went wrong.
31 changes: 0 additions & 31 deletions
31
...rations/__data__/repository/amazon_vpc_flow/assets/vpc_agg_60min_connections_mv-1.0.0.sql
This file was deleted.
Oops, something went wrong.
1 change: 0 additions & 1 deletion
1
...ns/__data__/repository/amazon_vpc_flow/assets/vpc_agg_60min_connections_refresh-1.0.0.sql
This file was deleted.
Oops, something went wrong.
61 changes: 0 additions & 61 deletions
61
...daptors/integrations/__data__/repository/amazon_vpc_flow/assets/vpc_live_all_mv-1.0.0.sql
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.