Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

993 cloud watch launch templates #999

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
Open

993 cloud watch launch templates #999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f7229d4
#993: ASG launch template user-data is now programmically set up for …
philipjyoon Sep 25, 2024
d0b65a4
#993: Fixed run_ log file and group names. We don't have control over…
philipjyoon Sep 25, 2024
7cffb48
#993: SCIFLO and non-SCIFLO queue names have different number of toke…
philipjyoon Sep 25, 2024
45a9d4d
#993: More fixes for SCIFLO log file name
philipjyoon Sep 25, 2024
10dcbe8
#993: Adding back run_on_demand log
philipjyoon Sep 26, 2024
b3dc630
Merge branch 'develop' into 993_cloud_watch_launch_templates
philipjyoon Sep 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cluster_provisioning/modules/common/asg.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ resource "aws_launch_template" "launch_template" {
var_venue = var.venue
local_counter = local.counter
var_environment = var.environment
run_log_group = length(split("-", lower(each.key))) == 4 ? split("-", lower(each.key))[3] : split("-", lower(each.key))[2]
log_file_name = lookup(each.value, "log_file_name", "run_job")
}))
vpc_security_group_ids = [lookup(each.value, "use_private_vpc", true) ? var.private_verdi_security_group_id : var.public_verdi_security_group_id]

Expand Down
99 changes: 6 additions & 93 deletions cluster_provisioning/modules/common/autoscaling_groups.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,102 +9,15 @@ resource "aws_cloudwatch_log_group" "autoscaling-log-groups-job-worker" {
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_hlsl30_query" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_hlsl30_query.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_hlss30_query" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_hlss30_query.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_hls_download" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_hls_download.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_slcs1a_query" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_slcs1a_query.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_slcs1b_query" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_slcs1b_query.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_slc_download" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_slc_download.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_ionosphere_download" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_ionosphere_download.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_rtc_query" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_rtc_query.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_rtc_download" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_rtc_download.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_cslc_query" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_cslc_query.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_cslc_download" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_cslc_download.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_submit_pending_jobs" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_submit_pending_jobs.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_batch_query" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_batch_query.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_pcm_int" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_pcm_int.log"
# SCIFLO queue names have 4 tokens when split by '-', the rest have 3.
resource "aws_cloudwatch_log_group" "autoscaling-log-groups-job-worker-run" {
for_each = var.queues
name = length(split("-", lower(each.key))) == 4 ? "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_${split("-", lower(each.key))[3]}.log" : "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_${split("-", lower(each.key))[2]}.log"
retention_in_days = var.lambda_log_retention_in_days
}

# This log groups doesn't have corresponding OPERA-defined queues.
resource "aws_cloudwatch_log_group" "run_on_demand" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_on_demand.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_sciflo_L3_DSWx_HLS" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L3_DSWx_HLS.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_sciflo_L2_CSLC_S1" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L2_CSLC_S1.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_sciflo_L2_RTC_S1" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L2_RTC_S1.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_sciflo_L3_DSWx_S1" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L3_DSWx_S1.log"
retention_in_days = var.lambda_log_retention_in_days
}

resource "aws_cloudwatch_log_group" "run_sciflo_L3_DISP_S1" {
name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L3_DISP_S1.log"
retention_in_days = var.lambda_log_retention_in_days
}
}
219 changes: 5 additions & 214 deletions cluster_provisioning/modules/common/launch_template_user_data.sh.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ echo "PASS" >> /tmp/user_data_test.txt

mkdir -p /opt/aws/amazon-cloudwatch-agent/etc/
touch /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json

echo '{
"agent": {
"metrics_collection_interval": 10,
Expand All @@ -23,231 +24,21 @@ echo '{
"timezone": "UTC"
},
{
"file_path": "/data/work/jobs/**/run_hlsl30_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_hlsl30_query.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_hlss30_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_hlss30_query.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_hls_download.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_hls_download.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_slcs1a_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_slcs1a_query.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_slcs1b_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_slcs1b_query.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_slc_download.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_slc_download.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_ionosphere_download.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_ionosphere_download.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_rtc_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_rtc_query.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_rtc_download.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_rtc_download.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_cslc_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_cslc_query.log",
"file_path": "/data/work/jobs/**/${log_file_name}.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_${run_log_group}.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_cslc_download.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_cslc_download.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_submit_pending_jobs.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_submit_pending_jobs.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_pcm_int.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_pcm_int.log",
"file_path": "/home/ops/verdi/log/${each_key}.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/${each_key}.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/data/work/jobs/**/run_on_demand.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_on_demand.log",
"timezone": "Local"
},
{
"file_path": "/data/work/jobs/**/run_sciflo_L3_DSWx_HLS.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L3_DSWx_HLS.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S"
},
{
"file_path": "/data/work/jobs/**/run_sciflo_L3_DSWx_S1.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L3_DSWx_S1.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S"
},
{
"file_path": "/data/work/jobs/**/run_sciflo_L3_DSWx_NI.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L3_DSWx_NI.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S"
},
{
"file_path": "/data/work/jobs/**/run_sciflo_L3_DISP_S1.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L3_DISP_S1.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S"
},
{
"file_path": "/data/work/jobs/**/run_sciflo_L2_CSLC_S1.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L2_CSLC_S1.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S"
},
{
"file_path": "/data/work/jobs/**/run_sciflo_L2_RTC_S1.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L2_RTC_S1.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-hls_data_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-hls_data_query.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-hls_data_download.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-hls_data_download.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-slc_data_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-slc_data_query.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-slc_data_download.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-slc_data_download.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-slc_data_download_ionosphere.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-slc_data_download_ionosphere.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-rtc_data_query.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-rtc_data_query.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-rtc_data_download.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-rtc_data_download.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-hls_data_ingest.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-hls_data_ingest.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l3_dswx_hls.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l3_dswx_hls.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l3_dswx_s1.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l3_dswx_s1.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l3_dswx_ni.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l3_dswx_ni.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l3_disp_s1.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l3_disp_s1.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l2_cslc_s1.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l2_cslc_s1.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l2_rtc_s1.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l2_rtc_s1.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l2_rtc_s1_static.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l2_rtc_s1_static.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-send_cnm_notify.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-send_cnm_notify.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-rcv_cnm_notify.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-rcv_cnm_notify.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
},
{
"file_path": "/home/ops/verdi/log/opera-job_worker-pge_smoke_test.log",
"log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-pge_smoke_test.log",
"timezone": "Local",
"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
}
]
}
Expand Down
Loading