diff --git a/cluster_provisioning/modules/common/asg.tf b/cluster_provisioning/modules/common/asg.tf index 5864f7fc..7778d8fa 100644 --- a/cluster_provisioning/modules/common/asg.tf +++ b/cluster_provisioning/modules/common/asg.tf @@ -21,6 +21,8 @@ resource "aws_launch_template" "launch_template" { var_venue = var.venue local_counter = local.counter var_environment = var.environment + run_log_group = length(split("-", lower(each.key))) == 4 ? split("-", lower(each.key))[3] : split("-", lower(each.key))[2] + log_file_name = lookup(each.value, "log_file_name", "run_job") })) vpc_security_group_ids = [lookup(each.value, "use_private_vpc", true) ? var.private_verdi_security_group_id : var.public_verdi_security_group_id] diff --git a/cluster_provisioning/modules/common/autoscaling_groups.tf b/cluster_provisioning/modules/common/autoscaling_groups.tf index 497c50d4..0e310b1c 100644 --- a/cluster_provisioning/modules/common/autoscaling_groups.tf +++ b/cluster_provisioning/modules/common/autoscaling_groups.tf @@ -9,102 +9,15 @@ resource "aws_cloudwatch_log_group" "autoscaling-log-groups-job-worker" { retention_in_days = var.lambda_log_retention_in_days } -resource "aws_cloudwatch_log_group" "run_hlsl30_query" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_hlsl30_query.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_hlss30_query" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_hlss30_query.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_hls_download" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_hls_download.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_slcs1a_query" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_slcs1a_query.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_slcs1b_query" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_slcs1b_query.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_slc_download" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_slc_download.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_ionosphere_download" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_ionosphere_download.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_rtc_query" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_rtc_query.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_rtc_download" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_rtc_download.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_cslc_query" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_cslc_query.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_cslc_download" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_cslc_download.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_submit_pending_jobs" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_submit_pending_jobs.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_batch_query" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_batch_query.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_pcm_int" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_pcm_int.log" +# SCIFLO queue names have 4 tokens when split by '-', the rest have 3. +resource "aws_cloudwatch_log_group" "autoscaling-log-groups-job-worker-run" { + for_each = var.queues + name = length(split("-", lower(each.key))) == 4 ? "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_${split("-", lower(each.key))[3]}.log" : "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_${split("-", lower(each.key))[2]}.log" retention_in_days = var.lambda_log_retention_in_days } +# This log groups doesn't have corresponding OPERA-defined queues. resource "aws_cloudwatch_log_group" "run_on_demand" { name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_on_demand.log" retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_sciflo_L3_DSWx_HLS" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L3_DSWx_HLS.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_sciflo_L2_CSLC_S1" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L2_CSLC_S1.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_sciflo_L2_RTC_S1" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L2_RTC_S1.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_sciflo_L3_DSWx_S1" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L3_DSWx_S1.log" - retention_in_days = var.lambda_log_retention_in_days -} - -resource "aws_cloudwatch_log_group" "run_sciflo_L3_DISP_S1" { - name = "/opera/sds/${var.project}-${var.venue}-${local.counter}/run_sciflo_L3_DISP_S1.log" - retention_in_days = var.lambda_log_retention_in_days -} +} \ No newline at end of file diff --git a/cluster_provisioning/modules/common/launch_template_user_data.sh.tmpl b/cluster_provisioning/modules/common/launch_template_user_data.sh.tmpl index adbc1696..b05a6c63 100644 --- a/cluster_provisioning/modules/common/launch_template_user_data.sh.tmpl +++ b/cluster_provisioning/modules/common/launch_template_user_data.sh.tmpl @@ -8,6 +8,7 @@ echo "PASS" >> /tmp/user_data_test.txt mkdir -p /opt/aws/amazon-cloudwatch-agent/etc/ touch /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json + echo '{ "agent": { "metrics_collection_interval": 10, @@ -23,80 +24,14 @@ echo '{ "timezone": "UTC" }, { - "file_path": "/data/work/jobs/**/run_hlsl30_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_hlsl30_query.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_hlss30_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_hlss30_query.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_hls_download.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_hls_download.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_slcs1a_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_slcs1a_query.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_slcs1b_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_slcs1b_query.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_slc_download.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_slc_download.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_ionosphere_download.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_ionosphere_download.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_rtc_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_rtc_query.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_rtc_download.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_rtc_download.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_cslc_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_cslc_query.log", + "file_path": "/data/work/jobs/**/${log_file_name}.log", + "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_${run_log_group}.log", "timezone": "Local", "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" }, { - "file_path": "/data/work/jobs/**/run_cslc_download.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_cslc_download.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_submit_pending_jobs.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_submit_pending_jobs.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/data/work/jobs/**/run_pcm_int.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_pcm_int.log", + "file_path": "/home/ops/verdi/log/${each_key}.log", + "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/${each_key}.log", "timezone": "Local", "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" }, @@ -104,150 +39,6 @@ echo '{ "file_path": "/data/work/jobs/**/run_on_demand.log", "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_on_demand.log", "timezone": "Local" - }, - { - "file_path": "/data/work/jobs/**/run_sciflo_L3_DSWx_HLS.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L3_DSWx_HLS.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S" - }, - { - "file_path": "/data/work/jobs/**/run_sciflo_L3_DSWx_S1.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L3_DSWx_S1.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S" - }, - { - "file_path": "/data/work/jobs/**/run_sciflo_L3_DSWx_NI.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L3_DSWx_NI.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S" - }, - { - "file_path": "/data/work/jobs/**/run_sciflo_L3_DISP_S1.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L3_DISP_S1.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S" - }, - { - "file_path": "/data/work/jobs/**/run_sciflo_L2_CSLC_S1.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L2_CSLC_S1.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S" - }, - { - "file_path": "/data/work/jobs/**/run_sciflo_L2_RTC_S1.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/run_sciflo_L2_RTC_S1.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-hls_data_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-hls_data_query.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-hls_data_download.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-hls_data_download.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-slc_data_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-slc_data_query.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-slc_data_download.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-slc_data_download.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-slc_data_download_ionosphere.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-slc_data_download_ionosphere.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-rtc_data_query.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-rtc_data_query.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-rtc_data_download.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-rtc_data_download.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-hls_data_ingest.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-hls_data_ingest.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l3_dswx_hls.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l3_dswx_hls.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l3_dswx_s1.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l3_dswx_s1.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l3_dswx_ni.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l3_dswx_ni.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l3_disp_s1.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l3_disp_s1.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l2_cslc_s1.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l2_cslc_s1.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l2_rtc_s1.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l2_rtc_s1.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-sciflo-l2_rtc_s1_static.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-sciflo-l2_rtc_s1_static.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-send_cnm_notify.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-send_cnm_notify.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-rcv_cnm_notify.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-rcv_cnm_notify.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" - }, - { - "file_path": "/home/ops/verdi/log/opera-job_worker-pge_smoke_test.log", - "log_group_name": "/opera/sds/${var_project}-${var_venue}-${local_counter}/opera-job_worker-pge_smoke_test.log", - "timezone": "Local", - "timestamp_format": "%Y-%m-%d %H:%M:%S,%f" } ] } diff --git a/cluster_provisioning/modules/common/variables.tf b/cluster_provisioning/modules/common/variables.tf index aa4acea0..767024d1 100644 --- a/cluster_provisioning/modules/common/variables.tf +++ b/cluster_provisioning/modules/common/variables.tf @@ -294,6 +294,7 @@ variable "queues" { } "opera-job_worker-sciflo-l2_cslc_s1" = { "name" = "opera-job_worker-sciflo-l2_cslc_s1" + "log_file_name" = "run_sciflo_L2_CSLC_S1" "instance_type" = ["c7i.2xlarge", "c6a.2xlarge", "c6i.2xlarge"] "root_dev_size" = 50 "data_dev_size" = 300 @@ -302,6 +303,7 @@ variable "queues" { } "opera-job_worker-sciflo-l2_cslc_s1_hist" = { "name" = "opera-job_worker-sciflo-l2_cslc_s1_hist" + "log_file_name" = "run_sciflo_L2_CSLC_S1" "instance_type" = ["c7i.2xlarge", "c6a.2xlarge", "c6i.2xlarge"] "root_dev_size" = 50 "data_dev_size" = 300 @@ -310,6 +312,7 @@ variable "queues" { } "opera-job_worker-sciflo-l2_rtc_s1" = { "name" = "opera-job_worker-sciflo-l2_rtc_s1" + "log_file_name" = "run_sciflo_L2_RTC_S1" "instance_type" = ["c7i.2xlarge", "c6a.2xlarge", "c6i.2xlarge", "c6a.4xlarge", "c6i.4xlarge"] "root_dev_size" = 50 "data_dev_size" = 100 @@ -318,6 +321,7 @@ variable "queues" { } "opera-job_worker-sciflo-l2_rtc_s1_static" = { "name" = "opera-job_worker-sciflo-l2_rtc_s1_static" + "log_file_name" = "run_sciflo_L2_RTC_S1" "instance_type" = ["r6a.2xlarge", "r6i.2xlarge"] "root_dev_size" = 50 "data_dev_size" = 100 @@ -326,6 +330,7 @@ variable "queues" { } "opera-job_worker-sciflo-l3_dswx_hls" = { "name" = "opera-job_worker-sciflo-l3_dswx_hls" + "log_file_name" = "run_sciflo_L3_DSWx_HLS" "instance_type" = ["c7a.large", "c6a.large", "c6i.large"] "root_dev_size" = 50 "data_dev_size" = 50 @@ -335,6 +340,7 @@ variable "queues" { } "opera-job_worker-sciflo-l3_dswx_s1" = { "name" = "opera-job_worker-sciflo-l3_dswx_s1" + "log_file_name" = "run_sciflo_L3_DSWx_S1" "instance_type" = ["c7i.xlarge", "c7i.2xlarge", "c7i.4xlarge", "c7i.8xlarge", "m7i.xlarge", "m7i.2xlarge", "m7i.4xlarge", "m7i.8xlarge"] "root_dev_size" = 50 "data_dev_size" = 100 @@ -343,6 +349,8 @@ variable "queues" { "total_jobs_metric" = true } "opera-job_worker-sciflo-l3_disp_s1" = { + "name" = "opera-job_worker-sciflo-l3_disp_s1" + "log_file_name" = "run_sciflo_L3_DISP_S1" "instance_type" = ["c7i.4xlarge", "c6a.4xlarge", "c6i.4xlarge", "c7a.4xlarge", "c5.4xlarge"] "root_dev_size" = 50 "data_dev_size" = 600 @@ -350,6 +358,8 @@ variable "queues" { "total_jobs_metric" = true } "opera-job_worker-sciflo-l3_disp_s1_hist" = { + "name" = "opera-job_worker-sciflo-l3_disp_s1_hist" + "log_file_name" = "run_sciflo_L3_DISP_S1" "instance_type" = ["r7i.4xlarge", "r6a.4xlarge", "r6i.4xlarge", "r7a.4xlarge", "r5a.4xlarge"] "root_dev_size" = 50 "data_dev_size" = 600 @@ -358,6 +368,7 @@ variable "queues" { } "opera-job_worker-sciflo-l3_dswx_ni" = { "name" = "opera-job_worker-sciflo-l3_dswx_ni" + "log_file_name" = "run_sciflo_L3_DSWx_NI" "instance_type" = ["c7i.2xlarge", "c6a.2xlarge", "m7i.2xlarge", "m7a.2xlarge", "c7a.2xlarge", "m6a.2xlarge", "c6i.2xlarge", "c5.2xlarge", "m6i.2xlarge", "c5a.2xlarge", "c5ad.2xlarge"] "root_dev_size" = 50 "data_dev_size" = 100 diff --git a/data_subscriber/cslc/cslc_download.sh b/data_subscriber/cslc/cslc_download.sh index c6cc3b60..32995e23 100755 --- a/data_subscriber/cslc/cslc_download.sh +++ b/data_subscriber/cslc/cslc_download.sh @@ -21,7 +21,7 @@ date # Forward processing use case; download all undownloaded files from ES index echo "python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p ASF-CSLC $* 2>&1" -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p ASF-CSLC $* > run_cslc_download.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p ASF-CSLC $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/cslc/cslc_query.sh b/data_subscriber/cslc/cslc_query.sh index 92438647..8b7539a7 100755 --- a/data_subscriber/cslc/cslc_query.sh +++ b/data_subscriber/cslc/cslc_query.sh @@ -21,7 +21,7 @@ date # Forward processing use case; query previous 60 minutes echo "python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query --collection-shortname=OPERA_L2_CSLC-S1_V1 $* 2>&1" -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query --collection-shortname=OPERA_L2_CSLC-S1_V1 $* > run_cslc_query.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query --collection-shortname=OPERA_L2_CSLC-S1_V1 $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/hls/hls_download.sh b/data_subscriber/hls/hls_download.sh index becb6b31..cde2a4ea 100755 --- a/data_subscriber/hls/hls_download.sh +++ b/data_subscriber/hls/hls_download.sh @@ -21,7 +21,7 @@ date # Forward processing use case; download all undownloaded files from ES index echo "python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p LPCLOUD" -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p LPCLOUD $* > run_hls_download.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p LPCLOUD $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/hls/hlsl30_query.sh b/data_subscriber/hls/hlsl30_query.sh index d044c4aa..95fcf2c7 100755 --- a/data_subscriber/hls/hlsl30_query.sh +++ b/data_subscriber/hls/hlsl30_query.sh @@ -21,7 +21,7 @@ date # Forward processing use case; query previous 60 minutes echo "python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c HLSL30 $* 2>&1" -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c HLSL30 $* > run_hlsl30_query.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c HLSL30 $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/hls/hlss30_query.sh b/data_subscriber/hls/hlss30_query.sh index c4af1f29..590925fe 100755 --- a/data_subscriber/hls/hlss30_query.sh +++ b/data_subscriber/hls/hlss30_query.sh @@ -21,7 +21,7 @@ date # Forward processing use case; query previous 60 minutes echo "python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c HLSS30 $* 2>&1" -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c HLSS30 $* > run_hlss30_query.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c HLSS30 $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/ionosphere_download.sh b/data_subscriber/ionosphere_download.sh index ebc19edf..0bb798b9 100755 --- a/data_subscriber/ionosphere_download.sh +++ b/data_subscriber/ionosphere_download.sh @@ -18,7 +18,7 @@ echo "##########################################" echo "Running job to download SLC ionosphere data" date -python $OPERA_HOME/data_subscriber/ionosphere_download.py $* > run_ionosphere_download.log 2>&1 +python $OPERA_HOME/data_subscriber/ionosphere_download.py $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/rtc/rtc_download.sh b/data_subscriber/rtc/rtc_download.sh index f52e5b34..0eab8450 100755 --- a/data_subscriber/rtc/rtc_download.sh +++ b/data_subscriber/rtc/rtc_download.sh @@ -19,7 +19,7 @@ echo "##########################################" echo "Running job to download ASFDAAC RTC data" date -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download --provider=ASF-RTC $* > run_rtc_download.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download --provider=ASF-RTC $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/rtc/rtc_query.sh b/data_subscriber/rtc/rtc_query.sh index f4bc7b70..460710d8 100755 --- a/data_subscriber/rtc/rtc_query.sh +++ b/data_subscriber/rtc/rtc_query.sh @@ -19,7 +19,7 @@ echo "##########################################" echo "Running job to query ASFDAAC RTC data" date -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query --collection-shortname=OPERA_L2_RTC-S1_V1 $* > run_rtc_query.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query --collection-shortname=OPERA_L2_RTC-S1_V1 $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/slc/slc_download.sh b/data_subscriber/slc/slc_download.sh index 1a1d4d46..f5202935 100755 --- a/data_subscriber/slc/slc_download.sh +++ b/data_subscriber/slc/slc_download.sh @@ -21,7 +21,7 @@ date # Forward processing use case; download all undownloaded files from ES index echo "python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p ASF $* 2>&1" -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p ASF $* > run_slc_download.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py download -p ASF $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/slc/slcs1a_query.sh b/data_subscriber/slc/slcs1a_query.sh index 38274da0..7d2953a9 100755 --- a/data_subscriber/slc/slcs1a_query.sh +++ b/data_subscriber/slc/slcs1a_query.sh @@ -21,7 +21,7 @@ date # Forward processing use case; query previous 60 minutes echo "python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c SENTINEL-1A_SLC $* 2>&1" -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c SENTINEL-1A_SLC $* > run_slcs1a_query.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c SENTINEL-1A_SLC $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/slc/slcs1b_query.sh b/data_subscriber/slc/slcs1b_query.sh index d4d4fd2e..14612610 100755 --- a/data_subscriber/slc/slcs1b_query.sh +++ b/data_subscriber/slc/slcs1b_query.sh @@ -21,7 +21,7 @@ date # Forward processing use case; query previous 60 minutes echo "python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c SENTINEL-1B_SLC $* 2>&1" -python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c SENTINEL-1B_SLC $* > run_slcs1b_query.log 2>&1 +python $OPERA_HOME/data_subscriber/daac_data_subscriber.py query -c SENTINEL-1B_SLC $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job" diff --git a/data_subscriber/submit_pending_jobs.sh b/data_subscriber/submit_pending_jobs.sh index e2c0e7b5..cecd5c40 100755 --- a/data_subscriber/submit_pending_jobs.sh +++ b/data_subscriber/submit_pending_jobs.sh @@ -18,7 +18,7 @@ echo "##########################################" echo "Running job to submit pending jobs that are ready to be run" date -python $OPERA_HOME/data_subscriber/submit_pending_jobs.py $* > run_submit_pending_jobs.log 2>&1 +python $OPERA_HOME/data_subscriber/submit_pending_jobs.py $* > run_job.log 2>&1 if [ $? -eq 0 ]; then echo "Finished running job"