Update to version v2.2.2 (#306)

* Update to version v2.1.1

* Update version to v2.2.1

* Update version to v2.2.2

---------

Co-authored-by: James Ma <[email protected]>

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.2.2] - 2024-08-23
+
+### Changed
+
+- Support editing the auto-generated Regular Expression of Nginx and Apache log in the Log Config. #301
+- Adjusted the error logs from info level to error level in the log-processor function. #305
+
+### Fixed
+
+- Fixed failed to retrieve instance status due to too many instances in the Instance Group details page. #298
+- Remove the redundant sign in alert dialog modal when user session expired. #303
+- Fixed the Grafana url validation failed with spaces when import a Grafana Server. #304
+
 ## [2.2.1] - 2024-07-22
 
 ### Added

source/constructs/lambda/api/app_log_ingestion/flb/flb_builder.py

@@ -331,6 +331,11 @@ def _get_flb_params(self):
             WithDecryption=True,
         )["Parameter"]["Value"]
 
+        storage_total_limit_size = ssm_cli.get_parameter(
+            Name=f"/{stack_prefix}/FLB/storage_total_limit_size",
+            WithDecryption=True,
+        )["Parameter"]["Value"]
+
         flb_params = {
             "log_level": log_level,
             "flush": flush,
@@ -342,6 +347,7 @@ def _get_flb_params(self):
             "store_dir_limit_size": store_dir_limit_size,
             "storage_type": storage_type,
             "storage_pause_on_chunks_overlimit": storage_pause_on_chunks_overlimit,
+            "storage_total_limit_size": storage_total_limit_size,
         }
         return flb_params
 

source/constructs/lambda/api/app_log_ingestion/flb/flb_template/pipeline.conf

@@ -178,6 +178,7 @@
 {% endif %}
 {{placeholder}}    Retry_Limit                 {{ssm_params.retry_limit}}
 {{placeholder}}    Role_arn                    {{item.role_arn}}
+{{placeholder}}    storage.total_limit_size    {{ssm_params.storage_total_limit_size}}
 {{placeholder}}
 {% elif item.output_name=='MSK' %}
 {{placeholder}}[OUTPUT]
@@ -194,6 +195,7 @@
 {{placeholder}}    rdkafka.compression.type    snappy
 {{placeholder}}    rdkafka.security.protocol   plaintext
 {{placeholder}}    Retry_Limit                 {{ssm_params.retry_limit}}
+{{placeholder}}    storage.total_limit_size    {{ssm_params.storage_total_limit_size}}
 {{placeholder}}
 {% elif item.output_name=='S3' %}
 {{placeholder}}[OUTPUT]
@@ -204,7 +206,9 @@
 {{placeholder}}    region                      {{item.region_name}}
 {{placeholder}}    total_file_size             {{item.s3.max_file_size}}M
 {{placeholder}}    upload_timeout              {{item.s3.upload_timeout}}s
-{{placeholder}}    store_dir_limit_size        {{ssm_params.store_dir_limit_size}}
+{{placeholder}}    #store_dir_limit_size        {{ssm_params.store_dir_limit_size}}
+{{placeholder}}    #FluentBit version 2.0 and above, please use the store_dir_limit_size parameter. Don't use storage.total_limit_size
+{{placeholder}}    storage.total_limit_size    {{ssm_params.store_dir_limit_size}}
 {{placeholder}}    use_put_object              true
 {% if item.s3.compression_type | lower == "gzip" %}
 {{placeholder}}    s3_key_format               /{{item.s3.prefix}}/%Y-%m-%d-%H-%M-%S-$UUID.gz
@@ -235,7 +239,7 @@
 {{placeholder}}    AWS_Auth                    On
 {{placeholder}}    TLS                         On
 {{placeholder}}    Suppress_Type_Name          On
-{{placeholder}}    Buffer_Size                 20M
+{{placeholder}}    Buffer_Size                 50M
 {{placeholder}}    #When enabled, generate _id for outgoing records. This prevents duplicate records when retrying.
 {{placeholder}}    Generate_ID                 On
 {{placeholder}}    Logstash_Format             Off
@@ -244,6 +248,7 @@
 {{placeholder}}    Time_Key_Nanos              Off
 {{placeholder}}    Write_Operation             create
 {{placeholder}}    AWS_Role_ARN                {{item.role_arn}}
+{{placeholder}}    storage.total_limit_size    {{ssm_params.storage_total_limit_size}}
 {{placeholder}}
 {% endif %}
 {% endwith %}

source/constructs/lambda/api/app_log_ingestion/test/test_flb.py

@@ -76,6 +76,12 @@ def mock_ssm_context():
             Type="String",
         )
 
+        ssm_client.put_parameter(
+            Name=f"/{stack_prefix}/FLB/storage_total_limit_size",
+            Value="off",
+            Type="String",
+        )
+
         yield
 
 

source/constructs/lambda/pipeline/log-processor/idx/idx_svc.py

@@ -91,10 +91,13 @@ def run_func_with_retry(
                 break
             logger.error("%s failed: %s", func_name, response.text)
             if response.status_code == 403 or response.status_code == 409:
-                logger.info(
-                    "Please add access to OpenSearch for this Lambda and rerun this"
-                )
+                logger.info("Please add access to OpenSearch for this Lambda")
                 if response.status_code == 403:
+                    logger.error(
+                        "the last response code is %d, the last response content is %s",
+                        response.status_code,
+                        response.content,
+                    )
                     self.map_backend_role()
                 raise APIException(
                     ErrorCode.UNKNOWN_ERROR,
@@ -107,7 +110,7 @@ def run_func_with_retry(
                     func_name,
                     retry,
                 )
-                logger.info(
+                logger.error(
                     "the last response code is %d, the last response content is %s",
                     response.status_code,
                     response.content,

source/constructs/lambda/pipeline/log-processor/idx/opensearch_client.py

@@ -120,9 +120,9 @@ def create_ism_policy(
             response = requests.put(
                 url, auth=self._awsauth, json=policy_doc, timeout=30
             )
-            logger.info("--> update ism policy response code %d", response.status_code)
+            logger.error("--> update ism policy response code %d", response.status_code)
             return response
-        logger.info(
+        logger.error(
             "the last response code is %d, the last response content is %s",
             response.status_code,
             response.content,
@@ -348,6 +348,12 @@ def add_master_role(self, role_arn: str):
                 status_code = resp["ResponseMetadata"]["HTTPStatusCode"]
                 logger.info("Response status: %d", status_code)
                 if status_code not in (200, 201):
+                    logger.error(
+                        "Add backend role %s to domain %s, response status: %d",
+                        role_arn,
+                        domain_name,
+                        status_code,
+                    )
                     raise APIException(
                         ErrorCode.UNKNOWN_ERROR,
                         "Failed to add backend role {role_arn} to domain {domain_name}",

source/constructs/lambda/pipeline/log-processor/lambda_function.py

@@ -71,7 +71,9 @@ def handle_sqs_retries(record):
             "This message has exceeded the maximum number of retries, verify that you can connect to OpenSearch or that the data type does not match the field type defined for the index",
         )
     else:
-        raise Exception(f"Error processing SQS message: {record}")
+        raise Exception(
+            f"Error processing SQS message: {record}, Lambda function has been called {approximate_receive_count} times, the message will be re-consumed and then retried!"
+        )
 
 
 def change_sqs_message_visibility(event_record):

source/constructs/lib/api/app-log-ingestion-stack.ts

@@ -122,11 +122,6 @@ export class AppLogIngestionStack extends Construct {
           description:
             'Download Fluent-Bit config file and reboot the Fluent-Bit',
           parameters: {
-            ARCHITECTURE: {
-              type: "String",
-              default: "",
-              description: "(Required) Machine Architecture"
-            },
             INSTANCEID: {
               type: 'String',
               default: '',
@@ -146,7 +141,7 @@ export class AppLogIngestionStack extends Construct {
               name: "updateFluentBitVersion",
               inputs: {
                 runCommand: [
-                  `[ -e /opt/fluent-bit/bin/fluent-bit ] && [ -z \"$(/opt/fluent-bit/bin/fluent-bit -V | grep '${FluentBitVersion}')\" ] && curl -o /opt/fluent-bit{{ARCHITECTURE}}.tar.gz 'https://${flb_s3_addr}/clo/${process.env.VERSION}/aws-for-fluent-bit/fluent-bit{{ARCHITECTURE}}.tar.gz' && tar xzvf /opt/fluent-bit{{ARCHITECTURE}}.tar.gz -C /opt/ --exclude=fluent-bit/etc; echo 0`
+                  `ARCHITECTURE=''; if [ \"$(uname -m)\" = \"aarch64\" ]; then ARCHITECTURE='-arm64'; fi; [ -e /opt/fluent-bit/bin/fluent-bit ] && [ -z \"$(/opt/fluent-bit/bin/fluent-bit -V | grep '${FluentBitVersion}')\" ] && curl -o /opt/fluent-bit$ARCHITECTURE.tar.gz https://${flb_s3_addr}/clo/${process.env.VERSION}/aws-for-fluent-bit/fluent-bit$ARCHITECTURE.tar.gz && tar xzvf /opt/fluent-bit$ARCHITECTURE.tar.gz -C /opt/ --exclude=fluent-bit/etc; echo 0`
                 ]
               },
             },

source/constructs/lib/api/fluent-bit-config-stack.ts

@@ -67,7 +67,7 @@ export class FluentBitConfigStack extends Construct {
         });
         new ssm.StringParameter(this, "FlbBufferSizeParameter", { //NOSONAR
             parameterName: `/${props.stackPrefix}/FLB/buffer_size`,
-            description: "Set the buffer size for HTTP client when reading responses from Kubernetes API server. A value of 0 results in no limit, and the buffer will expand as-needed.",
+            description: "Set the buffer size for HTTP client when reading responses from Kubernetes API server and the buffer size to read data in INPUT plugin. A value of 0 results in no limit, and the buffer will expand as-needed.",
             stringValue: "0",
         });
         new ssm.StringParameter(this, "FlbRetryLimitParameter", { //NOSONAR
@@ -90,6 +90,11 @@ export class FluentBitConfigStack extends Construct {
             description: "This parameter is to specifies if file storage is to be paused when reaching the chunk limit. Default is off",
             stringValue: "off",
         });
+        new ssm.StringParameter(this, "FlbStorageTotalLimitSizeParameter", { //NOSONAR
+            parameterName: `/${props.stackPrefix}/FLB/storage_total_limit_size`,
+            description: "This parameter is to limit the maximum number of Chunks in the filesystem for the current output logical destination. Default is 500M",
+            stringValue: "500M",
+        });
 
 
 

source/constructs/lib/api/instance-stack.ts

@@ -374,7 +374,7 @@ export class InstanceStack extends Construct {
         runtime: lambda.Runtime.PYTHON_3_11,
         handler: 'lambda_function.lambda_handler',
         timeout: Duration.minutes(5),
-        memorySize: 4096,
+        memorySize: 1024,
         logFormat: "JSON",
         applicationLogLevel: "INFO",
         systemLogLevel: "WARN",

source/constructs/lib/pipeline/common/opensearch-init-stack.ts

@@ -257,9 +257,6 @@ export class OpenSearchInitStack extends Construct {
       vpc: props.vpc,
       vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS },
       securityGroups: [props.securityGroup],
-      logFormat: "JSON",
-      applicationLogLevel: "ERROR",
-      systemLogLevel: "WARN",
       environment: Object.assign(
         {
           ENDPOINT: props.endpoint,
@@ -320,7 +317,7 @@ export class OpenSearchInitStack extends Construct {
           SOURCE: props.source,
           WRITE_IDX_DATA: props.writeIdxData || "True",
           NO_BUFFER_ACCESS_ROLE_ARN: props.noBufferAccessRoleArn || "",
-          // EVENT_BRIDGE_RULE_NAME: props.eventBridgeRuleName || "",
+          POWERTOOLS_LOG_LEVEL: "ERROR",
         },
         props.env),
       layers: [SharedPythonLayer.getInstance(this), pipeLayer],

source/constructs/package.json

@@ -25,8 +25,8 @@
     "@types/jest": "29.5.2",
     "@types/node": "20.9.0",
     "@types/prettier": "3.0.0",
-    "aws-cdk": "v2.149.0",
-    "aws-cdk-lib": "v2.149.0",
+    "aws-cdk": "2.151.0",
+    "aws-cdk-lib": "2.151.0",
     "constructs": "^10.3.0",
     "jest": "29.7.0",
     "ts-jest": "29.1.0",
@@ -43,12 +43,12 @@
   },
   "dependencies": {
     "@aws-cdk/aws-appsync-alpha": "2.59.0-alpha.0",
-    "@aws-cdk/aws-glue-alpha": "v2.149.0-alpha.0",
-    "@aws-cdk/aws-kinesisfirehose-alpha": "v2.149.0-alpha.0",
-    "@aws-cdk/aws-kinesisfirehose-destinations-alpha": "v2.149.0-alpha.0",
+    "@aws-cdk/aws-glue-alpha": "2.151.0-alpha.0",
+    "@aws-cdk/aws-kinesisfirehose-alpha": "2.151.0-alpha.0",
+    "@aws-cdk/aws-kinesisfirehose-destinations-alpha": "2.151.0-alpha.0",
     "@aws-solutions-constructs/aws-cloudfront-s3": "2.47.0",
-    "aws-cdk": "v2.149.0",
-    "aws-cdk-lib": "v2.149.0",
+    "aws-cdk": "2.151.0",
+    "aws-cdk-lib": "2.151.0",
     "cdk-nag": "2.27.66",
     "constructs": "10.3.0",
     "source-map-support": "0.5.21"

source/portal/package.json

@@ -24,7 +24,7 @@
     "aws-amplify": "^5.3.12",
     "aws-appsync-auth-link": "^3.0.7",
     "aws-appsync-subscription-link": "^3.1.2",
-    "axios": "^1.6.0",
+    "axios": "^1.7.4",
     "classnames": "^2.3.2",
     "date-fns": "^2.30.0",
     "graphql-tag": "^2.12.6",

source/portal/public/locales/en/cluster.json

@@ -191,22 +191,22 @@
       "creation": "Network creation",
       "method": "Creation Method",
       "auto": "Automatic",
-      "autoDesc": "Create a dedicated VPC for log processing layer and public access layer to connect to the OpenSearch domain via VPC peering.",
+      "autoDesc": "Automatically create the VPC Peering Connection between the log processing layer and the OpenSearch VPC if needed.",
       "manual": "Manual",
-      "manualDesc": "Manually specify the log processing layer and public access layer network settings.",
-      "layerNetwork": "Log processing layer network",
-      "layerNetworkDesc": "Solution requires one Log Processing Subnet Group. The log processing functions will reside in the log processing subnet and use the associated Security Group.",
+      "manualDesc": "Manually specify the log processing layer and update route tables and security group.",
+      "layerNetwork": "Log processing network",
+      "layerNetworkDesc": "Solution requires one log processing subnet group. The log processing functions will reside in the log processing subnet and use the associated security group.",
       "configTip": "Configure network",
-      "configTipDesc": "You have chosen a VPC other than the VPC of the selected OpenSearch domain. Please create VPC Peering, configure Route Table, and make sure the resources have access to the domain.",
+      "configTipDesc": "You have chosen a VPC other than the VPC of the selected Amazon OpenSearch domain. Please make sure the VPC can connect to the selected OpenSearch domain. ",
       "vpc": "VPC",
-      "vpcDesc": "Select the VPC of the log processing layer. The VPC must be the same VPC as the cluster or connected via VPC Peering.",
+      "vpcDesc": "Select the Virtual Private Cloud (VPC) for the log processing layer. The chosen VPC should align with the VPC of the cluster, or it must be connected through VPC Peering Connection or Transit Gateway.",
       "chooseVPC": "Choose a VPC",
-      "logProcessSubnetGroup": "Log Processing Subnet Group",
-      "logProcessSubnetGroupDesc": "Select at least two subnets of the log processing layer. It is recommended to select private subnets.",
-      "chooseSubnet": "Choose Subnet",
-      "logProcessSG": "Log Processing Security Group",
-      "logProcessSGDesc": "Select a Security Group for the log processing layer. The Security Group must have access the OpenSearch domain.",
-      "chooseSG": "Choose a Security Group"
+      "logProcessSubnetGroup": "Log processing subnet group",
+      "logProcessSubnetGroupDesc": "Choose a minimum of two subnets for the log processing layer. These subnets must be configured as private, with routing set up to connect to a NAT gateway.",
+      "chooseSubnet": "Choose two subnets",
+      "logProcessSG": "Log processing security group",
+      "logProcessSGDesc": "Choose a security group for the log processing layer. This security group must be configured to allow access to the OpenSearch cluster. If the connection is through a NAT gateway, port 443 needs to be open",
+      "chooseSG": "Choose a security group"
     }
   },
   "imported": {

source/portal/public/locales/en/ekslog.json

@@ -61,7 +61,7 @@
     },
     "daemonsetGuide": {
       "alert": "",
-      "alertDesc": "Please ensure that all log source(s) you have created are in the Created state before proceeding with the following steps. All steps in this procedure are to deploy Fluent Bit agent as a DaemonSet. By default, the namespace \"logging\" is used",
+      "alertDesc": "Please ensure that all log source(s) you have created are in the Created state before proceeding with the following steps. All steps in this procedure are to deploy Fluent Bit agent as a DaemonSet. By default, the namespace \"logging\" is used. If the YAML file changes, you need to restart the relevant FluentBit Pods.",
       "step1": "You will need to copy and paste all the content below in a file ~/fluent-bit-logging.yaml",
       "step2": "Use the kubectl command to deploy fluent-bit as DaemonSet.",
       "createIngestion": "Please create an log source first"

source/portal/public/locales/en/resource.json

@@ -160,6 +160,7 @@
       "apacheFormatError": "The specified format is invalid, a standard APACHE configuration file usually begins with LogFormat.",
       "regexFormat": "Regular Expression",
       "regexFormatDesc": "Input the regular expression to parse the log.",
+      "regexFormatDescNginxApache": "Input the regular expression to parse the log. Please do not change the regular expression match group name (log field).",
       "sampleParsing": "Sample log parsing",
       "parser": "Parser",
       "parserDesc": "Select the supported parser.",

source/portal/public/locales/zh/cluster.json

@@ -191,22 +191,22 @@
       "creation": "网络创建",
       "method": "创建方式",
       "auto": "自动",
-      "autoDesc": "为日志处理层和公共访问层创建一个专用 VPC，以通过 VPC 对等连接到 OpenSearch 域。",
+      "autoDesc": "如果需要，自动创建日志处理层和 OpenSearch VPC 之间的 VPC 对等连接。",
       "manual": "手动",
-      "manualDesc": "手动指定日志处理层和公共访问层网络设置。",
-      "layerNetwork": "日志处理层网络",
-      "layerNetworkDesc": "解决方案需要一个日志处理子网组。日志处理功能将驻留在日志处理子网中并使用关联的安全组。",
+      "manualDesc": "手动指定日志处理层并更新路由表和安全组。",
+      "layerNetwork": "日志处理网络",
+      "layerNetworkDesc": "解决方案需要一个日志处理子网组。日志处理功能将在日志处理子网中运行，并使用相关联的安全组。",
       "configTip": "配置网络",
-      "configTipDesc": "您选择的VPC不是所选 OpenSearch 域的 VPC。请创建VPC对等连接，配置路由表，并确保资源可以访问域。",
+      "configTipDesc": "您选择了与所选 Amazon OpenSearch 域的 VPC 不同的 VPC。请确保 VPC 可以连接到所选 OpenSearch 域。",
       "vpc": "VPC",
-      "vpcDesc": "选择日志处理层的 VPC。 VPC 必须与通过 VPC 对等连接的集群为同一 VPC。",
-      "chooseVPC": "选择一个VPC",
+      "vpcDesc": "选择用于日志处理层的虚拟私有云 (VPC)。选择的 VPC 应与集群的 VPC 对齐，或者必须通过 VPC 对等连接或传输网关进行连接。",
+      "chooseVPC": "选择一个 VPC",
       "logProcessSubnetGroup": "日志处理子网组",
-      "logProcessSubnetGroupDesc": "至少选择两个日志处理层的子网。建议选择私有子网。",
-      "chooseSubnet": "选择子网",
+      "logProcessSubnetGroupDesc": "选择至少两个子网用于日志处理层。这些子网必须配置为私有，并设置路由以连接到 NAT 网关。",
+      "chooseSubnet": "选择两个子网",
       "logProcessSG": "日志处理安全组",
-      "logProcessSGDesc": "为日志处理层选择一个安全组。 安全组必须有权访问 OpenSearch 域。",
-      "chooseSG": "选择安全组"
+      "logProcessSGDesc": "选择一个用于日志处理层的安全组。此安全组必须配置为允许访问 OpenSearch 集群。如果连接是 NAT 网关那么需要对外开放 443 端口",
+      "chooseSG": "选择一个安全组"
     }
   },
   "imported": {