fix: fix secret redaction

.github/workflows/dev-pytest.yml

@@ -72,4 +72,4 @@ jobs:
           echo "GITHUB WORKSPACE: ${GITHUB_WORKSPACE}"
           chmod +x testSchemachange.sh
           bash testSchemachange.sh
-        working-directory: .
+        working-directory: .

populateConnection.sh

@@ -9,4 +9,4 @@ echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml
 echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml
 echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml
 echo "cat connections.toml"
-cat ./connections.toml
+cat ./connections.toml

schemachange/config/ChangeHistoryTable.py

@@ -17,7 +17,7 @@ class ChangeHistoryTable:
     @property
     def fully_qualified(self) -> str:
         return f"{self.database_name}.{self.schema_name}.{self.table_name}"
-    
+
     @property
     def fully_qualified_schema_name(self) -> str:
         return f"{self.database_name}.{self.schema_name}"

schemachange/config/DeployConfig.py

@@ -8,9 +8,9 @@
 from schemachange.config.ChangeHistoryTable import ChangeHistoryTable
 from schemachange.config.utils import (
     get_snowflake_identifier_string,
-    get_snowflake_password,
 )
 
+
 @dataclasses.dataclass(frozen=True)
 class DeployConfig(BaseConfig):
     subcommand: Literal["deploy"] = "deploy"
@@ -84,12 +84,12 @@ def get_session_kwargs(self) -> dict:
             "connection_name": self.connection_name,
             "change_history_table": self.change_history_table,
             "autocommit": self.autocommit,
-            "query_tag": self.query_tag
+            "query_tag": self.query_tag,
         }
-        
-        # TODO: Discuss the need for check for snowflake password before passing the session 
+
+        # TODO: Discuss the need for check for snowflake password before passing the session
         # kwargs to open a snowflake session
         # snowflake_password = get_snowflake_password()
         # if snowflake_password is not None and snowflake_password:
-        #    session_kwargs["password"] = snowflake_password 
+        #    session_kwargs["password"] = snowflake_password
         return {k: v for k, v in session_kwargs.items() if v is not None}

schemachange/config/get_merged_config.py

@@ -11,7 +11,7 @@
 from schemachange.config.utils import (
     load_yaml_config,
     validate_directory,
-    validate_file_path
+    validate_file_path,
 )
 
 

schemachange/redact_config_secrets.py

@@ -2,7 +2,7 @@
 
 import copy
 import warnings
-from typing import Callable
+from typing import Callable, Any
 
 import structlog
 from structlog import PrintLogger
@@ -14,39 +14,41 @@ def get_redact_config_secrets_processor(
     def redact_config_secrets_processor(
         _: PrintLogger, __: str, event_dict: dict
     ) -> dict:
-        def redact_dict(level: int, sub_event_dict: dict) -> dict:
+        def redact_value(level: int, value: Any):
             if level > 6:
                 warnings.warn(
                     "Unable to redact deeply nested secrets in log: %(event)s"
                     % {"event": event_dict["event"]}
                 )
-                return sub_event_dict
-            for sub_k, sub_v in sub_event_dict.items():
-                if isinstance(sub_v, dict):
-                    sub_event_dict[sub_k] = redact_dict(
-                        level=level + 1, sub_event_dict=sub_v
-                    )
-                elif isinstance(sub_v, str):
-                    for secret in config_secrets:
-                        if secret in sub_v:
-                            sub_event_dict[sub_k] = sub_event_dict[sub_k].replace(
-                                secret, "*" * len(secret)
-                            )
-                elif isinstance(sub_v, int):
-                    for secret in config_secrets:
-                        if secret in str(sub_v):
-                            sub_event_dict[sub_k] = str(sub_event_dict[sub_k]).replace(
-                                secret, "*" * len(secret)
-                            )
-                else:
+                return value
+            if isinstance(value, dict):
+                for sub_k, sub_v in value.items():
+                    value[sub_k] = redact_value(level=level + 1, value=sub_v)
+                return value
+            elif isinstance(value, list):
+                for i, sub_v in enumerate(value):
+                    value[i] = redact_value(level=level + 1, value=sub_v)
+                return value
+            elif isinstance(value, set):
+                return {redact_value(level=level + 1, value=sub_v) for sub_v in value}
+            elif isinstance(value, tuple):
+                return tuple(
+                    redact_value(level=level + 1, value=sub_v) for sub_v in value
+                )
+            elif not isinstance(value, str):
+                try:
+                    value = str(value)
+                except Exception:
                     warnings.warn(
                         "Unable to redact %(type)s log arguments in log: %(event)s"
-                        % {"type": type(sub_v).__name__, "event": event_dict["event"]}
+                        % {"type": type(value).__name__, "event": event_dict["event"]}
                     )
-                    return sub_event_dict
-            return sub_event_dict
+                    return value
+            for secret in config_secrets:
+                value = value.replace(secret, "*" * len(secret))
+            return value
 
-        return redact_dict(level=0, sub_event_dict=copy.deepcopy(event_dict))
+        return redact_value(level=0, value=copy.deepcopy(event_dict))
 
     return redact_config_secrets_processor