fix normalization

airbytehq · Jul 20, 2022 · 0edcf1b · 0edcf1b
1 parent 247c8ab
commit 0edcf1b
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 11 deletions.
diff --git a/...integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/...integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py
@@ -510,12 +510,12 @@ def cast_property_type(self, property_name: str, column_name: str, jinja_column:
         elif is_object(definition["type"]):
             sql_type = jinja_call("type_json()")
         # Treat simple types from narrower to wider scope type: boolean < integer < number < string
-        elif is_boolean(definition):
+        elif is_boolean(definition["type"], definition):
             cast_operation = jinja_call(f"cast_to_boolean({jinja_column})")
             return f"{cast_operation} as {column_name}"
         elif is_integer(definition):
             sql_type = jinja_call("dbt_utils.type_int()")
-        elif is_long(definition):
+        elif is_long(definition["type"], definition):
             sql_type = jinja_call("dbt_utils.type_bigint()")
         elif is_number(definition["type"]):
             sql_type = jinja_call("dbt_utils.type_float()")
@@ -686,7 +686,7 @@ def safe_cast_to_string(definition: Dict, column_name: str, destination_type: De
 
         if "type" not in definition:
             col = column_name
-        elif is_boolean(definition):
+        elif is_boolean(definition["type"], definition):
             col = f"boolean_to_string({column_name})"
         elif is_array(definition["type"]):
             col = f"array_to_string({column_name})"

diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py
@@ -60,23 +60,22 @@ def is_number(property_type) -> bool:
 
 
 def is_integer(definition: dict) -> bool:
-    # By default, {type: integer} will be treated as a long (see is_long)
+    # By default, {type: integer} will be treated as an int64 (see is_long)
+    # Only {type: number, airbyte_type: integer} is treated as an int32
     return "airbyte_type" in definition and definition["airbyte_type"] == "integer"
 
 
-def is_long(definition: dict) -> bool:
-    if "airbyte_type" in definition and definition["airbyte_type"] == "long":
+def is_long(property_type, definition: dict) -> bool:
+    if "airbyte_type" in definition and definition["airbyte_type"] == "big_integer":
         return True
-    property_type = definition["type"]
     if is_string(property_type) or is_number(property_type):
         # Handle union type, give priority to wider scope types
         return False
     return property_type == "integer" or "integer" in property_type
 
 
-def is_boolean(definition: dict) -> bool:
-    property_type = definition["type"]
-    if is_string(property_type) or is_number(property_type) or is_integer(definition) or is_long(definition):
+def is_boolean(property_type, definition: dict) -> bool:
+    if is_string(property_type) or is_number(property_type) or is_integer(definition) or is_long(property_type, definition):
         # Handle union type, give priority to wider scope types
         return False
     return property_type == "boolean" or "boolean" in property_type
@@ -99,7 +98,13 @@ def is_simple_property(definition: dict) -> bool:
         property_type = "object"
     else:
         property_type = definition["type"]
-    return is_string(property_type) or is_integer(definition) or is_long(definition) or is_number(property_type) or is_boolean(definition)
+    return (
+        is_string(property_type)
+        or is_integer(definition)
+        or is_long(property_type, definition)
+        or is_number(property_type)
+        or is_boolean(property_type, definition)
+    )
 
 
 def is_combining_node(properties: dict) -> Set[str]: