From 9e4f3bea534ae8089e2f4b18116e43efe1b4f03c Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 1 Jul 2022 13:20:19 -0700 Subject: [PATCH] handles ints+longs --- .../transform_catalog/stream_processor.py | 3 +++ .../normalization/transform_catalog/utils.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index f76d268ef771..4d16b490d502 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -24,6 +24,7 @@ is_datetime_with_timezone, is_datetime_without_timezone, is_integer, + is_long, is_number, is_object, is_simple_property, @@ -513,6 +514,8 @@ def cast_property_type(self, property_name: str, column_name: str, jinja_column: cast_operation = jinja_call(f"cast_to_boolean({jinja_column})") return f"{cast_operation} as {column_name}" elif is_integer(definition): + sql_type = jinja_call("dbt_utils.type_int()") + elif is_long(definition): sql_type = jinja_call("dbt_utils.type_bigint()") elif is_number(definition["type"]): sql_type = jinja_call("dbt_utils.type_float()") diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py index 583fb3e5ce40..bf21293bfc4e 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py @@ -60,7 +60,12 @@ def is_number(property_type) -> bool: def is_integer(definition: dict) -> bool: - if "airbyte_type" in definition and definition["airbyte_type"] == "integer": + # By default, {type: integer} will be treated as a long (see is_long) + return "airbyte_type" in definition and definition["airbyte_type"] == "integer" + + +def is_long(definition: dict) -> bool: + if "airbyte_type" in definition and definition["airbyte_type"] == "long": return True property_type = definition["type"] if is_string(property_type) or is_number(property_type): @@ -71,7 +76,7 @@ def is_integer(definition: dict) -> bool: def is_boolean(definition: dict) -> bool: property_type = definition["type"] - if is_string(property_type) or is_number(property_type) or is_integer(definition): + if is_string(property_type) or is_number(property_type) or is_integer(definition) or is_long(definition): # Handle union type, give priority to wider scope types return False return property_type == "boolean" or "boolean" in property_type @@ -94,7 +99,7 @@ def is_simple_property(definition: dict) -> bool: property_type = "object" else: property_type = definition["type"] - return is_string(property_type) or is_integer(definition) or is_number(property_type) or is_boolean(definition) + return is_string(property_type) or is_integer(definition) or is_long(definition) or is_number(property_type) or is_boolean(definition) def is_combining_node(properties: dict) -> Set[str]: