diff --git a/fennel/CHANGELOG.md b/fennel/CHANGELOG.md index ace4c9b03..b8a2ee84e 100644 --- a/fennel/CHANGELOG.md +++ b/fennel/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## [1.5.34] - 2024-09-30 +- Add support for secrets manager in connectors. + ## [1.5.33] - 2024-09-30 - Add support for now in expressions. diff --git a/fennel/_vendor/pydantic/errors.py b/fennel/_vendor/pydantic/errors.py index 7bdafdd17..5e6038d01 100644 --- a/fennel/_vendor/pydantic/errors.py +++ b/fennel/_vendor/pydantic/errors.py @@ -586,6 +586,8 @@ class IPv4NetworkError(PydanticValueError): class IPv6NetworkError(PydanticValueError): msg_template = 'value is not a valid IPv6 network' +class AWSSecretError(PydanticValueError): + msg_template = 'value is not a valid AWS secret' class IPv4InterfaceError(PydanticValueError): msg_template = 'value is not a valid IPv4 interface' diff --git a/fennel/_vendor/pydantic/validators.py b/fennel/_vendor/pydantic/validators.py index 4e8580f18..deec0d7ac 100644 --- a/fennel/_vendor/pydantic/validators.py +++ b/fennel/_vendor/pydantic/validators.py @@ -42,6 +42,7 @@ is_typeddict, ) from .utils import almost_equal_floats, lenient_issubclass, sequence_like +from ...integrations.aws import Secret if TYPE_CHECKING: from fennel._vendor.typing_extensions import Literal, TypedDict @@ -252,6 +253,10 @@ def dict_validator(v: Any) -> Dict[Any, Any]: if isinstance(v, dict): return v + # without this check, we get a recursive loop which tries to convert secret to dict by calling __getitem__ + if isinstance(v, Secret): + raise errors.DictError() + try: return dict(v) except (TypeError, ValueError): @@ -413,6 +418,14 @@ def ip_v6_network_validator(v: Any) -> IPv6Network: except ValueError: raise errors.IPv6NetworkError() +def aws_secret_validator(v: Any) -> Secret: + if isinstance(v, Secret): + return v + try: + return Secret(v) + except ValueError: + raise errors.AWSSecretError() + def ip_v4_interface_validator(v: Any) -> IPv4Interface: if isinstance(v, IPv4Interface): @@ -692,6 +705,7 @@ def check(self, config: Type['BaseConfig']) -> bool: (IPv6Address, [ip_v6_address_validator]), (IPv4Network, [ip_v4_network_validator]), (IPv6Network, [ip_v6_network_validator]), + (Secret, [aws_secret_validator]), ] diff --git a/fennel/connectors/connectors.py b/fennel/connectors/connectors.py index c414536f0..f9820f69f 100644 --- a/fennel/connectors/connectors.py +++ b/fennel/connectors/connectors.py @@ -21,6 +21,7 @@ from fennel._vendor.pydantic import validator # type: ignore from fennel.connectors.kinesis import at_timestamp from fennel.expr.expr import Expr, TypedExpr +from fennel.integrations.aws import Secret from fennel.internal_lib.duration import ( Duration, ) @@ -325,8 +326,8 @@ def identifier(self) -> str: class SQLSource(DataSource): host: str db_name: str - username: str - password: str + username: Union[str, Secret] + password: Union[str, Secret] jdbc_params: Optional[str] = None _get: bool = False @@ -341,8 +342,8 @@ class CSV: class S3(DataSource): - aws_access_key_id: Optional[str] - aws_secret_access_key: Optional[str] + aws_access_key_id: Optional[Union[str, Secret]] + aws_secret_access_key: Optional[Union[str, Secret]] role_arn: Optional[str] def bucket( @@ -372,8 +373,8 @@ def get(name: str) -> S3: return S3( name=name, _get=True, - aws_access_key_id="", - aws_secret_access_key="", + aws_access_key_id=None, + aws_secret_access_key=None, role_arn=None, ) @@ -384,7 +385,7 @@ def identifier(self) -> str: class BigQuery(DataSource): project_id: str dataset_id: str - service_account_key: dict[str, str] + service_account_key: Union[dict[str, str], Secret] def table(self, table_name: str, cursor: str) -> TableConnector: return TableConnector(self, table_name, cursor) @@ -409,25 +410,25 @@ def identifier(self) -> str: class Avro(BaseModel): registry: str url: str - username: Optional[str] - password: Optional[str] - token: Optional[str] + username: Optional[Union[str, Secret]] + password: Optional[Union[str, Secret]] + token: Optional[Union[str, Secret]] class Protobuf(BaseModel): registry: str url: str - username: Optional[str] - password: Optional[str] - token: Optional[str] + username: Optional[Union[str, Secret]] + password: Optional[Union[str, Secret]] + token: Optional[Union[str, Secret]] def __init__( self, registry: str, url: str, - username: Optional[str] = None, - password: Optional[str] = None, - token: Optional[str] = None, + username: Optional[Union[str, Secret]] = None, + password: Optional[Union[str, Secret]] = None, + token: Optional[Union[str, Secret]] = None, ): if username or password: if token is not None: @@ -453,8 +454,8 @@ class Kafka(DataSource): bootstrap_servers: str security_protocol: Literal["PLAINTEXT", "SASL_PLAINTEXT", "SASL_SSL"] sasl_mechanism: Literal["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "GSSAPI"] - sasl_plain_username: Optional[str] - sasl_plain_password: Optional[str] + sasl_plain_username: Optional[Union[str, Secret]] + sasl_plain_password: Optional[Union[str, Secret]] def required_fields(self) -> List[str]: return ["topic"] @@ -525,8 +526,8 @@ def identifier(self) -> str: class Snowflake(DataSource): account: str db_name: str - username: str - password: str + username: Union[str, Secret] + password: Union[str, Secret] warehouse: str src_schema: str = Field(alias="schema") role: str @@ -584,8 +585,8 @@ def required_fields(self) -> List[str]: class Redshift(DataSource): s3_access_role_arn: Optional[str] - username: Optional[str] - password: Optional[str] + username: Optional[Union[str, Secret]] + password: Optional[Union[str, Secret]] db_name: str host: str port: int = 5439 @@ -617,8 +618,8 @@ def identifier(self) -> str: class Mongo(DataSource): host: str db_name: str - username: str - password: str + username: Union[str, Secret] + password: Union[str, Secret] def collection(self, collection_name: str, cursor: str) -> TableConnector: return TableConnector(self, table_name=collection_name, cursor=cursor) @@ -643,7 +644,7 @@ def identifier(self) -> str: class PubSub(DataSource): project_id: str - service_account_key: dict[str, str] + service_account_key: Union[dict[str, str], Secret] def required_fields(self) -> List[str]: return ["topic_id", "format"] diff --git a/fennel/connectors/test_connectors.py b/fennel/connectors/test_connectors.py index b65e1a3dc..e4de6a84f 100644 --- a/fennel/connectors/test_connectors.py +++ b/fennel/connectors/test_connectors.py @@ -25,9 +25,10 @@ PubSub, eval, ) -from fennel.connectors.connectors import CSV +from fennel.connectors.connectors import CSV, Postgres from fennel.datasets import dataset, field, pipeline, Dataset from fennel.expr import col, lit +from fennel.integrations.aws import Secret from fennel.lib import meta from fennel.lib.params import inputs @@ -357,14 +358,49 @@ class UserInfoDataset: ) +aws_secret = Secret( + arn="arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + role_arn="arn:aws:iam::123456789012:role/fennel-test-role", +) + s3 = S3( name="ratings_source", aws_access_key_id="ALIAQOTFAKEACCCESSKEYIDGTAXJY6MZWLP", aws_secret_access_key="8YCvIs8f0+FAKESECRETKEY+7uYSDmq164v9hNjOIIi3q1uV8rv", ) +s3_with_secret = S3( + name="ratings_source", + aws_access_key_id=aws_secret["s3_aws_access_key_id"], + aws_secret_access_key=aws_secret["s3_aws_secret_access_key"], +) + simple_s3 = S3(name="my_s3_src") +mysql_with_secret = MySQL( + name="mysql", + host="localhost", + db_name="test", + username=aws_secret["mysql_username"], + password=aws_secret["mysql_password"], +) + +postgres = Postgres( + name="postgres", + host="localhost", + db_name="test", + username="root", + password="root", +) + +postgres_with_secret = Postgres( + name="postgres", + host="localhost", + db_name="test", + username=aws_secret["postgres_username"], + password=aws_secret["postgres_password"], +) + bigquery = BigQuery( name="bq_movie_tags", project_id="gold-cocoa-356105", @@ -380,6 +416,13 @@ class UserInfoDataset: }, ) +bigquery_with_secret = BigQuery( + name="bq_movie_tags", + project_id="gold-cocoa-356105", + dataset_id="movie_tags", + service_account_key=aws_secret["bigquery_service_account_key"], +) + snowflake = Snowflake( name="snowflake_src", account="nhb38793.us-west-2.snowflakecomputing.com", @@ -391,6 +434,17 @@ class UserInfoDataset: password="", ) +snowflake_with_secret = Snowflake( + name="snowflake_src", + account="nhb38793.us-west-2.snowflakecomputing.com", + warehouse="TEST", + db_name="MOVIELENS", + schema="PUBLIC", + role="ACCOUNTADMIN", + username=aws_secret["us-west-2"]["username"], + password=aws_secret["us-west-2"]["password"], +) + kafka = Kafka( name="kafka_src", bootstrap_servers="localhost:9092", @@ -400,6 +454,15 @@ class UserInfoDataset: sasl_plain_password="test", ) +kafka_with_secret = Kafka( + name="kafka_src", + bootstrap_servers="localhost:9092", + security_protocol="PLAINTEXT", + sasl_mechanism="PLAIN", + sasl_plain_username=aws_secret["sasl_plain_username"], + sasl_plain_password=aws_secret["sasl_plain_password"], +) + s3_console = S3.get( name="s3_test", ) @@ -426,6 +489,15 @@ class UserInfoDataset: password="password", ) +redshift_with_secret = Redshift( + name="redshift_src_2", + db_name="test", + host="test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", + schema="public", + username=aws_secret["redshift_username"], + password=aws_secret["redshift_password"], +) + mongo = Mongo( name="mongo_src", host="atlascluster.ushabcd.mongodb.net", @@ -434,6 +506,22 @@ class UserInfoDataset: password="password", ) +mongo_with_secret = Mongo( + name="mongo_src", + host="atlascluster.ushabcd.mongodb.net", + db_name="mongo", + username=aws_secret["mongo_username"], + password=aws_secret["mongo_password"], +) + +mongo_with_only_password_secret = Mongo( + name="mongo_src", + host="atlascluster.ushabcd.mongodb.net", + db_name="mongo", + username="username", + password=aws_secret["mongo_password"], +) + pubsub = PubSub( name="pubsub_src", project_id="test_project", @@ -448,6 +536,12 @@ class UserInfoDataset: }, ) +pubsub_with_secret = PubSub( + name="pubsub_src", + project_id="test_project", + service_account_key=aws_secret["pubsub_service_account_key"], +) + def test_env_selector_on_connector(): @meta(owner="test@test.com") @@ -861,23 +955,17 @@ def create_user_transactions(cls, dataset: Dataset): ) -def test_multiple_sources(): +def test_multiple_sources_mysql(): @meta(owner="test@test.com") @source( - s3.bucket( - bucket_name="all_ratings", - prefix="prod/apac/", - presorted=True, - format="delta", - ), + mysql.table("users_mysql", cursor="added_on"), every="1h", - disorder="2d", - cdc="native", + disorder="14d", + cdc="upsert", since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), - until=datetime.strptime("2022-02-28T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), ) @dataset - class UserInfoDatasetS3: + class UserInfoDatasetMySql: user_id: int = field(key=True) name: str gender: str @@ -888,40 +976,35 @@ class UserInfoDatasetS3: country: Optional[str] timestamp: datetime = field(timestamp=True) + # mysql source view = InternalTestClient() - view.add(UserInfoDatasetS3) + view.add(UserInfoDatasetMySql) sync_request = view._get_sync_request_proto() - assert len(sync_request.datasets) == 1 - assert len(sync_request.sources) == 1 - assert len(sync_request.extdbs) == 1 - - # s3 source source_request = sync_request.sources[0] s = { "table": { - "s3Table": { - "bucket": "all_ratings", - "pathPrefix": "prod/apac/", - "pathSuffix": "", - "format": "delta", - "preSorted": True, + "mysql_table": { "db": { - "name": "ratings_source", - "s3": { - "awsSecretAccessKey": "8YCvIs8f0+FAKESECRETKEY+7uYSDmq164v9hNjOIIi3q1uV8rv", - "awsAccessKeyId": "ALIAQOTFAKEACCCESSKEYIDGTAXJY6MZWLP", + "name": "mysql", + "mysql": { + "host": "localhost", + "database": "test", + "user": "root", + "password": "root", + "port": 3306, }, }, - } + "table_name": "users_mysql", + }, }, - "dataset": "UserInfoDatasetS3", + "dataset": "UserInfoDatasetMySql", "dsVersion": 1, "every": "3600s", - "cdc": "Native", - "disorder": "172800s", + "cdc": "Upsert", + "disorder": "1209600s", + "cursor": "added_on", + "timestampField": "timestamp", "startingFrom": "2021-08-10T00:00:00Z", - "until": "2022-02-28T00:00:00Z", - "timestamp_field": "timestamp", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( @@ -929,10 +1012,13 @@ class UserInfoDatasetS3: ) extdb_request = sync_request.extdbs[0] e = { - "name": "ratings_source", - "s3": { - "awsSecretAccessKey": "8YCvIs8f0+FAKESECRETKEY+7uYSDmq164v9hNjOIIi3q1uV8rv", - "awsAccessKeyId": "ALIAQOTFAKEACCCESSKEYIDGTAXJY6MZWLP", + "name": "mysql", + "mysql": { + "host": "localhost", + "database": "test", + "user": "root", + "password": "root", + "port": 3306, }, } expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) @@ -942,13 +1028,14 @@ class UserInfoDatasetS3: @meta(owner="test@test.com") @source( - snowflake.table("users_Sf", cursor="added_on"), + mysql_with_secret.table("users_mysql", cursor="added_on"), + every="1h", disorder="14d", cdc="upsert", - every="1h", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), ) @dataset - class UserInfoDatasetSnowFlake: + class UserInfoDatasetMySqlWithSecret: user_id: int = field(key=True) name: str gender: str @@ -959,36 +1046,43 @@ class UserInfoDatasetSnowFlake: country: Optional[str] timestamp: datetime = field(timestamp=True) - # snowflake source + # mysql source with secret view = InternalTestClient() - view.add(UserInfoDatasetSnowFlake) + view.add(UserInfoDatasetMySqlWithSecret) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] s = { "table": { - "snowflakeTable": { + "mysql_table": { "db": { - "snowflake": { - "account": "nhb38793.us-west-2.snowflakecomputing.com", - "user": "", - "password": "", - "schema": "PUBLIC", - "warehouse": "TEST", - "role": "ACCOUNTADMIN", - "database": "MOVIELENS", + "name": "mysql", + "mysql": { + "host": "localhost", + "database": "test", + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mysql_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mysql_password"], + }, + "port": 3306, }, - "name": "snowflake_src", }, - "tableName": "users_Sf", - } + "table_name": "users_mysql", + }, }, - "dataset": "UserInfoDatasetSnowFlake", + "dataset": "UserInfoDatasetMySqlWithSecret", "dsVersion": 1, "every": "3600s", "cdc": "Upsert", "disorder": "1209600s", "cursor": "added_on", "timestampField": "timestamp", + "startingFrom": "2021-08-10T00:00:00Z", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( @@ -996,15 +1090,21 @@ class UserInfoDatasetSnowFlake: ) extdb_request = sync_request.extdbs[0] e = { - "name": "snowflake_src", - "snowflake": { - "account": "nhb38793.us-west-2.snowflakecomputing.com", - "user": "", - "password": "", - "schema": "PUBLIC", - "warehouse": "TEST", - "role": "ACCOUNTADMIN", - "database": "MOVIELENS", + "name": "mysql", + "mysql": { + "host": "localhost", + "database": "test", + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mysql_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mysql_password"], + }, + "port": 3306, }, } expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) @@ -1012,16 +1112,18 @@ class UserInfoDatasetSnowFlake: extdb_request, expected_extdb_request ) + +def test_multiple_sources_postgres(): @meta(owner="test@test.com") @source( - snowflake.table("users_Sf", cursor="added_on"), + postgres.table("users_postgres", cursor="added_on"), every="1h", disorder="14d", cdc="upsert", - until=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), ) @dataset - class UserInfoDatasetSnowFlakeStartingFrom: + class UserInfoDatasetPostgres: user_id: int = field(key=True) name: str gender: str @@ -1032,37 +1134,35 @@ class UserInfoDatasetSnowFlakeStartingFrom: country: Optional[str] timestamp: datetime = field(timestamp=True) - # snowflake source + # postgres source view = InternalTestClient() - view.add(UserInfoDatasetSnowFlakeStartingFrom) + view.add(UserInfoDatasetPostgres) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] s = { "table": { - "snowflakeTable": { + "pg_table": { "db": { - "snowflake": { - "account": "nhb38793.us-west-2.snowflakecomputing.com", - "user": "", - "password": "", - "schema": "PUBLIC", - "warehouse": "TEST", - "role": "ACCOUNTADMIN", - "database": "MOVIELENS", + "name": "postgres", + "postgres": { + "host": "localhost", + "database": "test", + "user": "root", + "password": "root", + "port": 5432, }, - "name": "snowflake_src", }, - "tableName": "users_Sf", - } + "table_name": "users_postgres", + }, }, - "dataset": "UserInfoDatasetSnowFlakeStartingFrom", + "dataset": "UserInfoDatasetPostgres", "dsVersion": 1, "every": "3600s", "cdc": "Upsert", "disorder": "1209600s", "cursor": "added_on", "timestampField": "timestamp", - "until": "2021-08-10T00:00:00Z", + "startingFrom": "2021-08-10T00:00:00Z", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( @@ -1070,15 +1170,13 @@ class UserInfoDatasetSnowFlakeStartingFrom: ) extdb_request = sync_request.extdbs[0] e = { - "name": "snowflake_src", - "snowflake": { - "account": "nhb38793.us-west-2.snowflakecomputing.com", - "user": "", - "password": "", - "schema": "PUBLIC", - "warehouse": "TEST", - "role": "ACCOUNTADMIN", - "database": "MOVIELENS", + "name": "postgres", + "postgres": { + "host": "localhost", + "database": "test", + "user": "root", + "password": "root", + "port": 5432, }, } expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) @@ -1088,14 +1186,14 @@ class UserInfoDatasetSnowFlakeStartingFrom: @meta(owner="test@test.com") @source( - bigquery.table("users_bq", cursor="added_on"), + postgres_with_secret.table("users_postgres", cursor="added_on"), every="1h", - disorder="2h", + disorder="14d", cdc="upsert", since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), ) @dataset - class UserInfoDatasetBigQuery: + class UserInfoDatasetPostgresWithSecret: user_id: int = field(key=True) name: str gender: str @@ -1106,36 +1204,40 @@ class UserInfoDatasetBigQuery: country: Optional[str] timestamp: datetime = field(timestamp=True) - # bigquery source + # postgres source with secret view = InternalTestClient() - view.add(UserInfoDatasetBigQuery) + view.add(UserInfoDatasetPostgresWithSecret) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] s = { "table": { - "bigqueryTable": { + "pg_table": { "db": { - "name": "bq_movie_tags", - "bigquery": { - "datasetId": "movie_tags", - "serviceAccountKey": '{"type": "service_account", "project_id": "fake-project-356105", ' - '"client_email": ' - '"randomstring@fake-project-356105.iam.gserviceaccount.com", ' - '"client_id": "103688493243243272951", "auth_uri": ' - '"https://accounts.google.com/o/oauth2/auth", "token_uri": ' - '"https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": ' - '"https://www.googleapis.com/oauth2/v1/certs"}', - "projectId": "gold-cocoa-356105", + "name": "postgres", + "postgres": { + "host": "localhost", + "database": "test", + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["postgres_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["postgres_password"], + }, + "port": 5432, }, }, - "tableName": "users_bq", - } + "table_name": "users_postgres", + }, }, - "dataset": "UserInfoDatasetBigQuery", + "dataset": "UserInfoDatasetPostgresWithSecret", "dsVersion": 1, "every": "3600s", "cdc": "Upsert", - "disorder": "7200s", + "disorder": "1209600s", "cursor": "added_on", "timestampField": "timestamp", "startingFrom": "2021-08-10T00:00:00Z", @@ -1146,15 +1248,21 @@ class UserInfoDatasetBigQuery: ) extdb_request = sync_request.extdbs[0] e = { - "name": "bq_movie_tags", - "bigquery": { - "datasetId": "movie_tags", - "serviceAccountKey": '{"type": "service_account", "project_id": "fake-project-356105", "client_email": ' - '"randomstring@fake-project-356105.iam.gserviceaccount.com", "client_id": ' - '"103688493243243272951", "auth_uri": "https://accounts.google.com/o/oauth2/auth", ' - '"token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": ' - '"https://www.googleapis.com/oauth2/v1/certs"}', - "projectId": "gold-cocoa-356105", + "name": "postgres", + "postgres": { + "host": "localhost", + "database": "test", + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["postgres_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["postgres_password"], + }, + "port": 5432, }, } expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) @@ -1162,16 +1270,17 @@ class UserInfoDatasetBigQuery: extdb_request, expected_extdb_request ) + +def test_multiple_sources_mongo(): @meta(owner="test@test.com") @source( - mysql.table("users_mysql", cursor="added_on"), - every="1h", + mongo.collection("test_table", cursor="added_on"), disorder="14d", cdc="upsert", - since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + every="1h", ) @dataset - class UserInfoDatasetMySql: + class UserInfoDatasetMongo: user_id: int = field(key=True) name: str gender: str @@ -1182,35 +1291,34 @@ class UserInfoDatasetMySql: country: Optional[str] timestamp: datetime = field(timestamp=True) - # mysql source + # mongo source view = InternalTestClient() - view.add(UserInfoDatasetMySql) + view.add(UserInfoDatasetMongo) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] + s = { "table": { - "mysql_table": { + "mongoCollection": { "db": { - "name": "mysql", - "mysql": { - "host": "localhost", - "database": "test", - "user": "root", - "password": "root", - "port": 3306, + "mongo": { + "host": "atlascluster.ushabcd.mongodb.net", + "database": "mongo", + "user": "username", + "password": "password", }, + "name": "mongo_src", }, - "table_name": "users_mysql", - }, + "collectionName": "test_table", + } }, - "dataset": "UserInfoDatasetMySql", + "dataset": "UserInfoDatasetMongo", "dsVersion": 1, "every": "3600s", "cdc": "Upsert", "disorder": "1209600s", "cursor": "added_on", "timestampField": "timestamp", - "startingFrom": "2021-08-10T00:00:00Z", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( @@ -1218,13 +1326,12 @@ class UserInfoDatasetMySql: ) extdb_request = sync_request.extdbs[0] e = { - "name": "mysql", - "mysql": { - "host": "localhost", - "database": "test", - "user": "root", - "password": "root", - "port": 3306, + "name": "mongo_src", + "mongo": { + "host": "atlascluster.ushabcd.mongodb.net", + "database": "mongo", + "user": "username", + "password": "password", }, } expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) @@ -1234,13 +1341,13 @@ class UserInfoDatasetMySql: @meta(owner="test@test.com") @source( - kafka.topic("test_topic"), + mongo_with_secret.collection("test_table", cursor="added_on"), disorder="14d", cdc="upsert", - since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + every="1h", ) @dataset - class UserInfoDatasetKafka: + class UserInfoDatasetMongoWithSecret: user_id: int = field(key=True) name: str gender: str @@ -1251,59 +1358,81 @@ class UserInfoDatasetKafka: country: Optional[str] timestamp: datetime = field(timestamp=True) + # mongo source with secret view = InternalTestClient() - view.add(UserInfoDatasetKafka) + view.add(UserInfoDatasetMongoWithSecret) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] + s = { "table": { - "kafkaTopic": { + "mongoCollection": { "db": { - "name": "kafka_src", - "kafka": { - "bootstrapServers": "localhost:9092", - "securityProtocol": "PLAINTEXT", - "saslMechanism": "PLAIN", - "saslPlainUsername": "test", - "saslPlainPassword": "test", + "mongo": { + "host": "atlascluster.ushabcd.mongodb.net", + "database": "mongo", + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mongo_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mongo_password"], + }, }, + "name": "mongo_src", }, - "topic": "test_topic", - "format": { - "json": {}, - }, + "collectionName": "test_table", } }, - "dataset": "UserInfoDatasetKafka", + "dataset": "UserInfoDatasetMongoWithSecret", "dsVersion": 1, + "every": "3600s", "cdc": "Upsert", "disorder": "1209600s", - "startingFrom": "2021-08-10T00:00:00Z", + "cursor": "added_on", + "timestampField": "timestamp", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( source_request, expected_source_request ) - - avro = Avro( - registry="confluent", - url="http://localhost:8000", - username="user", - password="pwd", + extdb_request = sync_request.extdbs[0] + e = { + "name": "mongo_src", + "mongo": { + "host": "atlascluster.ushabcd.mongodb.net", + "database": "mongo", + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mongo_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mongo_password"], + }, + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request ) @meta(owner="test@test.com") @source( - kafka.topic( - "test_topic", - format=avro, + mongo_with_only_password_secret.collection( + "test_table", cursor="added_on" ), - cdc="debezium", disorder="14d", - since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + cdc="upsert", + every="1h", ) @dataset - class UserInfoDatasetKafka: + class UserInfoDatasetMongoWithPasswordSecret: user_id: int = field(key=True) name: str gender: str @@ -1314,66 +1443,73 @@ class UserInfoDatasetKafka: country: Optional[str] timestamp: datetime = field(timestamp=True) + # mongo source with only password secret view = InternalTestClient() - view.add(UserInfoDatasetKafka) + view.add(UserInfoDatasetMongoWithPasswordSecret) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] + s = { "table": { - "kafkaTopic": { + "mongoCollection": { "db": { - "name": "kafka_src", - "kafka": { - "bootstrapServers": "localhost:9092", - "securityProtocol": "PLAINTEXT", - "saslMechanism": "PLAIN", - "saslPlainUsername": "test", - "saslPlainPassword": "test", + "mongo": { + "host": "atlascluster.ushabcd.mongodb.net", + "database": "mongo", + "user": "username", + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mongo_password"], + }, }, + "name": "mongo_src", }, - "topic": "test_topic", - "format": { - "avro": { - "schemaRegistry": { - "url": "http://localhost:8000", - "auth": { - "basic": {"username": "user", "password": "pwd"} - }, - } - } - }, + "collectionName": "test_table", } }, - "dataset": "UserInfoDatasetKafka", + "dataset": "UserInfoDatasetMongoWithPasswordSecret", "dsVersion": 1, + "every": "3600s", + "cdc": "Upsert", "disorder": "1209600s", - "cdc": "Debezium", - "startingFrom": "2021-08-10T00:00:00Z", + "cursor": "added_on", + "timestampField": "timestamp", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( source_request, expected_source_request ) - - protobuf = Protobuf( - registry="confluent", - url="http://localhost:8000", - username="user", - password="pwd", + extdb_request = sync_request.extdbs[0] + e = { + "name": "mongo_src", + "mongo": { + "host": "atlascluster.ushabcd.mongodb.net", + "database": "mongo", + "user": "username", + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["mongo_password"], + }, + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request ) + +def test_multiple_sources_redshift(): @meta(owner="test@test.com") @source( - kafka.topic( - "test_topic", - format=protobuf, - ), - cdc="debezium", + redshift.table("test_table", cursor="added_on"), disorder="14d", - since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + cdc="upsert", + every="1h", ) @dataset - class UserInfoDatasetKafka: + class UserInfoDatasetRedshift: user_id: int = field(key=True) name: str gender: str @@ -1384,56 +1520,68 @@ class UserInfoDatasetKafka: country: Optional[str] timestamp: datetime = field(timestamp=True) + # redshift source view = InternalTestClient() - view.add(UserInfoDatasetKafka) + view.add(UserInfoDatasetRedshift) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] s = { "table": { - "kafkaTopic": { + "redshiftTable": { "db": { - "name": "kafka_src", - "kafka": { - "bootstrapServers": "localhost:9092", - "securityProtocol": "PLAINTEXT", - "saslMechanism": "PLAIN", - "saslPlainUsername": "test", - "saslPlainPassword": "test", + "redshift": { + "redshiftAuthentication": { + "s3_access_role_arn": "arn:aws:iam::123:role/Redshift" + }, + "database": "test", + "host": "test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", + "port": 5439, + "schema": "public", }, + "name": "redshift_src", }, - "topic": "test_topic", - "format": { - "protobuf": { - "schemaRegistry": { - "url": "http://localhost:8000", - "auth": { - "basic": {"username": "user", "password": "pwd"} - }, - } - } - }, + "tableName": "test_table", } }, - "dataset": "UserInfoDatasetKafka", + "dataset": "UserInfoDatasetRedshift", "dsVersion": 1, + "every": "3600s", + "cdc": "Upsert", "disorder": "1209600s", - "cdc": "Debezium", - "startingFrom": "2021-08-10T00:00:00Z", + "cursor": "added_on", + "timestampField": "timestamp", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( source_request, expected_source_request ) + extdb_request = sync_request.extdbs[0] + e = { + "name": "redshift_src", + "redshift": { + "redshiftAuthentication": { + "s3_access_role_arn": "arn:aws:iam::123:role/Redshift" + }, + "database": "test", + "host": "test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", + "port": 5439, + "schema": "public", + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request + ) @meta(owner="test@test.com") @source( - kinesis.stream("test_stream", init_position="latest", format="json"), + redshift2.table("test_table", cursor="added_on"), disorder="14d", cdc="upsert", - since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + every="1h", ) @dataset - class UserInfoDatasetKinesis: + class UserInfoDatasetRedshiftUsingCreds: user_id: int = field(key=True) name: str gender: str @@ -1444,148 +1592,71 @@ class UserInfoDatasetKinesis: country: Optional[str] timestamp: datetime = field(timestamp=True) + # redshift source view = InternalTestClient() - view.add(UserInfoDatasetKinesis) + view.add(UserInfoDatasetRedshiftUsingCreds) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] s = { "table": { - "kinesisStream": { - "streamArn": "test_stream", - "initPosition": "LATEST", - "format": "json", + "redshiftTable": { "db": { - "name": "kinesis_src", - "kinesis": { - "roleArn": "arn:aws:iam::123456789012:role/test-role" + "redshift": { + "database": "test", + "host": "test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", + "port": 5439, + "schema": "public", + "redshiftAuthentication": { + "credentials": { + "username": "username", + "password": "password", + } + }, }, + "name": "redshift_src_2", }, + "tableName": "test_table", } }, - "dataset": "UserInfoDatasetKinesis", + "dataset": "UserInfoDatasetRedshiftUsingCreds", "dsVersion": 1, + "every": "3600s", "cdc": "Upsert", "disorder": "1209600s", - "startingFrom": "2021-08-10T00:00:00Z", + "cursor": "added_on", + "timestampField": "timestamp", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( source_request, expected_source_request ) - - @meta(owner="test@test.com") - @source( - kinesis.stream( - "test_stream", - init_position="2023-05-31 15:30:00", - format="json", - ), - disorder="14d", - cdc="upsert", - since=datetime(2023, 5, 31), - ) - @dataset - class UserInfoDatasetKinesis: - user_id: int = field(key=True) - name: str - gender: str - timestamp: datetime = field(timestamp=True) - - @meta(owner="test@test.com") - @source( - kinesis.stream( - "test_stream2", - init_position=datetime(2023, 5, 31), - format="json", - ), - disorder="14d", - cdc="upsert", - ) - @dataset - class UserInfoDatasetKinesis2: - user_id: int = field(key=True) - name: str - gender: str - timestamp: datetime = field(timestamp=True) - - view = InternalTestClient() - view.add(UserInfoDatasetKinesis) - view.add(UserInfoDatasetKinesis2) - sync_request = view._get_sync_request_proto() - extdb_request = sync_request.extdbs[0] e = { - "name": "kinesis_src", - "kinesis": {"roleArn": "arn:aws:iam::123456789012:role/test-role"}, + "name": "redshift_src_2", + "redshift": { + "database": "test", + "host": "test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", + "port": 5439, + "schema": "public", + "redshiftAuthentication": { + "credentials": {"username": "username", "password": "password"} + }, + }, } - expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) assert extdb_request == expected_extdb_request, error_message( extdb_request, expected_extdb_request ) - source_req = sync_request.sources[0] - e = { - "table": { - "kinesisStream": { - "streamArn": "test_stream", - "initPosition": "AT_TIMESTAMP", - "initTimestamp": "2023-05-31T15:30:00Z", - "format": "json", - "db": { - "name": "kinesis_src", - "kinesis": { - "roleArn": "arn:aws:iam::123456789012:role/test-role" - }, - }, - } - }, - "dataset": "UserInfoDatasetKinesis", - "dsVersion": 1, - "cdc": "Upsert", - "disorder": "1209600s", - "startingFrom": "2023-05-31T00:00:00Z", - } - expected_source = ParseDict(e, connector_proto.Source()) - assert source_req == expected_source, error_message( - source_req, expected_source - ) - - source_req = sync_request.sources[1] - e = { - "table": { - "kinesisStream": { - "streamArn": "test_stream2", - "initPosition": "AT_TIMESTAMP", - "initTimestamp": "2023-05-31T00:00:00Z", - "format": "json", - "db": { - "name": "kinesis_src", - "kinesis": { - "roleArn": "arn:aws:iam::123456789012:role/test-role" - }, - }, - } - }, - "dataset": "UserInfoDatasetKinesis2", - "dsVersion": 1, - "cdc": "Upsert", - "disorder": "1209600s", - } - expected_source = ParseDict(e, connector_proto.Source()) - assert source_req == expected_source, error_message( - source_req, expected_source - ) - @meta(owner="test@test.com") @source( - redshift.table("test_table", cursor="added_on"), + redshift_with_secret.table("test_table", cursor="added_on"), disorder="14d", cdc="upsert", every="1h", ) @dataset - class UserInfoDatasetRedshift: + class UserInfoDatasetRedshiftUsingCredsWithSecret: user_id: int = field(key=True) name: str gender: str @@ -1598,7 +1669,7 @@ class UserInfoDatasetRedshift: # redshift source view = InternalTestClient() - view.add(UserInfoDatasetRedshift) + view.add(UserInfoDatasetRedshiftUsingCredsWithSecret) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] s = { @@ -1606,20 +1677,31 @@ class UserInfoDatasetRedshift: "redshiftTable": { "db": { "redshift": { - "redshiftAuthentication": { - "s3_access_role_arn": "arn:aws:iam::123:role/Redshift" - }, "database": "test", "host": "test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", "port": 5439, "schema": "public", + "redshiftAuthentication": { + "credentials": { + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["redshift_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["redshift_password"], + }, + } + }, }, - "name": "redshift_src", + "name": "redshift_src_2", }, "tableName": "test_table", } }, - "dataset": "UserInfoDatasetRedshift", + "dataset": "UserInfoDatasetRedshiftUsingCredsWithSecret", "dsVersion": 1, "every": "3600s", "cdc": "Upsert", @@ -1633,15 +1715,26 @@ class UserInfoDatasetRedshift: ) extdb_request = sync_request.extdbs[0] e = { - "name": "redshift_src", + "name": "redshift_src_2", "redshift": { - "redshiftAuthentication": { - "s3_access_role_arn": "arn:aws:iam::123:role/Redshift" - }, "database": "test", "host": "test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", "port": 5439, "schema": "public", + "redshiftAuthentication": { + "credentials": { + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["redshift_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["redshift_password"], + }, + } + }, }, } expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) @@ -1649,15 +1742,19 @@ class UserInfoDatasetRedshift: extdb_request, expected_extdb_request ) + +def test_multiple_sources_snowflake(): + @meta(owner="test@test.com") @source( - redshift2.table("test_table", cursor="added_on"), + snowflake.table("users_Sf", cursor="added_on"), + every="1h", disorder="14d", cdc="upsert", - every="1h", + until=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), ) @dataset - class UserInfoDatasetRedshiftUsingCreds: + class UserInfoDatasetSnowFlakeStartingFrom: user_id: int = field(key=True) name: str gender: str @@ -1668,39 +1765,37 @@ class UserInfoDatasetRedshiftUsingCreds: country: Optional[str] timestamp: datetime = field(timestamp=True) - # redshift source + # snowflake source view = InternalTestClient() - view.add(UserInfoDatasetRedshiftUsingCreds) + view.add(UserInfoDatasetSnowFlakeStartingFrom) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] s = { "table": { - "redshiftTable": { + "snowflakeTable": { "db": { - "redshift": { - "database": "test", - "host": "test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", - "port": 5439, - "schema": "public", - "redshiftAuthentication": { - "credentials": { - "username": "username", - "password": "password", - } - }, + "snowflake": { + "account": "nhb38793.us-west-2.snowflakecomputing.com", + "user": "", + "password": "", + "schema": "PUBLIC", + "warehouse": "TEST", + "role": "ACCOUNTADMIN", + "database": "MOVIELENS", }, - "name": "redshift_src_2", + "name": "snowflake_src", }, - "tableName": "test_table", + "tableName": "users_Sf", } }, - "dataset": "UserInfoDatasetRedshiftUsingCreds", + "dataset": "UserInfoDatasetSnowFlakeStartingFrom", "dsVersion": 1, "every": "3600s", "cdc": "Upsert", "disorder": "1209600s", "cursor": "added_on", "timestampField": "timestamp", + "until": "2021-08-10T00:00:00Z", } expected_source_request = ParseDict(s, connector_proto.Source()) assert source_request == expected_source_request, error_message( @@ -1708,15 +1803,15 @@ class UserInfoDatasetRedshiftUsingCreds: ) extdb_request = sync_request.extdbs[0] e = { - "name": "redshift_src_2", - "redshift": { - "database": "test", - "host": "test-workgroup.1234.us-west-2.redshift-serverless.amazonaws.com", - "port": 5439, - "schema": "public", - "redshiftAuthentication": { - "credentials": {"username": "username", "password": "password"} - }, + "name": "snowflake_src", + "snowflake": { + "account": "nhb38793.us-west-2.snowflakecomputing.com", + "user": "", + "password": "", + "schema": "PUBLIC", + "warehouse": "TEST", + "role": "ACCOUNTADMIN", + "database": "MOVIELENS", }, } expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) @@ -1726,13 +1821,13 @@ class UserInfoDatasetRedshiftUsingCreds: @meta(owner="test@test.com") @source( - mongo.collection("test_table", cursor="added_on"), + snowflake.table("users_Sf", cursor="added_on"), disorder="14d", cdc="upsert", every="1h", ) @dataset - class UserInfoDatasetMongo: + class UserInfoDatasetSnowFlake: user_id: int = field(key=True) name: str gender: str @@ -1743,28 +1838,30 @@ class UserInfoDatasetMongo: country: Optional[str] timestamp: datetime = field(timestamp=True) - # mongo source + # snowflake source view = InternalTestClient() - view.add(UserInfoDatasetMongo) + view.add(UserInfoDatasetSnowFlake) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] - s = { "table": { - "mongoCollection": { + "snowflakeTable": { "db": { - "mongo": { - "host": "atlascluster.ushabcd.mongodb.net", - "database": "mongo", - "user": "username", - "password": "password", + "snowflake": { + "account": "nhb38793.us-west-2.snowflakecomputing.com", + "user": "", + "password": "", + "schema": "PUBLIC", + "warehouse": "TEST", + "role": "ACCOUNTADMIN", + "database": "MOVIELENS", }, - "name": "mongo_src", + "name": "snowflake_src", }, - "collectionName": "test_table", + "tableName": "users_Sf", } }, - "dataset": "UserInfoDatasetMongo", + "dataset": "UserInfoDatasetSnowFlake", "dsVersion": 1, "every": "3600s", "cdc": "Upsert", @@ -1778,12 +1875,15 @@ class UserInfoDatasetMongo: ) extdb_request = sync_request.extdbs[0] e = { - "name": "mongo_src", - "mongo": { - "host": "atlascluster.ushabcd.mongodb.net", - "database": "mongo", - "user": "username", - "password": "password", + "name": "snowflake_src", + "snowflake": { + "account": "nhb38793.us-west-2.snowflakecomputing.com", + "user": "", + "password": "", + "schema": "PUBLIC", + "warehouse": "TEST", + "role": "ACCOUNTADMIN", + "database": "MOVIELENS", }, } expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) @@ -1793,13 +1893,13 @@ class UserInfoDatasetMongo: @meta(owner="test@test.com") @source( - pubsub.topic("test_topic", format="json"), + snowflake_with_secret.table("users_Sf", cursor="added_on"), disorder="14d", cdc="upsert", every="1h", ) @dataset - class UserInfoDatasetPubSub: + class UserInfoDatasetSnowFlakeWithSecret: user_id: int = field(key=True) name: str gender: str @@ -1810,24 +1910,1018 @@ class UserInfoDatasetPubSub: country: Optional[str] timestamp: datetime = field(timestamp=True) - # mongo source + # snowflake source with secret view = InternalTestClient() - view.add(UserInfoDatasetPubSub) + view.add(UserInfoDatasetSnowFlakeWithSecret) sync_request = view._get_sync_request_proto() source_request = sync_request.sources[0] - s = { "table": { - "pubsubTopic": { + "snowflakeTable": { "db": { - "pubsub": { - "projectId": "test_project", - "serviceAccountKey": '{"type": "service_account", "project_id": "fake-project-356105", ' - '"client_email": ' - '"randomstring@fake-project-356105.iam.gserviceaccount.com", "client_id": ' - '"103688493243243272951", "auth_uri": ' - '"https://accounts.google.com/o/oauth2/auth", "token_uri": ' - '"https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": ' + "snowflake": { + "account": "nhb38793.us-west-2.snowflakecomputing.com", + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["us-west-2", "username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["us-west-2", "password"], + }, + "schema": "PUBLIC", + "warehouse": "TEST", + "role": "ACCOUNTADMIN", + "database": "MOVIELENS", + }, + "name": "snowflake_src", + }, + "tableName": "users_Sf", + } + }, + "dataset": "UserInfoDatasetSnowFlakeWithSecret", + "dsVersion": 1, + "every": "3600s", + "cdc": "Upsert", + "disorder": "1209600s", + "cursor": "added_on", + "timestampField": "timestamp", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + extdb_request = sync_request.extdbs[0] + e = { + "name": "snowflake_src", + "snowflake": { + "account": "nhb38793.us-west-2.snowflakecomputing.com", + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["us-west-2", "username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["us-west-2", "password"], + }, + "schema": "PUBLIC", + "warehouse": "TEST", + "role": "ACCOUNTADMIN", + "database": "MOVIELENS", + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request + ) + + +def test_multiple_sources_bigquery(): + @meta(owner="test@test.com") + @source( + bigquery.table("users_bq", cursor="added_on"), + every="1h", + disorder="2h", + cdc="upsert", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetBigQuery: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + # bigquery source + view = InternalTestClient() + view.add(UserInfoDatasetBigQuery) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "bigqueryTable": { + "db": { + "name": "bq_movie_tags", + "bigquery": { + "datasetId": "movie_tags", + "serviceAccountKey": '{"type": "service_account", "project_id": "fake-project-356105", ' + '"client_email": ' + '"randomstring@fake-project-356105.iam.gserviceaccount.com", ' + '"client_id": "103688493243243272951", "auth_uri": ' + '"https://accounts.google.com/o/oauth2/auth", "token_uri": ' + '"https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": ' + '"https://www.googleapis.com/oauth2/v1/certs"}', + "projectId": "gold-cocoa-356105", + }, + }, + "tableName": "users_bq", + } + }, + "dataset": "UserInfoDatasetBigQuery", + "dsVersion": 1, + "every": "3600s", + "cdc": "Upsert", + "disorder": "7200s", + "cursor": "added_on", + "timestampField": "timestamp", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + extdb_request = sync_request.extdbs[0] + e = { + "name": "bq_movie_tags", + "bigquery": { + "datasetId": "movie_tags", + "serviceAccountKey": '{"type": "service_account", "project_id": "fake-project-356105", "client_email": ' + '"randomstring@fake-project-356105.iam.gserviceaccount.com", "client_id": ' + '"103688493243243272951", "auth_uri": "https://accounts.google.com/o/oauth2/auth", ' + '"token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": ' + '"https://www.googleapis.com/oauth2/v1/certs"}', + "projectId": "gold-cocoa-356105", + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request + ) + + @meta(owner="test@test.com") + @source( + bigquery_with_secret.table("users_bq", cursor="added_on"), + every="1h", + disorder="2h", + cdc="upsert", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetBigQueryWithSecret: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + # bigquery source + view = InternalTestClient() + view.add(UserInfoDatasetBigQueryWithSecret) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "bigqueryTable": { + "db": { + "name": "bq_movie_tags", + "bigquery": { + "datasetId": "movie_tags", + "serviceAccountKeySecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["bigquery_service_account_key"], + }, + "projectId": "gold-cocoa-356105", + }, + }, + "tableName": "users_bq", + } + }, + "dataset": "UserInfoDatasetBigQueryWithSecret", + "dsVersion": 1, + "every": "3600s", + "cdc": "Upsert", + "disorder": "7200s", + "cursor": "added_on", + "timestampField": "timestamp", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + extdb_request = sync_request.extdbs[0] + e = { + "name": "bq_movie_tags", + "bigquery": { + "datasetId": "movie_tags", + "serviceAccountKeySecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["bigquery_service_account_key"], + }, + "projectId": "gold-cocoa-356105", + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request + ) + + +def test_multiple_sources_kafka(): + @meta(owner="test@test.com") + @source( + kafka.topic("test_topic"), + disorder="14d", + cdc="upsert", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKafka: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKafka) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kafkaTopic": { + "db": { + "name": "kafka_src", + "kafka": { + "bootstrapServers": "localhost:9092", + "securityProtocol": "PLAINTEXT", + "saslMechanism": "PLAIN", + "saslPlainUsername": "test", + "saslPlainPassword": "test", + }, + }, + "topic": "test_topic", + "format": { + "json": {}, + }, + } + }, + "dataset": "UserInfoDatasetKafka", + "dsVersion": 1, + "cdc": "Upsert", + "disorder": "1209600s", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + avro = Avro( + registry="confluent", + url="http://localhost:8000", + username="user", + password="pwd", + ) + + @meta(owner="test@test.com") + @source( + kafka.topic( + "test_topic", + format=avro, + ), + cdc="debezium", + disorder="14d", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKafka: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKafka) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kafkaTopic": { + "db": { + "name": "kafka_src", + "kafka": { + "bootstrapServers": "localhost:9092", + "securityProtocol": "PLAINTEXT", + "saslMechanism": "PLAIN", + "saslPlainUsername": "test", + "saslPlainPassword": "test", + }, + }, + "topic": "test_topic", + "format": { + "avro": { + "schemaRegistry": { + "url": "http://localhost:8000", + "auth": { + "basic": {"username": "user", "password": "pwd"} + }, + } + } + }, + } + }, + "dataset": "UserInfoDatasetKafka", + "dsVersion": 1, + "disorder": "1209600s", + "cdc": "Debezium", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + protobuf = Protobuf( + registry="confluent", + url="http://localhost:8000", + username="user", + password="pwd", + ) + + @meta(owner="test@test.com") + @source( + kafka.topic( + "test_topic", + format=protobuf, + ), + cdc="debezium", + disorder="14d", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKafka: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKafka) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kafkaTopic": { + "db": { + "name": "kafka_src", + "kafka": { + "bootstrapServers": "localhost:9092", + "securityProtocol": "PLAINTEXT", + "saslMechanism": "PLAIN", + "saslPlainUsername": "test", + "saslPlainPassword": "test", + }, + }, + "topic": "test_topic", + "format": { + "protobuf": { + "schemaRegistry": { + "url": "http://localhost:8000", + "auth": { + "basic": {"username": "user", "password": "pwd"} + }, + } + } + }, + } + }, + "dataset": "UserInfoDatasetKafka", + "dsVersion": 1, + "disorder": "1209600s", + "cdc": "Debezium", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + @meta(owner="test@test.com") + @source( + kafka_with_secret.topic( + "test_topic", + format=protobuf, + ), + cdc="debezium", + disorder="14d", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKafkaWithSecret: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKafkaWithSecret) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kafkaTopic": { + "db": { + "name": "kafka_src", + "kafka": { + "bootstrapServers": "localhost:9092", + "securityProtocol": "PLAINTEXT", + "saslMechanism": "PLAIN", + "saslUsernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["sasl_plain_username"], + }, + "saslPasswordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["sasl_plain_password"], + }, + }, + }, + "topic": "test_topic", + "format": { + "protobuf": { + "schemaRegistry": { + "url": "http://localhost:8000", + "auth": { + "basic": {"username": "user", "password": "pwd"} + }, + } + } + }, + } + }, + "dataset": "UserInfoDatasetKafkaWithSecret", + "dsVersion": 1, + "disorder": "1209600s", + "cdc": "Debezium", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + avro = Avro( + registry="confluent", + url="http://localhost:8000", + username=aws_secret["avro_username"], + password=aws_secret["avro_password"], + ) + + @meta(owner="test@test.com") + @source( + kafka.topic( + "test_topic", + format=avro, + ), + cdc="debezium", + disorder="14d", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKafka: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKafka) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kafkaTopic": { + "db": { + "name": "kafka_src", + "kafka": { + "bootstrapServers": "localhost:9092", + "securityProtocol": "PLAINTEXT", + "saslMechanism": "PLAIN", + "saslPlainUsername": "test", + "saslPlainPassword": "test", + }, + }, + "topic": "test_topic", + "format": { + "avro": { + "schemaRegistry": { + "url": "http://localhost:8000", + "auth": { + "basic": { + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["avro_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["avro_password"], + }, + } + }, + } + } + }, + } + }, + "dataset": "UserInfoDatasetKafka", + "dsVersion": 1, + "disorder": "1209600s", + "cdc": "Debezium", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + protobuf = Protobuf( + registry="confluent", + url="http://localhost:8000", + username=aws_secret["protobuf_username"], + password=aws_secret["protobuf_password"], + ) + + @meta(owner="test@test.com") + @source( + kafka.topic( + "test_topic", + format=protobuf, + ), + cdc="debezium", + disorder="14d", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKafka: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKafka) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kafkaTopic": { + "db": { + "name": "kafka_src", + "kafka": { + "bootstrapServers": "localhost:9092", + "securityProtocol": "PLAINTEXT", + "saslMechanism": "PLAIN", + "saslPlainUsername": "test", + "saslPlainPassword": "test", + }, + }, + "topic": "test_topic", + "format": { + "protobuf": { + "schemaRegistry": { + "url": "http://localhost:8000", + "auth": { + "basic": { + "usernameSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["protobuf_username"], + }, + "passwordSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["protobuf_password"], + }, + } + }, + } + } + }, + } + }, + "dataset": "UserInfoDatasetKafka", + "dsVersion": 1, + "disorder": "1209600s", + "cdc": "Debezium", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + avro = Avro( + registry="confluent", + url="http://localhost:8000", + token=aws_secret["avro_token"], + ) + + @meta(owner="test@test.com") + @source( + kafka.topic( + "test_topic", + format=avro, + ), + cdc="debezium", + disorder="14d", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKafka: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKafka) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kafkaTopic": { + "db": { + "name": "kafka_src", + "kafka": { + "bootstrapServers": "localhost:9092", + "securityProtocol": "PLAINTEXT", + "saslMechanism": "PLAIN", + "saslPlainUsername": "test", + "saslPlainPassword": "test", + }, + }, + "topic": "test_topic", + "format": { + "avro": { + "schemaRegistry": { + "url": "http://localhost:8000", + "auth": { + "token": { + "tokenSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["avro_token"], + } + } + }, + } + } + }, + } + }, + "dataset": "UserInfoDatasetKafka", + "dsVersion": 1, + "disorder": "1209600s", + "cdc": "Debezium", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + protobuf = Protobuf( + registry="confluent", + url="http://localhost:8000", + token=aws_secret["protobuf_token"], + ) + + @meta(owner="test@test.com") + @source( + kafka.topic( + "test_topic", + format=protobuf, + ), + cdc="debezium", + disorder="14d", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKafka: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKafka) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kafkaTopic": { + "db": { + "name": "kafka_src", + "kafka": { + "bootstrapServers": "localhost:9092", + "securityProtocol": "PLAINTEXT", + "saslMechanism": "PLAIN", + "saslPlainUsername": "test", + "saslPlainPassword": "test", + }, + }, + "topic": "test_topic", + "format": { + "protobuf": { + "schemaRegistry": { + "url": "http://localhost:8000", + "auth": { + "token": { + "tokenSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["protobuf_token"], + } + } + }, + } + } + }, + } + }, + "dataset": "UserInfoDatasetKafka", + "dsVersion": 1, + "disorder": "1209600s", + "cdc": "Debezium", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + +def test_multiple_sources_s3(): + @meta(owner="test@test.com") + @source( + s3.bucket( + bucket_name="all_ratings", + prefix="prod/apac/", + presorted=True, + format="delta", + ), + every="1h", + disorder="2d", + cdc="native", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + until=datetime.strptime("2022-02-28T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetS3: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetS3) + sync_request = view._get_sync_request_proto() + assert len(sync_request.datasets) == 1 + assert len(sync_request.sources) == 1 + assert len(sync_request.extdbs) == 1 + + # s3 source + source_request = sync_request.sources[0] + s = { + "table": { + "s3Table": { + "bucket": "all_ratings", + "pathPrefix": "prod/apac/", + "pathSuffix": "", + "format": "delta", + "preSorted": True, + "db": { + "name": "ratings_source", + "s3": { + "awsSecretAccessKey": "8YCvIs8f0+FAKESECRETKEY+7uYSDmq164v9hNjOIIi3q1uV8rv", + "awsAccessKeyId": "ALIAQOTFAKEACCCESSKEYIDGTAXJY6MZWLP", + }, + }, + } + }, + "dataset": "UserInfoDatasetS3", + "dsVersion": 1, + "every": "3600s", + "cdc": "Native", + "disorder": "172800s", + "startingFrom": "2021-08-10T00:00:00Z", + "until": "2022-02-28T00:00:00Z", + "timestamp_field": "timestamp", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + extdb_request = sync_request.extdbs[0] + e = { + "name": "ratings_source", + "s3": { + "awsSecretAccessKey": "8YCvIs8f0+FAKESECRETKEY+7uYSDmq164v9hNjOIIi3q1uV8rv", + "awsAccessKeyId": "ALIAQOTFAKEACCCESSKEYIDGTAXJY6MZWLP", + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request + ) + + @meta(owner="test@test.com") + @source( + s3_with_secret.bucket( + bucket_name="all_ratings", + prefix="prod/apac/", + presorted=True, + format="delta", + ), + every="1h", + disorder="2d", + cdc="native", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + until=datetime.strptime("2022-02-28T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetS3WithSecret: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetS3WithSecret) + sync_request = view._get_sync_request_proto() + assert len(sync_request.datasets) == 1 + assert len(sync_request.sources) == 1 + assert len(sync_request.extdbs) == 1 + + # s3 source with secret + source_request = sync_request.sources[0] + s = { + "table": { + "s3Table": { + "bucket": "all_ratings", + "pathPrefix": "prod/apac/", + "pathSuffix": "", + "format": "delta", + "preSorted": True, + "db": { + "name": "ratings_source", + "s3": { + "awsSecretAccessKeySecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["s3_aws_secret_access_key"], + }, + "awsAccessKeyIdSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["s3_aws_access_key_id"], + }, + }, + }, + } + }, + "dataset": "UserInfoDatasetS3WithSecret", + "dsVersion": 1, + "every": "3600s", + "cdc": "Native", + "disorder": "172800s", + "startingFrom": "2021-08-10T00:00:00Z", + "until": "2022-02-28T00:00:00Z", + "timestamp_field": "timestamp", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + extdb_request = sync_request.extdbs[0] + e = { + "name": "ratings_source", + "s3": { + "awsSecretAccessKeySecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["s3_aws_secret_access_key"], + }, + "awsAccessKeyIdSecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["s3_aws_access_key_id"], + }, + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request + ) + + +def test_multiple_sources_pubsub(): + @meta(owner="test@test.com") + @source( + pubsub.topic("test_topic", format="json"), + disorder="14d", + cdc="upsert", + every="1h", + ) + @dataset + class UserInfoDatasetPubSub: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + # pubsub source + view = InternalTestClient() + view.add(UserInfoDatasetPubSub) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + + s = { + "table": { + "pubsubTopic": { + "db": { + "pubsub": { + "projectId": "test_project", + "serviceAccountKey": '{"type": "service_account", "project_id": "fake-project-356105", ' + '"client_email": ' + '"randomstring@fake-project-356105.iam.gserviceaccount.com", "client_id": ' + '"103688493243243272951", "auth_uri": ' + '"https://accounts.google.com/o/oauth2/auth", "token_uri": ' + '"https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": ' '"https://www.googleapis.com/oauth2/v1/certs"}', }, "name": "pubsub_src", @@ -1866,6 +2960,231 @@ class UserInfoDatasetPubSub: extdb_request, expected_extdb_request ) + @meta(owner="test@test.com") + @source( + pubsub_with_secret.topic("test_topic", format="json"), + disorder="14d", + cdc="upsert", + every="1h", + ) + @dataset + class UserInfoDatasetPubSubWithSecret: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + # pubsub source + view = InternalTestClient() + view.add(UserInfoDatasetPubSubWithSecret) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + + s = { + "table": { + "pubsubTopic": { + "db": { + "pubsub": { + "projectId": "test_project", + "serviceAccountKeySecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["pubsub_service_account_key"], + }, + }, + "name": "pubsub_src", + }, + "topicId": "test_topic", + "format": { + "json": {}, + }, + } + }, + "dataset": "UserInfoDatasetPubSubWithSecret", + "dsVersion": 1, + "cdc": "Upsert", + "disorder": "1209600s", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + extdb_request = sync_request.extdbs[0] + e = { + "name": "pubsub_src", + "pubsub": { + "projectId": "test_project", + "serviceAccountKeySecret": { + "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1", + "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role", + "path": ["pubsub_service_account_key"], + }, + }, + } + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request + ) + + +def test_multiple_sources_kinesis(): + @meta(owner="test@test.com") + @source( + kinesis.stream("test_stream", init_position="latest", format="json"), + disorder="14d", + cdc="upsert", + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + @dataset + class UserInfoDatasetKinesis: + user_id: int = field(key=True) + name: str + gender: str + # Users date of birth + dob: str + age: int + account_creation_date: datetime + country: Optional[str] + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKinesis) + sync_request = view._get_sync_request_proto() + source_request = sync_request.sources[0] + s = { + "table": { + "kinesisStream": { + "streamArn": "test_stream", + "initPosition": "LATEST", + "format": "json", + "db": { + "name": "kinesis_src", + "kinesis": { + "roleArn": "arn:aws:iam::123456789012:role/test-role" + }, + }, + } + }, + "dataset": "UserInfoDatasetKinesis", + "dsVersion": 1, + "cdc": "Upsert", + "disorder": "1209600s", + "startingFrom": "2021-08-10T00:00:00Z", + } + expected_source_request = ParseDict(s, connector_proto.Source()) + assert source_request == expected_source_request, error_message( + source_request, expected_source_request + ) + + @meta(owner="test@test.com") + @source( + kinesis.stream( + "test_stream", + init_position="2023-05-31 15:30:00", + format="json", + ), + disorder="14d", + cdc="upsert", + since=datetime(2023, 5, 31), + ) + @dataset + class UserInfoDatasetKinesis: + user_id: int = field(key=True) + name: str + gender: str + timestamp: datetime = field(timestamp=True) + + @meta(owner="test@test.com") + @source( + kinesis.stream( + "test_stream2", + init_position=datetime(2023, 5, 31), + format="json", + ), + disorder="14d", + cdc="upsert", + ) + @dataset + class UserInfoDatasetKinesis2: + user_id: int = field(key=True) + name: str + gender: str + timestamp: datetime = field(timestamp=True) + + view = InternalTestClient() + view.add(UserInfoDatasetKinesis) + view.add(UserInfoDatasetKinesis2) + sync_request = view._get_sync_request_proto() + + extdb_request = sync_request.extdbs[0] + e = { + "name": "kinesis_src", + "kinesis": {"roleArn": "arn:aws:iam::123456789012:role/test-role"}, + } + + expected_extdb_request = ParseDict(e, connector_proto.ExtDatabase()) + assert extdb_request == expected_extdb_request, error_message( + extdb_request, expected_extdb_request + ) + + source_req = sync_request.sources[0] + e = { + "table": { + "kinesisStream": { + "streamArn": "test_stream", + "initPosition": "AT_TIMESTAMP", + "initTimestamp": "2023-05-31T15:30:00Z", + "format": "json", + "db": { + "name": "kinesis_src", + "kinesis": { + "roleArn": "arn:aws:iam::123456789012:role/test-role" + }, + }, + } + }, + "dataset": "UserInfoDatasetKinesis", + "dsVersion": 1, + "cdc": "Upsert", + "disorder": "1209600s", + "startingFrom": "2023-05-31T00:00:00Z", + } + expected_source = ParseDict(e, connector_proto.Source()) + assert source_req == expected_source, error_message( + source_req, expected_source + ) + + source_req = sync_request.sources[1] + e = { + "table": { + "kinesisStream": { + "streamArn": "test_stream2", + "initPosition": "AT_TIMESTAMP", + "initTimestamp": "2023-05-31T00:00:00Z", + "format": "json", + "db": { + "name": "kinesis_src", + "kinesis": { + "roleArn": "arn:aws:iam::123456789012:role/test-role" + }, + }, + } + }, + "dataset": "UserInfoDatasetKinesis2", + "dsVersion": 1, + "cdc": "Upsert", + "disorder": "1209600s", + } + expected_source = ParseDict(e, connector_proto.Source()) + assert source_req == expected_source, error_message( + source_req, expected_source + ) + def test_console_source(): @source( diff --git a/fennel/gen/auth_pb2.py b/fennel/gen/auth_pb2.py index 5fb3361ee..c51bec9ca 100644 --- a/fennel/gen/auth_pb2.py +++ b/fennel/gen/auth_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: auth.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/connector_pb2.py b/fennel/gen/connector_pb2.py index 5ae41e6b0..bde0dd7e9 100644 --- a/fennel/gen/connector_pb2.py +++ b/fennel/gen/connector_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: connector.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -18,9 +19,10 @@ import fennel.gen.pycode_pb2 as pycode__pb2 import fennel.gen.schema_registry_pb2 as schema__registry__pb2 import fennel.gen.schema_pb2 as schema__pb2 +import fennel.gen.secret_pb2 as secret__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x63onnector.proto\x12\x16\x66\x65nnel.proto.connector\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\nexpr.proto\x1a\rkinesis.proto\x1a\x0cpycode.proto\x1a\x15schema_registry.proto\x1a\x0cschema.proto\"\x8c\x05\n\x0b\x45xtDatabase\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x05mysql\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.connector.MySQLH\x00\x12\x34\n\x08postgres\x18\x03 \x01(\x0b\x32 .fennel.proto.connector.PostgresH\x00\x12\x36\n\treference\x18\x04 \x01(\x0b\x32!.fennel.proto.connector.ReferenceH\x00\x12(\n\x02s3\x18\x05 \x01(\x0b\x32\x1a.fennel.proto.connector.S3H\x00\x12\x34\n\x08\x62igquery\x18\x06 \x01(\x0b\x32 .fennel.proto.connector.BigqueryH\x00\x12\x36\n\tsnowflake\x18\x07 \x01(\x0b\x32!.fennel.proto.connector.SnowflakeH\x00\x12.\n\x05kafka\x18\x08 \x01(\x0b\x32\x1d.fennel.proto.connector.KafkaH\x00\x12\x32\n\x07webhook\x18\t \x01(\x0b\x32\x1f.fennel.proto.connector.WebhookH\x00\x12\x32\n\x07kinesis\x18\n \x01(\x0b\x32\x1f.fennel.proto.connector.KinesisH\x00\x12\x34\n\x08redshift\x18\x0b \x01(\x0b\x32 .fennel.proto.connector.RedshiftH\x00\x12.\n\x05mongo\x18\x0c \x01(\x0b\x32\x1d.fennel.proto.connector.MongoH\x00\x12\x30\n\x06pubsub\x18\r \x01(\x0b\x32\x1e.fennel.proto.connector.PubSubH\x00\x42\t\n\x07variant\"M\n\x0cPubSubFormat\x12\x32\n\x04json\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.JsonFormatH\x00\x42\t\n\x07variant\"\xbc\x01\n\x0bKafkaFormat\x12\x32\n\x04json\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.JsonFormatH\x00\x12\x32\n\x04\x61vro\x18\x02 \x01(\x0b\x32\".fennel.proto.connector.AvroFormatH\x00\x12:\n\x08protobuf\x18\x03 \x01(\x0b\x32&.fennel.proto.connector.ProtobufFormatH\x00\x42\t\n\x07variant\"\x0c\n\nJsonFormat\"S\n\nAvroFormat\x12\x45\n\x0fschema_registry\x18\x01 \x01(\x0b\x32,.fennel.proto.schema_registry.SchemaRegistry\"W\n\x0eProtobufFormat\x12\x45\n\x0fschema_registry\x18\x01 \x01(\x0b\x32,.fennel.proto.schema_registry.SchemaRegistry\"\xde\x01\n\tReference\x12;\n\x06\x64\x62type\x18\x01 \x01(\x0e\x32+.fennel.proto.connector.Reference.ExtDBType\"\x93\x01\n\tExtDBType\x12\t\n\x05MYSQL\x10\x00\x12\x0c\n\x08POSTGRES\x10\x01\x12\x06\n\x02S3\x10\x02\x12\t\n\x05KAFKA\x10\x03\x12\x0c\n\x08\x42IGQUERY\x10\x04\x12\r\n\tSNOWFLAKE\x10\x05\x12\x0b\n\x07WEBHOOK\x10\x06\x12\x0b\n\x07KINESIS\x10\x07\x12\x0c\n\x08REDSHIFT\x10\x08\x12\t\n\x05MONGO\x10\t\x12\n\n\x06PUBSUB\x10\n\"E\n\x07Webhook\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\tretention\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\"j\n\x05MySQL\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x10\n\x08password\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\r\x12\x13\n\x0bjdbc_params\x18\x06 \x01(\t\"m\n\x08Postgres\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x10\n\x08password\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\r\x12\x13\n\x0bjdbc_params\x18\x06 \x01(\t\"b\n\x02S3\x12\x1d\n\x15\x61ws_secret_access_key\x18\x01 \x01(\t\x12\x19\n\x11\x61ws_access_key_id\x18\x02 \x01(\t\x12\x15\n\x08role_arn\x18\x03 \x01(\tH\x00\x88\x01\x01\x42\x0b\n\t_role_arn\"O\n\x08\x42igquery\x12\x12\n\ndataset_id\x18\x01 \x01(\t\x12\x1b\n\x13service_account_key\x18\x02 \x01(\t\x12\x12\n\nproject_id\x18\x03 \x01(\t\"\x7f\n\tSnowflake\x12\x0f\n\x07\x61\x63\x63ount\x18\x01 \x01(\t\x12\x0c\n\x04user\x18\x02 \x01(\t\x12\x10\n\x08password\x18\x03 \x01(\t\x12\x0e\n\x06schema\x18\x04 \x01(\t\x12\x11\n\twarehouse\x18\x05 \x01(\t\x12\x0c\n\x04role\x18\x06 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x07 \x01(\t\"\xfd\x01\n\x05Kafka\x12\x19\n\x11\x62ootstrap_servers\x18\x01 \x01(\t\x12\x19\n\x11security_protocol\x18\x02 \x01(\t\x12\x16\n\x0esasl_mechanism\x18\x03 \x01(\t\x12\x1c\n\x10sasl_jaas_config\x18\x04 \x01(\tB\x02\x18\x01\x12 \n\x13sasl_plain_username\x18\x05 \x01(\tH\x00\x88\x01\x01\x12 \n\x13sasl_plain_password\x18\x06 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x08group_id\x18\x07 \x01(\tB\x02\x18\x01\x42\x16\n\x14_sasl_plain_usernameB\x16\n\x14_sasl_plain_password\"\x1b\n\x07Kinesis\x12\x10\n\x08role_arn\x18\x01 \x01(\t\"1\n\x0b\x43redentials\x12\x10\n\x08username\x18\x01 \x01(\t\x12\x10\n\x08password\x18\x02 \x01(\t\"}\n\x16RedshiftAuthentication\x12\x1c\n\x12s3_access_role_arn\x18\x01 \x01(\tH\x00\x12:\n\x0b\x63redentials\x18\x02 \x01(\x0b\x32#.fennel.proto.connector.CredentialsH\x00\x42\t\n\x07variant\"\x99\x01\n\x08Redshift\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x03 \x01(\r\x12\x0e\n\x06schema\x18\x04 \x01(\t\x12O\n\x17redshift_authentication\x18\x05 \x01(\x0b\x32..fennel.proto.connector.RedshiftAuthentication\"G\n\x05Mongo\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x10\n\x08password\x18\x04 \x01(\t\"9\n\x06PubSub\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x1b\n\x13service_account_key\x18\x02 \x01(\t\"\xc0\x05\n\x08\x45xtTable\x12\x39\n\x0bmysql_table\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.MySQLTableH\x00\x12\x39\n\x08pg_table\x18\x02 \x01(\x0b\x32%.fennel.proto.connector.PostgresTableH\x00\x12\x33\n\x08s3_table\x18\x03 \x01(\x0b\x32\x1f.fennel.proto.connector.S3TableH\x00\x12\x39\n\x0bkafka_topic\x18\x04 \x01(\x0b\x32\".fennel.proto.connector.KafkaTopicH\x00\x12\x41\n\x0fsnowflake_table\x18\x05 \x01(\x0b\x32&.fennel.proto.connector.SnowflakeTableH\x00\x12?\n\x0e\x62igquery_table\x18\x06 \x01(\x0b\x32%.fennel.proto.connector.BigqueryTableH\x00\x12;\n\x08\x65ndpoint\x18\x07 \x01(\x0b\x32\'.fennel.proto.connector.WebhookEndpointH\x00\x12?\n\x0ekinesis_stream\x18\x08 \x01(\x0b\x32%.fennel.proto.connector.KinesisStreamH\x00\x12?\n\x0eredshift_table\x18\t \x01(\x0b\x32%.fennel.proto.connector.RedshiftTableH\x00\x12\x43\n\x10mongo_collection\x18\n \x01(\x0b\x32\'.fennel.proto.connector.MongoCollectionH\x00\x12;\n\x0cpubsub_topic\x18\x0b \x01(\x0b\x32#.fennel.proto.connector.PubSubTopicH\x00\x42\t\n\x07variant\"Q\n\nMySQLTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"z\n\rPostgresTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12\x16\n\tslot_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x42\x0c\n\n_slot_name\"\xf7\x01\n\x07S3Table\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x13\n\x0bpath_prefix\x18\x02 \x01(\t\x12\x11\n\tdelimiter\x18\x04 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x05 \x01(\t\x12/\n\x02\x64\x62\x18\x06 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\npre_sorted\x18\x07 \x01(\x08\x12\x13\n\x0bpath_suffix\x18\x08 \x01(\t\x12.\n\x06spread\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x88\x01\x01\x12\x0f\n\x07headers\x18\t \x03(\tB\t\n\x07_spread\"\x81\x01\n\nKafkaTopic\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x33\n\x06\x66ormat\x18\x03 \x01(\x0b\x32#.fennel.proto.connector.KafkaFormat\"T\n\rBigqueryTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"U\n\x0eSnowflakeTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"\x81\x01\n\x0fWebhookEndpoint\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12+\n\x08\x64uration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\"\xd3\x01\n\rKinesisStream\x12\x12\n\nstream_arn\x18\x01 \x01(\t\x12\x39\n\rinit_position\x18\x02 \x01(\x0e\x32\".fennel.proto.kinesis.InitPosition\x12\x32\n\x0einit_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x66ormat\x18\x04 \x01(\t\x12/\n\x02\x64\x62\x18\x05 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\"T\n\rRedshiftTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"\x86\x01\n\x0bPubSubTopic\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08topic_id\x18\x02 \x01(\t\x12\x34\n\x06\x66ormat\x18\x03 \x01(\x0b\x32$.fennel.proto.connector.PubSubFormat\"\xab\x02\n\x0cPreProcValue\x12\r\n\x03ref\x18\x01 \x01(\tH\x00\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1a.fennel.proto.schema.ValueH\x00\x12\x39\n\x04\x65val\x18\x03 \x01(\x0b\x32).fennel.proto.connector.PreProcValue.EvalH\x00\x1a\x98\x01\n\x04\x45val\x12+\n\x06schema\x18\x01 \x01(\x0b\x32\x1b.fennel.proto.schema.Schema\x12\'\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x17.fennel.proto.expr.ExprH\x00\x12-\n\x06pycode\x18\x03 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCodeH\x00\x42\x0b\n\teval_typeB\t\n\x07variant\"[\n\x0fMongoCollection\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x17\n\x0f\x63ollection_name\x18\x02 \x01(\t\"2\n\x0cSnapshotData\x12\x0e\n\x06marker\x18\x01 \x01(\t\x12\x12\n\nnum_retain\x18\x02 \x01(\r\"\r\n\x0bIncremental\"\n\n\x08Recreate\"\xbc\x01\n\x05Style\x12:\n\x0bincremental\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.IncrementalH\x00\x12\x34\n\x08recreate\x18\x02 \x01(\x0b\x32 .fennel.proto.connector.RecreateH\x00\x12\x38\n\x08snapshot\x18\x03 \x01(\x0b\x32$.fennel.proto.connector.SnapshotDataH\x00\x42\x07\n\x05Style\"\xb1\x05\n\x06Source\x12/\n\x05table\x18\x01 \x01(\x0b\x32 .fennel.proto.connector.ExtTable\x12\x0f\n\x07\x64\x61taset\x18\x02 \x01(\t\x12\x12\n\nds_version\x18\x03 \x01(\r\x12(\n\x05\x65very\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x06\x63ursor\x18\x05 \x01(\tH\x00\x88\x01\x01\x12+\n\x08\x64isorder\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x17\n\x0ftimestamp_field\x18\x07 \x01(\t\x12\x30\n\x03\x63\x64\x63\x18\x08 \x01(\x0e\x32#.fennel.proto.connector.CDCStrategy\x12\x31\n\rstarting_from\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12=\n\x08pre_proc\x18\n \x03(\x0b\x32+.fennel.proto.connector.Source.PreProcEntry\x12\x0f\n\x07version\x18\x0b \x01(\r\x12\x0f\n\x07\x62ounded\x18\x0c \x01(\x08\x12\x30\n\x08idleness\x18\r \x01(\x0b\x32\x19.google.protobuf.DurationH\x01\x88\x01\x01\x12)\n\x05until\x18\x0e \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x06\x66ilter\x18\x0f \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCodeH\x02\x88\x01\x01\x1aT\n\x0cPreProcEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.fennel.proto.connector.PreProcValue:\x02\x38\x01\x42\t\n\x07_cursorB\x0b\n\t_idlenessB\t\n\x07_filter\"\xee\x03\n\x04Sink\x12/\n\x05table\x18\x01 \x01(\x0b\x32 .fennel.proto.connector.ExtTable\x12\x0f\n\x07\x64\x61taset\x18\x02 \x01(\t\x12\x12\n\nds_version\x18\x03 \x01(\r\x12\x35\n\x03\x63\x64\x63\x18\x04 \x01(\x0e\x32#.fennel.proto.connector.CDCStrategyH\x00\x88\x01\x01\x12(\n\x05\x65very\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x03how\x18\x06 \x01(\x0b\x32\x1d.fennel.proto.connector.StyleH\x01\x88\x01\x01\x12\x0e\n\x06\x63reate\x18\x07 \x01(\x08\x12:\n\x07renames\x18\x08 \x03(\x0b\x32).fennel.proto.connector.Sink.RenamesEntry\x12.\n\x05since\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x02\x88\x01\x01\x12.\n\x05until\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x03\x88\x01\x01\x1a.\n\x0cRenamesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x06\n\x04_cdcB\x06\n\x04_howB\x08\n\x06_sinceB\x08\n\x06_until*K\n\x0b\x43\x44\x43Strategy\x12\n\n\x06\x41ppend\x10\x00\x12\n\n\x06Upsert\x10\x01\x12\x0c\n\x08\x44\x65\x62\x65zium\x10\x02\x12\n\n\x06Native\x10\x03\x12\n\n\x06\x44\x65lete\x10\x04\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x63onnector.proto\x12\x16\x66\x65nnel.proto.connector\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\nexpr.proto\x1a\rkinesis.proto\x1a\x0cpycode.proto\x1a\x15schema_registry.proto\x1a\x0cschema.proto\x1a\x0csecret.proto\"\x8c\x05\n\x0b\x45xtDatabase\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x05mysql\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.connector.MySQLH\x00\x12\x34\n\x08postgres\x18\x03 \x01(\x0b\x32 .fennel.proto.connector.PostgresH\x00\x12\x36\n\treference\x18\x04 \x01(\x0b\x32!.fennel.proto.connector.ReferenceH\x00\x12(\n\x02s3\x18\x05 \x01(\x0b\x32\x1a.fennel.proto.connector.S3H\x00\x12\x34\n\x08\x62igquery\x18\x06 \x01(\x0b\x32 .fennel.proto.connector.BigqueryH\x00\x12\x36\n\tsnowflake\x18\x07 \x01(\x0b\x32!.fennel.proto.connector.SnowflakeH\x00\x12.\n\x05kafka\x18\x08 \x01(\x0b\x32\x1d.fennel.proto.connector.KafkaH\x00\x12\x32\n\x07webhook\x18\t \x01(\x0b\x32\x1f.fennel.proto.connector.WebhookH\x00\x12\x32\n\x07kinesis\x18\n \x01(\x0b\x32\x1f.fennel.proto.connector.KinesisH\x00\x12\x34\n\x08redshift\x18\x0b \x01(\x0b\x32 .fennel.proto.connector.RedshiftH\x00\x12.\n\x05mongo\x18\x0c \x01(\x0b\x32\x1d.fennel.proto.connector.MongoH\x00\x12\x30\n\x06pubsub\x18\r \x01(\x0b\x32\x1e.fennel.proto.connector.PubSubH\x00\x42\t\n\x07variant\"M\n\x0cPubSubFormat\x12\x32\n\x04json\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.JsonFormatH\x00\x42\t\n\x07variant\"\xbc\x01\n\x0bKafkaFormat\x12\x32\n\x04json\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.JsonFormatH\x00\x12\x32\n\x04\x61vro\x18\x02 \x01(\x0b\x32\".fennel.proto.connector.AvroFormatH\x00\x12:\n\x08protobuf\x18\x03 \x01(\x0b\x32&.fennel.proto.connector.ProtobufFormatH\x00\x42\t\n\x07variant\"\x0c\n\nJsonFormat\"S\n\nAvroFormat\x12\x45\n\x0fschema_registry\x18\x01 \x01(\x0b\x32,.fennel.proto.schema_registry.SchemaRegistry\"W\n\x0eProtobufFormat\x12\x45\n\x0fschema_registry\x18\x01 \x01(\x0b\x32,.fennel.proto.schema_registry.SchemaRegistry\"\xde\x01\n\tReference\x12;\n\x06\x64\x62type\x18\x01 \x01(\x0e\x32+.fennel.proto.connector.Reference.ExtDBType\"\x93\x01\n\tExtDBType\x12\t\n\x05MYSQL\x10\x00\x12\x0c\n\x08POSTGRES\x10\x01\x12\x06\n\x02S3\x10\x02\x12\t\n\x05KAFKA\x10\x03\x12\x0c\n\x08\x42IGQUERY\x10\x04\x12\r\n\tSNOWFLAKE\x10\x05\x12\x0b\n\x07WEBHOOK\x10\x06\x12\x0b\n\x07KINESIS\x10\x07\x12\x0c\n\x08REDSHIFT\x10\x08\x12\t\n\x05MONGO\x10\t\x12\n\n\x06PUBSUB\x10\n\"E\n\x07Webhook\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\tretention\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\"\x8c\x02\n\x05MySQL\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x07 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\r\x12\x13\n\x0bjdbc_params\x18\x06 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant\"\x8f\x02\n\x08Postgres\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x07 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\r\x12\x13\n\x0bjdbc_params\x18\x06 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant\"\xb0\x02\n\x02S3\x12\x1f\n\x15\x61ws_secret_access_key\x18\x01 \x01(\tH\x00\x12\x46\n\x1c\x61ws_secret_access_key_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x1b\n\x11\x61ws_access_key_id\x18\x02 \x01(\tH\x01\x12\x42\n\x18\x61ws_access_key_id_secret\x18\x05 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x15\n\x08role_arn\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x1f\n\x1d\x61ws_secret_access_key_variantB\x1b\n\x19\x61ws_access_key_id_variantB\x0b\n\t_role_arn\"\xb6\x01\n\x08\x42igquery\x12\x1d\n\x13service_account_key\x18\x02 \x01(\tH\x00\x12\x44\n\x1aservice_account_key_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\ndataset_id\x18\x01 \x01(\t\x12\x12\n\nproject_id\x18\x03 \x01(\tB\x1d\n\x1bservice_account_key_variant\"\xa1\x02\n\tSnowflake\x12\x0e\n\x04user\x18\x02 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x03 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\t \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0f\n\x07\x61\x63\x63ount\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x04 \x01(\t\x12\x11\n\twarehouse\x18\x05 \x01(\t\x12\x0c\n\x04role\x18\x06 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x07 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant\"\xf9\x02\n\x05Kafka\x12\x1d\n\x13sasl_plain_username\x18\x05 \x01(\tH\x00\x12>\n\x14sasl_username_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x1d\n\x13sasl_plain_password\x18\x06 \x01(\tH\x01\x12>\n\x14sasl_password_secret\x18\t \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x19\n\x11\x62ootstrap_servers\x18\x01 \x01(\t\x12\x19\n\x11security_protocol\x18\x02 \x01(\t\x12\x16\n\x0esasl_mechanism\x18\x03 \x01(\t\x12\x1c\n\x10sasl_jaas_config\x18\x04 \x01(\tB\x02\x18\x01\x12\x14\n\x08group_id\x18\x07 \x01(\tB\x02\x18\x01\x42\x17\n\x15sasl_username_variantB\x17\n\x15sasl_password_variant\"\x1b\n\x07Kinesis\x12\x10\n\x08role_arn\x18\x01 \x01(\t\"\xd3\x01\n\x0b\x43redentials\x12\x12\n\x08username\x18\x01 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x03 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x02 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x42\x12\n\x10username_variantB\x12\n\x10password_variant\"}\n\x16RedshiftAuthentication\x12\x1c\n\x12s3_access_role_arn\x18\x01 \x01(\tH\x00\x12:\n\x0b\x63redentials\x18\x02 \x01(\x0b\x32#.fennel.proto.connector.CredentialsH\x00\x42\t\n\x07variant\"\x99\x01\n\x08Redshift\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x03 \x01(\r\x12\x0e\n\x06schema\x18\x04 \x01(\t\x12O\n\x17redshift_authentication\x18\x05 \x01(\x0b\x32..fennel.proto.connector.RedshiftAuthentication\"\xe9\x01\n\x05Mongo\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x05 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x06 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant\"\xa0\x01\n\x06PubSub\x12\x1d\n\x13service_account_key\x18\x02 \x01(\tH\x00\x12\x44\n\x1aservice_account_key_secret\x18\x03 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\nproject_id\x18\x01 \x01(\tB\x1d\n\x1bservice_account_key_variant\"\xc0\x05\n\x08\x45xtTable\x12\x39\n\x0bmysql_table\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.MySQLTableH\x00\x12\x39\n\x08pg_table\x18\x02 \x01(\x0b\x32%.fennel.proto.connector.PostgresTableH\x00\x12\x33\n\x08s3_table\x18\x03 \x01(\x0b\x32\x1f.fennel.proto.connector.S3TableH\x00\x12\x39\n\x0bkafka_topic\x18\x04 \x01(\x0b\x32\".fennel.proto.connector.KafkaTopicH\x00\x12\x41\n\x0fsnowflake_table\x18\x05 \x01(\x0b\x32&.fennel.proto.connector.SnowflakeTableH\x00\x12?\n\x0e\x62igquery_table\x18\x06 \x01(\x0b\x32%.fennel.proto.connector.BigqueryTableH\x00\x12;\n\x08\x65ndpoint\x18\x07 \x01(\x0b\x32\'.fennel.proto.connector.WebhookEndpointH\x00\x12?\n\x0ekinesis_stream\x18\x08 \x01(\x0b\x32%.fennel.proto.connector.KinesisStreamH\x00\x12?\n\x0eredshift_table\x18\t \x01(\x0b\x32%.fennel.proto.connector.RedshiftTableH\x00\x12\x43\n\x10mongo_collection\x18\n \x01(\x0b\x32\'.fennel.proto.connector.MongoCollectionH\x00\x12;\n\x0cpubsub_topic\x18\x0b \x01(\x0b\x32#.fennel.proto.connector.PubSubTopicH\x00\x42\t\n\x07variant\"Q\n\nMySQLTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"z\n\rPostgresTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12\x16\n\tslot_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x42\x0c\n\n_slot_name\"\xf7\x01\n\x07S3Table\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x13\n\x0bpath_prefix\x18\x02 \x01(\t\x12\x11\n\tdelimiter\x18\x04 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x05 \x01(\t\x12/\n\x02\x64\x62\x18\x06 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\npre_sorted\x18\x07 \x01(\x08\x12\x13\n\x0bpath_suffix\x18\x08 \x01(\t\x12.\n\x06spread\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x88\x01\x01\x12\x0f\n\x07headers\x18\t \x03(\tB\t\n\x07_spread\"\x81\x01\n\nKafkaTopic\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x33\n\x06\x66ormat\x18\x03 \x01(\x0b\x32#.fennel.proto.connector.KafkaFormat\"T\n\rBigqueryTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"U\n\x0eSnowflakeTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"\x81\x01\n\x0fWebhookEndpoint\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12+\n\x08\x64uration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\"\xd3\x01\n\rKinesisStream\x12\x12\n\nstream_arn\x18\x01 \x01(\t\x12\x39\n\rinit_position\x18\x02 \x01(\x0e\x32\".fennel.proto.kinesis.InitPosition\x12\x32\n\x0einit_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x66ormat\x18\x04 \x01(\t\x12/\n\x02\x64\x62\x18\x05 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\"T\n\rRedshiftTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"\x86\x01\n\x0bPubSubTopic\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08topic_id\x18\x02 \x01(\t\x12\x34\n\x06\x66ormat\x18\x03 \x01(\x0b\x32$.fennel.proto.connector.PubSubFormat\"\xab\x02\n\x0cPreProcValue\x12\r\n\x03ref\x18\x01 \x01(\tH\x00\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1a.fennel.proto.schema.ValueH\x00\x12\x39\n\x04\x65val\x18\x03 \x01(\x0b\x32).fennel.proto.connector.PreProcValue.EvalH\x00\x1a\x98\x01\n\x04\x45val\x12+\n\x06schema\x18\x01 \x01(\x0b\x32\x1b.fennel.proto.schema.Schema\x12\'\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x17.fennel.proto.expr.ExprH\x00\x12-\n\x06pycode\x18\x03 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCodeH\x00\x42\x0b\n\teval_typeB\t\n\x07variant\"[\n\x0fMongoCollection\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x17\n\x0f\x63ollection_name\x18\x02 \x01(\t\"2\n\x0cSnapshotData\x12\x0e\n\x06marker\x18\x01 \x01(\t\x12\x12\n\nnum_retain\x18\x02 \x01(\r\"\r\n\x0bIncremental\"\n\n\x08Recreate\"\xbc\x01\n\x05Style\x12:\n\x0bincremental\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.IncrementalH\x00\x12\x34\n\x08recreate\x18\x02 \x01(\x0b\x32 .fennel.proto.connector.RecreateH\x00\x12\x38\n\x08snapshot\x18\x03 \x01(\x0b\x32$.fennel.proto.connector.SnapshotDataH\x00\x42\x07\n\x05Style\"\xb1\x05\n\x06Source\x12/\n\x05table\x18\x01 \x01(\x0b\x32 .fennel.proto.connector.ExtTable\x12\x0f\n\x07\x64\x61taset\x18\x02 \x01(\t\x12\x12\n\nds_version\x18\x03 \x01(\r\x12(\n\x05\x65very\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x06\x63ursor\x18\x05 \x01(\tH\x00\x88\x01\x01\x12+\n\x08\x64isorder\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x17\n\x0ftimestamp_field\x18\x07 \x01(\t\x12\x30\n\x03\x63\x64\x63\x18\x08 \x01(\x0e\x32#.fennel.proto.connector.CDCStrategy\x12\x31\n\rstarting_from\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12=\n\x08pre_proc\x18\n \x03(\x0b\x32+.fennel.proto.connector.Source.PreProcEntry\x12\x0f\n\x07version\x18\x0b \x01(\r\x12\x0f\n\x07\x62ounded\x18\x0c \x01(\x08\x12\x30\n\x08idleness\x18\r \x01(\x0b\x32\x19.google.protobuf.DurationH\x01\x88\x01\x01\x12)\n\x05until\x18\x0e \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x06\x66ilter\x18\x0f \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCodeH\x02\x88\x01\x01\x1aT\n\x0cPreProcEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.fennel.proto.connector.PreProcValue:\x02\x38\x01\x42\t\n\x07_cursorB\x0b\n\t_idlenessB\t\n\x07_filter\"\xee\x03\n\x04Sink\x12/\n\x05table\x18\x01 \x01(\x0b\x32 .fennel.proto.connector.ExtTable\x12\x0f\n\x07\x64\x61taset\x18\x02 \x01(\t\x12\x12\n\nds_version\x18\x03 \x01(\r\x12\x35\n\x03\x63\x64\x63\x18\x04 \x01(\x0e\x32#.fennel.proto.connector.CDCStrategyH\x00\x88\x01\x01\x12(\n\x05\x65very\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x03how\x18\x06 \x01(\x0b\x32\x1d.fennel.proto.connector.StyleH\x01\x88\x01\x01\x12\x0e\n\x06\x63reate\x18\x07 \x01(\x08\x12:\n\x07renames\x18\x08 \x03(\x0b\x32).fennel.proto.connector.Sink.RenamesEntry\x12.\n\x05since\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x02\x88\x01\x01\x12.\n\x05until\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x03\x88\x01\x01\x1a.\n\x0cRenamesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x06\n\x04_cdcB\x06\n\x04_howB\x08\n\x06_sinceB\x08\n\x06_until*K\n\x0b\x43\x44\x43Strategy\x12\n\n\x06\x41ppend\x10\x00\x12\n\n\x06Upsert\x10\x01\x12\x0c\n\x08\x44\x65\x62\x65zium\x10\x02\x12\n\n\x06Native\x10\x03\x12\n\n\x06\x44\x65lete\x10\x04\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -35,92 +37,92 @@ _globals['_SOURCE_PREPROCENTRY']._serialized_options = b'8\001' _globals['_SINK_RENAMESENTRY']._options = None _globals['_SINK_RENAMESENTRY']._serialized_options = b'8\001' - _globals['_CDCSTRATEGY']._serialized_start=6767 - _globals['_CDCSTRATEGY']._serialized_end=6842 - _globals['_EXTDATABASE']._serialized_start=187 - _globals['_EXTDATABASE']._serialized_end=839 - _globals['_PUBSUBFORMAT']._serialized_start=841 - _globals['_PUBSUBFORMAT']._serialized_end=918 - _globals['_KAFKAFORMAT']._serialized_start=921 - _globals['_KAFKAFORMAT']._serialized_end=1109 - _globals['_JSONFORMAT']._serialized_start=1111 - _globals['_JSONFORMAT']._serialized_end=1123 - _globals['_AVROFORMAT']._serialized_start=1125 - _globals['_AVROFORMAT']._serialized_end=1208 - _globals['_PROTOBUFFORMAT']._serialized_start=1210 - _globals['_PROTOBUFFORMAT']._serialized_end=1297 - _globals['_REFERENCE']._serialized_start=1300 - _globals['_REFERENCE']._serialized_end=1522 - _globals['_REFERENCE_EXTDBTYPE']._serialized_start=1375 - _globals['_REFERENCE_EXTDBTYPE']._serialized_end=1522 - _globals['_WEBHOOK']._serialized_start=1524 - _globals['_WEBHOOK']._serialized_end=1593 - _globals['_MYSQL']._serialized_start=1595 - _globals['_MYSQL']._serialized_end=1701 - _globals['_POSTGRES']._serialized_start=1703 - _globals['_POSTGRES']._serialized_end=1812 - _globals['_S3']._serialized_start=1814 - _globals['_S3']._serialized_end=1912 - _globals['_BIGQUERY']._serialized_start=1914 - _globals['_BIGQUERY']._serialized_end=1993 - _globals['_SNOWFLAKE']._serialized_start=1995 - _globals['_SNOWFLAKE']._serialized_end=2122 - _globals['_KAFKA']._serialized_start=2125 - _globals['_KAFKA']._serialized_end=2378 - _globals['_KINESIS']._serialized_start=2380 - _globals['_KINESIS']._serialized_end=2407 - _globals['_CREDENTIALS']._serialized_start=2409 - _globals['_CREDENTIALS']._serialized_end=2458 - _globals['_REDSHIFTAUTHENTICATION']._serialized_start=2460 - _globals['_REDSHIFTAUTHENTICATION']._serialized_end=2585 - _globals['_REDSHIFT']._serialized_start=2588 - _globals['_REDSHIFT']._serialized_end=2741 - _globals['_MONGO']._serialized_start=2743 - _globals['_MONGO']._serialized_end=2814 - _globals['_PUBSUB']._serialized_start=2816 - _globals['_PUBSUB']._serialized_end=2873 - _globals['_EXTTABLE']._serialized_start=2876 - _globals['_EXTTABLE']._serialized_end=3580 - _globals['_MYSQLTABLE']._serialized_start=3582 - _globals['_MYSQLTABLE']._serialized_end=3663 - _globals['_POSTGRESTABLE']._serialized_start=3665 - _globals['_POSTGRESTABLE']._serialized_end=3787 - _globals['_S3TABLE']._serialized_start=3790 - _globals['_S3TABLE']._serialized_end=4037 - _globals['_KAFKATOPIC']._serialized_start=4040 - _globals['_KAFKATOPIC']._serialized_end=4169 - _globals['_BIGQUERYTABLE']._serialized_start=4171 - _globals['_BIGQUERYTABLE']._serialized_end=4255 - _globals['_SNOWFLAKETABLE']._serialized_start=4257 - _globals['_SNOWFLAKETABLE']._serialized_end=4342 - _globals['_WEBHOOKENDPOINT']._serialized_start=4345 - _globals['_WEBHOOKENDPOINT']._serialized_end=4474 - _globals['_KINESISSTREAM']._serialized_start=4477 - _globals['_KINESISSTREAM']._serialized_end=4688 - _globals['_REDSHIFTTABLE']._serialized_start=4690 - _globals['_REDSHIFTTABLE']._serialized_end=4774 - _globals['_PUBSUBTOPIC']._serialized_start=4777 - _globals['_PUBSUBTOPIC']._serialized_end=4911 - _globals['_PREPROCVALUE']._serialized_start=4914 - _globals['_PREPROCVALUE']._serialized_end=5213 - _globals['_PREPROCVALUE_EVAL']._serialized_start=5050 - _globals['_PREPROCVALUE_EVAL']._serialized_end=5202 - _globals['_MONGOCOLLECTION']._serialized_start=5215 - _globals['_MONGOCOLLECTION']._serialized_end=5306 - _globals['_SNAPSHOTDATA']._serialized_start=5308 - _globals['_SNAPSHOTDATA']._serialized_end=5358 - _globals['_INCREMENTAL']._serialized_start=5360 - _globals['_INCREMENTAL']._serialized_end=5373 - _globals['_RECREATE']._serialized_start=5375 - _globals['_RECREATE']._serialized_end=5385 - _globals['_STYLE']._serialized_start=5388 - _globals['_STYLE']._serialized_end=5576 - _globals['_SOURCE']._serialized_start=5579 - _globals['_SOURCE']._serialized_end=6268 - _globals['_SOURCE_PREPROCENTRY']._serialized_start=6149 - _globals['_SOURCE_PREPROCENTRY']._serialized_end=6233 - _globals['_SINK']._serialized_start=6271 - _globals['_SINK']._serialized_end=6765 - _globals['_SINK_RENAMESENTRY']._serialized_start=6683 - _globals['_SINK_RENAMESENTRY']._serialized_end=6729 + _globals['_CDCSTRATEGY']._serialized_start=8135 + _globals['_CDCSTRATEGY']._serialized_end=8210 + _globals['_EXTDATABASE']._serialized_start=201 + _globals['_EXTDATABASE']._serialized_end=853 + _globals['_PUBSUBFORMAT']._serialized_start=855 + _globals['_PUBSUBFORMAT']._serialized_end=932 + _globals['_KAFKAFORMAT']._serialized_start=935 + _globals['_KAFKAFORMAT']._serialized_end=1123 + _globals['_JSONFORMAT']._serialized_start=1125 + _globals['_JSONFORMAT']._serialized_end=1137 + _globals['_AVROFORMAT']._serialized_start=1139 + _globals['_AVROFORMAT']._serialized_end=1222 + _globals['_PROTOBUFFORMAT']._serialized_start=1224 + _globals['_PROTOBUFFORMAT']._serialized_end=1311 + _globals['_REFERENCE']._serialized_start=1314 + _globals['_REFERENCE']._serialized_end=1536 + _globals['_REFERENCE_EXTDBTYPE']._serialized_start=1389 + _globals['_REFERENCE_EXTDBTYPE']._serialized_end=1536 + _globals['_WEBHOOK']._serialized_start=1538 + _globals['_WEBHOOK']._serialized_end=1607 + _globals['_MYSQL']._serialized_start=1610 + _globals['_MYSQL']._serialized_end=1878 + _globals['_POSTGRES']._serialized_start=1881 + _globals['_POSTGRES']._serialized_end=2152 + _globals['_S3']._serialized_start=2155 + _globals['_S3']._serialized_end=2459 + _globals['_BIGQUERY']._serialized_start=2462 + _globals['_BIGQUERY']._serialized_end=2644 + _globals['_SNOWFLAKE']._serialized_start=2647 + _globals['_SNOWFLAKE']._serialized_end=2936 + _globals['_KAFKA']._serialized_start=2939 + _globals['_KAFKA']._serialized_end=3316 + _globals['_KINESIS']._serialized_start=3318 + _globals['_KINESIS']._serialized_end=3345 + _globals['_CREDENTIALS']._serialized_start=3348 + _globals['_CREDENTIALS']._serialized_end=3559 + _globals['_REDSHIFTAUTHENTICATION']._serialized_start=3561 + _globals['_REDSHIFTAUTHENTICATION']._serialized_end=3686 + _globals['_REDSHIFT']._serialized_start=3689 + _globals['_REDSHIFT']._serialized_end=3842 + _globals['_MONGO']._serialized_start=3845 + _globals['_MONGO']._serialized_end=4078 + _globals['_PUBSUB']._serialized_start=4081 + _globals['_PUBSUB']._serialized_end=4241 + _globals['_EXTTABLE']._serialized_start=4244 + _globals['_EXTTABLE']._serialized_end=4948 + _globals['_MYSQLTABLE']._serialized_start=4950 + _globals['_MYSQLTABLE']._serialized_end=5031 + _globals['_POSTGRESTABLE']._serialized_start=5033 + _globals['_POSTGRESTABLE']._serialized_end=5155 + _globals['_S3TABLE']._serialized_start=5158 + _globals['_S3TABLE']._serialized_end=5405 + _globals['_KAFKATOPIC']._serialized_start=5408 + _globals['_KAFKATOPIC']._serialized_end=5537 + _globals['_BIGQUERYTABLE']._serialized_start=5539 + _globals['_BIGQUERYTABLE']._serialized_end=5623 + _globals['_SNOWFLAKETABLE']._serialized_start=5625 + _globals['_SNOWFLAKETABLE']._serialized_end=5710 + _globals['_WEBHOOKENDPOINT']._serialized_start=5713 + _globals['_WEBHOOKENDPOINT']._serialized_end=5842 + _globals['_KINESISSTREAM']._serialized_start=5845 + _globals['_KINESISSTREAM']._serialized_end=6056 + _globals['_REDSHIFTTABLE']._serialized_start=6058 + _globals['_REDSHIFTTABLE']._serialized_end=6142 + _globals['_PUBSUBTOPIC']._serialized_start=6145 + _globals['_PUBSUBTOPIC']._serialized_end=6279 + _globals['_PREPROCVALUE']._serialized_start=6282 + _globals['_PREPROCVALUE']._serialized_end=6581 + _globals['_PREPROCVALUE_EVAL']._serialized_start=6418 + _globals['_PREPROCVALUE_EVAL']._serialized_end=6570 + _globals['_MONGOCOLLECTION']._serialized_start=6583 + _globals['_MONGOCOLLECTION']._serialized_end=6674 + _globals['_SNAPSHOTDATA']._serialized_start=6676 + _globals['_SNAPSHOTDATA']._serialized_end=6726 + _globals['_INCREMENTAL']._serialized_start=6728 + _globals['_INCREMENTAL']._serialized_end=6741 + _globals['_RECREATE']._serialized_start=6743 + _globals['_RECREATE']._serialized_end=6753 + _globals['_STYLE']._serialized_start=6756 + _globals['_STYLE']._serialized_end=6944 + _globals['_SOURCE']._serialized_start=6947 + _globals['_SOURCE']._serialized_end=7636 + _globals['_SOURCE_PREPROCENTRY']._serialized_start=7517 + _globals['_SOURCE_PREPROCENTRY']._serialized_end=7601 + _globals['_SINK']._serialized_start=7639 + _globals['_SINK']._serialized_end=8133 + _globals['_SINK_RENAMESENTRY']._serialized_start=8051 + _globals['_SINK_RENAMESENTRY']._serialized_end=8097 # @@protoc_insertion_point(module_scope) diff --git a/fennel/gen/connector_pb2.pyi b/fennel/gen/connector_pb2.pyi index c9550885a..04e38755e 100644 --- a/fennel/gen/connector_pb2.pyi +++ b/fennel/gen/connector_pb2.pyi @@ -24,6 +24,7 @@ import kinesis_pb2 import pycode_pb2 import schema_pb2 import schema_registry_pb2 +import secret_pb2 import sys import typing @@ -286,29 +287,42 @@ global___Webhook = Webhook class MySQL(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - HOST_FIELD_NUMBER: builtins.int - DATABASE_FIELD_NUMBER: builtins.int USER_FIELD_NUMBER: builtins.int + USERNAME_SECRET_FIELD_NUMBER: builtins.int PASSWORD_FIELD_NUMBER: builtins.int + PASSWORD_SECRET_FIELD_NUMBER: builtins.int + HOST_FIELD_NUMBER: builtins.int + DATABASE_FIELD_NUMBER: builtins.int PORT_FIELD_NUMBER: builtins.int JDBC_PARAMS_FIELD_NUMBER: builtins.int - host: builtins.str - database: builtins.str user: builtins.str + @property + def username_secret(self) -> secret_pb2.SecretRef: ... password: builtins.str + @property + def password_secret(self) -> secret_pb2.SecretRef: ... + host: builtins.str + database: builtins.str port: builtins.int jdbc_params: builtins.str def __init__( self, *, - host: builtins.str = ..., - database: builtins.str = ..., user: builtins.str = ..., + username_secret: secret_pb2.SecretRef | None = ..., password: builtins.str = ..., + password_secret: secret_pb2.SecretRef | None = ..., + host: builtins.str = ..., + database: builtins.str = ..., port: builtins.int = ..., jdbc_params: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "jdbc_params", b"jdbc_params", "password", b"password", "port", b"port", "user", b"user"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "jdbc_params", b"jdbc_params", "password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "port", b"port", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["user", "username_secret"] | None: ... global___MySQL = MySQL @@ -316,29 +330,42 @@ global___MySQL = MySQL class Postgres(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - HOST_FIELD_NUMBER: builtins.int - DATABASE_FIELD_NUMBER: builtins.int USER_FIELD_NUMBER: builtins.int + USERNAME_SECRET_FIELD_NUMBER: builtins.int PASSWORD_FIELD_NUMBER: builtins.int + PASSWORD_SECRET_FIELD_NUMBER: builtins.int + HOST_FIELD_NUMBER: builtins.int + DATABASE_FIELD_NUMBER: builtins.int PORT_FIELD_NUMBER: builtins.int JDBC_PARAMS_FIELD_NUMBER: builtins.int - host: builtins.str - database: builtins.str user: builtins.str + @property + def username_secret(self) -> secret_pb2.SecretRef: ... password: builtins.str + @property + def password_secret(self) -> secret_pb2.SecretRef: ... + host: builtins.str + database: builtins.str port: builtins.int jdbc_params: builtins.str def __init__( self, *, - host: builtins.str = ..., - database: builtins.str = ..., user: builtins.str = ..., + username_secret: secret_pb2.SecretRef | None = ..., password: builtins.str = ..., + password_secret: secret_pb2.SecretRef | None = ..., + host: builtins.str = ..., + database: builtins.str = ..., port: builtins.int = ..., jdbc_params: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "jdbc_params", b"jdbc_params", "password", b"password", "port", b"port", "user", b"user"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "jdbc_params", b"jdbc_params", "password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "port", b"port", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["user", "username_secret"] | None: ... global___Postgres = Postgres @@ -347,21 +374,34 @@ class S3(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor AWS_SECRET_ACCESS_KEY_FIELD_NUMBER: builtins.int + AWS_SECRET_ACCESS_KEY_SECRET_FIELD_NUMBER: builtins.int AWS_ACCESS_KEY_ID_FIELD_NUMBER: builtins.int + AWS_ACCESS_KEY_ID_SECRET_FIELD_NUMBER: builtins.int ROLE_ARN_FIELD_NUMBER: builtins.int aws_secret_access_key: builtins.str + @property + def aws_secret_access_key_secret(self) -> secret_pb2.SecretRef: ... aws_access_key_id: builtins.str + @property + def aws_access_key_id_secret(self) -> secret_pb2.SecretRef: ... role_arn: builtins.str def __init__( self, *, aws_secret_access_key: builtins.str = ..., + aws_secret_access_key_secret: secret_pb2.SecretRef | None = ..., aws_access_key_id: builtins.str = ..., + aws_access_key_id_secret: secret_pb2.SecretRef | None = ..., role_arn: builtins.str | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_role_arn", b"_role_arn", "role_arn", b"role_arn"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_role_arn", b"_role_arn", "aws_access_key_id", b"aws_access_key_id", "aws_secret_access_key", b"aws_secret_access_key", "role_arn", b"role_arn"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_role_arn", b"_role_arn", "aws_access_key_id", b"aws_access_key_id", "aws_access_key_id_secret", b"aws_access_key_id_secret", "aws_access_key_id_variant", b"aws_access_key_id_variant", "aws_secret_access_key", b"aws_secret_access_key", "aws_secret_access_key_secret", b"aws_secret_access_key_secret", "aws_secret_access_key_variant", b"aws_secret_access_key_variant", "role_arn", b"role_arn"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_role_arn", b"_role_arn", "aws_access_key_id", b"aws_access_key_id", "aws_access_key_id_secret", b"aws_access_key_id_secret", "aws_access_key_id_variant", b"aws_access_key_id_variant", "aws_secret_access_key", b"aws_secret_access_key", "aws_secret_access_key_secret", b"aws_secret_access_key_secret", "aws_secret_access_key_variant", b"aws_secret_access_key_variant", "role_arn", b"role_arn"]) -> None: ... + @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_role_arn", b"_role_arn"]) -> typing_extensions.Literal["role_arn"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["aws_access_key_id_variant", b"aws_access_key_id_variant"]) -> typing_extensions.Literal["aws_access_key_id", "aws_access_key_id_secret"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["aws_secret_access_key_variant", b"aws_secret_access_key_variant"]) -> typing_extensions.Literal["aws_secret_access_key", "aws_secret_access_key_secret"] | None: ... global___S3 = S3 @@ -369,20 +409,26 @@ global___S3 = S3 class Bigquery(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - DATASET_ID_FIELD_NUMBER: builtins.int SERVICE_ACCOUNT_KEY_FIELD_NUMBER: builtins.int + SERVICE_ACCOUNT_KEY_SECRET_FIELD_NUMBER: builtins.int + DATASET_ID_FIELD_NUMBER: builtins.int PROJECT_ID_FIELD_NUMBER: builtins.int - dataset_id: builtins.str service_account_key: builtins.str + @property + def service_account_key_secret(self) -> secret_pb2.SecretRef: ... + dataset_id: builtins.str project_id: builtins.str def __init__( self, *, - dataset_id: builtins.str = ..., service_account_key: builtins.str = ..., + service_account_key_secret: secret_pb2.SecretRef | None = ..., + dataset_id: builtins.str = ..., project_id: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["dataset_id", b"dataset_id", "project_id", b"project_id", "service_account_key", b"service_account_key"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["service_account_key", b"service_account_key", "service_account_key_secret", b"service_account_key_secret", "service_account_key_variant", b"service_account_key_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["dataset_id", b"dataset_id", "project_id", b"project_id", "service_account_key", b"service_account_key", "service_account_key_secret", b"service_account_key_secret", "service_account_key_variant", b"service_account_key_variant"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["service_account_key_variant", b"service_account_key_variant"]) -> typing_extensions.Literal["service_account_key", "service_account_key_secret"] | None: ... global___Bigquery = Bigquery @@ -390,16 +436,22 @@ global___Bigquery = Bigquery class Snowflake(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - ACCOUNT_FIELD_NUMBER: builtins.int USER_FIELD_NUMBER: builtins.int + USERNAME_SECRET_FIELD_NUMBER: builtins.int PASSWORD_FIELD_NUMBER: builtins.int + PASSWORD_SECRET_FIELD_NUMBER: builtins.int + ACCOUNT_FIELD_NUMBER: builtins.int SCHEMA_FIELD_NUMBER: builtins.int WAREHOUSE_FIELD_NUMBER: builtins.int ROLE_FIELD_NUMBER: builtins.int DATABASE_FIELD_NUMBER: builtins.int - account: builtins.str user: builtins.str + @property + def username_secret(self) -> secret_pb2.SecretRef: ... password: builtins.str + @property + def password_secret(self) -> secret_pb2.SecretRef: ... + account: builtins.str schema: builtins.str warehouse: builtins.str role: builtins.str @@ -407,15 +459,22 @@ class Snowflake(google.protobuf.message.Message): def __init__( self, *, - account: builtins.str = ..., user: builtins.str = ..., + username_secret: secret_pb2.SecretRef | None = ..., password: builtins.str = ..., + password_secret: secret_pb2.SecretRef | None = ..., + account: builtins.str = ..., schema: builtins.str = ..., warehouse: builtins.str = ..., role: builtins.str = ..., database: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["account", b"account", "database", b"database", "password", b"password", "role", b"role", "schema", b"schema", "user", b"user", "warehouse", b"warehouse"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["account", b"account", "database", b"database", "password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "role", b"role", "schema", b"schema", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant", "warehouse", b"warehouse"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["user", "username_secret"] | None: ... global___Snowflake = Snowflake @@ -423,37 +482,45 @@ global___Snowflake = Snowflake class Kafka(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor + SASL_PLAIN_USERNAME_FIELD_NUMBER: builtins.int + SASL_USERNAME_SECRET_FIELD_NUMBER: builtins.int + SASL_PLAIN_PASSWORD_FIELD_NUMBER: builtins.int + SASL_PASSWORD_SECRET_FIELD_NUMBER: builtins.int BOOTSTRAP_SERVERS_FIELD_NUMBER: builtins.int SECURITY_PROTOCOL_FIELD_NUMBER: builtins.int SASL_MECHANISM_FIELD_NUMBER: builtins.int SASL_JAAS_CONFIG_FIELD_NUMBER: builtins.int - SASL_PLAIN_USERNAME_FIELD_NUMBER: builtins.int - SASL_PLAIN_PASSWORD_FIELD_NUMBER: builtins.int GROUP_ID_FIELD_NUMBER: builtins.int + sasl_plain_username: builtins.str + @property + def sasl_username_secret(self) -> secret_pb2.SecretRef: ... + sasl_plain_password: builtins.str + @property + def sasl_password_secret(self) -> secret_pb2.SecretRef: ... bootstrap_servers: builtins.str security_protocol: builtins.str sasl_mechanism: builtins.str sasl_jaas_config: builtins.str - sasl_plain_username: builtins.str - sasl_plain_password: builtins.str group_id: builtins.str def __init__( self, *, + sasl_plain_username: builtins.str = ..., + sasl_username_secret: secret_pb2.SecretRef | None = ..., + sasl_plain_password: builtins.str = ..., + sasl_password_secret: secret_pb2.SecretRef | None = ..., bootstrap_servers: builtins.str = ..., security_protocol: builtins.str = ..., sasl_mechanism: builtins.str = ..., sasl_jaas_config: builtins.str = ..., - sasl_plain_username: builtins.str | None = ..., - sasl_plain_password: builtins.str | None = ..., group_id: builtins.str = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_sasl_plain_password", b"_sasl_plain_password", "_sasl_plain_username", b"_sasl_plain_username", "sasl_plain_password", b"sasl_plain_password", "sasl_plain_username", b"sasl_plain_username"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_sasl_plain_password", b"_sasl_plain_password", "_sasl_plain_username", b"_sasl_plain_username", "bootstrap_servers", b"bootstrap_servers", "group_id", b"group_id", "sasl_jaas_config", b"sasl_jaas_config", "sasl_mechanism", b"sasl_mechanism", "sasl_plain_password", b"sasl_plain_password", "sasl_plain_username", b"sasl_plain_username", "security_protocol", b"security_protocol"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["sasl_password_secret", b"sasl_password_secret", "sasl_password_variant", b"sasl_password_variant", "sasl_plain_password", b"sasl_plain_password", "sasl_plain_username", b"sasl_plain_username", "sasl_username_secret", b"sasl_username_secret", "sasl_username_variant", b"sasl_username_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["bootstrap_servers", b"bootstrap_servers", "group_id", b"group_id", "sasl_jaas_config", b"sasl_jaas_config", "sasl_mechanism", b"sasl_mechanism", "sasl_password_secret", b"sasl_password_secret", "sasl_password_variant", b"sasl_password_variant", "sasl_plain_password", b"sasl_plain_password", "sasl_plain_username", b"sasl_plain_username", "sasl_username_secret", b"sasl_username_secret", "sasl_username_variant", b"sasl_username_variant", "security_protocol", b"security_protocol"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_sasl_plain_password", b"_sasl_plain_password"]) -> typing_extensions.Literal["sasl_plain_password"] | None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["sasl_password_variant", b"sasl_password_variant"]) -> typing_extensions.Literal["sasl_plain_password", "sasl_password_secret"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_sasl_plain_username", b"_sasl_plain_username"]) -> typing_extensions.Literal["sasl_plain_username"] | None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["sasl_username_variant", b"sasl_username_variant"]) -> typing_extensions.Literal["sasl_plain_username", "sasl_username_secret"] | None: ... global___Kafka = Kafka @@ -479,16 +546,29 @@ class Credentials(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor USERNAME_FIELD_NUMBER: builtins.int + USERNAME_SECRET_FIELD_NUMBER: builtins.int PASSWORD_FIELD_NUMBER: builtins.int + PASSWORD_SECRET_FIELD_NUMBER: builtins.int username: builtins.str + @property + def username_secret(self) -> secret_pb2.SecretRef: ... password: builtins.str + @property + def password_secret(self) -> secret_pb2.SecretRef: ... def __init__( self, *, username: builtins.str = ..., + username_secret: secret_pb2.SecretRef | None = ..., password: builtins.str = ..., + password_secret: secret_pb2.SecretRef | None = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["password", b"password", "username", b"username"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "username", b"username", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "username", b"username", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["username", "username_secret"] | None: ... global___Credentials = Credentials @@ -546,23 +626,36 @@ global___Redshift = Redshift class Mongo(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - HOST_FIELD_NUMBER: builtins.int - DATABASE_FIELD_NUMBER: builtins.int USER_FIELD_NUMBER: builtins.int + USERNAME_SECRET_FIELD_NUMBER: builtins.int PASSWORD_FIELD_NUMBER: builtins.int - host: builtins.str - database: builtins.str + PASSWORD_SECRET_FIELD_NUMBER: builtins.int + HOST_FIELD_NUMBER: builtins.int + DATABASE_FIELD_NUMBER: builtins.int user: builtins.str + @property + def username_secret(self) -> secret_pb2.SecretRef: ... password: builtins.str + @property + def password_secret(self) -> secret_pb2.SecretRef: ... + host: builtins.str + database: builtins.str def __init__( self, *, - host: builtins.str = ..., - database: builtins.str = ..., user: builtins.str = ..., + username_secret: secret_pb2.SecretRef | None = ..., password: builtins.str = ..., + password_secret: secret_pb2.SecretRef | None = ..., + host: builtins.str = ..., + database: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "password", b"password", "user", b"user"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["user", "username_secret"] | None: ... global___Mongo = Mongo @@ -570,17 +663,23 @@ global___Mongo = Mongo class PubSub(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - PROJECT_ID_FIELD_NUMBER: builtins.int SERVICE_ACCOUNT_KEY_FIELD_NUMBER: builtins.int - project_id: builtins.str + SERVICE_ACCOUNT_KEY_SECRET_FIELD_NUMBER: builtins.int + PROJECT_ID_FIELD_NUMBER: builtins.int service_account_key: builtins.str + @property + def service_account_key_secret(self) -> secret_pb2.SecretRef: ... + project_id: builtins.str def __init__( self, *, - project_id: builtins.str = ..., service_account_key: builtins.str = ..., + service_account_key_secret: secret_pb2.SecretRef | None = ..., + project_id: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["project_id", b"project_id", "service_account_key", b"service_account_key"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["service_account_key", b"service_account_key", "service_account_key_secret", b"service_account_key_secret", "service_account_key_variant", b"service_account_key_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["project_id", b"project_id", "service_account_key", b"service_account_key", "service_account_key_secret", b"service_account_key_secret", "service_account_key_variant", b"service_account_key_variant"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["service_account_key_variant", b"service_account_key_variant"]) -> typing_extensions.Literal["service_account_key", "service_account_key_secret"] | None: ... global___PubSub = PubSub diff --git a/fennel/gen/dataset_pb2.py b/fennel/gen/dataset_pb2.py index 30befc88d..e6176a5a7 100644 --- a/fennel/gen/dataset_pb2.py +++ b/fennel/gen/dataset_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: dataset.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/expectations_pb2.py b/fennel/gen/expectations_pb2.py index 67ed27576..a149c87e1 100644 --- a/fennel/gen/expectations_pb2.py +++ b/fennel/gen/expectations_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: expectations.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/expr_pb2.py b/fennel/gen/expr_pb2.py index e14582dab..95cfae375 100644 --- a/fennel/gen/expr_pb2.py +++ b/fennel/gen/expr_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: expr.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/featureset_pb2.py b/fennel/gen/featureset_pb2.py index 20c5a4ba5..4df083d90 100644 --- a/fennel/gen/featureset_pb2.py +++ b/fennel/gen/featureset_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: featureset.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/format_pb2.py b/fennel/gen/format_pb2.py index 5e7459a77..a83bd4cd6 100644 --- a/fennel/gen/format_pb2.py +++ b/fennel/gen/format_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: format.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/http_auth_pb2.py b/fennel/gen/http_auth_pb2.py index a51d66082..8816026d1 100644 --- a/fennel/gen/http_auth_pb2.py +++ b/fennel/gen/http_auth_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: http_auth.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -12,19 +13,20 @@ from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 +import fennel.gen.secret_pb2 as secret__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0fhttp_auth.proto\x12\x16\x66\x65nnel.proto.http_auth\x1a\x1egoogle/protobuf/wrappers.proto\"\x9b\x01\n\x12HTTPAuthentication\x12<\n\x05\x62\x61sic\x18\x01 \x01(\x0b\x32+.fennel.proto.http_auth.BasicAuthenticationH\x00\x12<\n\x05token\x18\x02 \x01(\x0b\x32+.fennel.proto.http_auth.TokenAuthenticationH\x00\x42\t\n\x07variant\"W\n\x13\x42\x61sicAuthentication\x12\x10\n\x08username\x18\x01 \x01(\t\x12.\n\x08password\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"$\n\x13TokenAuthentication\x12\r\n\x05token\x18\x01 \x01(\tb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0fhttp_auth.proto\x12\x16\x66\x65nnel.proto.http_auth\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x0csecret.proto\"\x9b\x01\n\x12HTTPAuthentication\x12<\n\x05\x62\x61sic\x18\x01 \x01(\x0b\x32+.fennel.proto.http_auth.BasicAuthenticationH\x00\x12<\n\x05token\x18\x02 \x01(\x0b\x32+.fennel.proto.http_auth.TokenAuthenticationH\x00\x42\t\n\x07variant\"\xf9\x01\n\x13\x42\x61sicAuthentication\x12\x12\n\x08username\x18\x01 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x03 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x30\n\x08password\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x01\x12\x39\n\x0fpassword_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x42\x12\n\x10username_variantB\x12\n\x10password_variant\"o\n\x13TokenAuthentication\x12\x0f\n\x05token\x18\x01 \x01(\tH\x00\x12\x36\n\x0ctoken_secret\x18\x02 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x42\x0f\n\rtoken_variantb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'http_auth_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _globals['_HTTPAUTHENTICATION']._serialized_start=76 - _globals['_HTTPAUTHENTICATION']._serialized_end=231 - _globals['_BASICAUTHENTICATION']._serialized_start=233 - _globals['_BASICAUTHENTICATION']._serialized_end=320 - _globals['_TOKENAUTHENTICATION']._serialized_start=322 - _globals['_TOKENAUTHENTICATION']._serialized_end=358 + _globals['_HTTPAUTHENTICATION']._serialized_start=90 + _globals['_HTTPAUTHENTICATION']._serialized_end=245 + _globals['_BASICAUTHENTICATION']._serialized_start=248 + _globals['_BASICAUTHENTICATION']._serialized_end=497 + _globals['_TOKENAUTHENTICATION']._serialized_start=499 + _globals['_TOKENAUTHENTICATION']._serialized_end=610 # @@protoc_insertion_point(module_scope) diff --git a/fennel/gen/http_auth_pb2.pyi b/fennel/gen/http_auth_pb2.pyi index f5f8cf6c3..e4607fff3 100644 --- a/fennel/gen/http_auth_pb2.pyi +++ b/fennel/gen/http_auth_pb2.pyi @@ -6,7 +6,9 @@ import builtins import google.protobuf.descriptor import google.protobuf.message import google.protobuf.wrappers_pb2 +import secret_pb2 import sys +import typing if sys.version_info >= (3, 8): import typing as typing_extensions @@ -42,18 +44,30 @@ class BasicAuthentication(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor USERNAME_FIELD_NUMBER: builtins.int + USERNAME_SECRET_FIELD_NUMBER: builtins.int PASSWORD_FIELD_NUMBER: builtins.int + PASSWORD_SECRET_FIELD_NUMBER: builtins.int username: builtins.str @property + def username_secret(self) -> secret_pb2.SecretRef: ... + @property def password(self) -> google.protobuf.wrappers_pb2.StringValue: ... + @property + def password_secret(self) -> secret_pb2.SecretRef: ... def __init__( self, *, username: builtins.str = ..., + username_secret: secret_pb2.SecretRef | None = ..., password: google.protobuf.wrappers_pb2.StringValue | None = ..., + password_secret: secret_pb2.SecretRef | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["password", b"password"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["password", b"password", "username", b"username"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "username", b"username", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "username", b"username", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["username", "username_secret"] | None: ... global___BasicAuthentication = BasicAuthentication @@ -62,12 +76,18 @@ class TokenAuthentication(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor TOKEN_FIELD_NUMBER: builtins.int + TOKEN_SECRET_FIELD_NUMBER: builtins.int token: builtins.str + @property + def token_secret(self) -> secret_pb2.SecretRef: ... def __init__( self, *, token: builtins.str = ..., + token_secret: secret_pb2.SecretRef | None = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["token", b"token"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["token", b"token", "token_secret", b"token_secret", "token_variant", b"token_variant"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["token", b"token", "token_secret", b"token_secret", "token_variant", b"token_variant"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["token_variant", b"token_variant"]) -> typing_extensions.Literal["token", "token_secret"] | None: ... global___TokenAuthentication = TokenAuthentication diff --git a/fennel/gen/index_pb2.py b/fennel/gen/index_pb2.py index 9e3863f89..d7cfdffba 100644 --- a/fennel/gen/index_pb2.py +++ b/fennel/gen/index_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: index.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/kinesis_pb2.py b/fennel/gen/kinesis_pb2.py index 520ff7fd2..37fe3cc62 100644 --- a/fennel/gen/kinesis_pb2.py +++ b/fennel/gen/kinesis_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: kinesis.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/metadata_pb2.py b/fennel/gen/metadata_pb2.py index d9d355cc9..82b9ffc47 100644 --- a/fennel/gen/metadata_pb2.py +++ b/fennel/gen/metadata_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: metadata.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/pycode_pb2.py b/fennel/gen/pycode_pb2.py index d17598f79..d53c3edee 100644 --- a/fennel/gen/pycode_pb2.py +++ b/fennel/gen/pycode_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: pycode.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/schema_pb2.py b/fennel/gen/schema_pb2.py index e8d3e1501..4f56afb6f 100644 --- a/fennel/gen/schema_pb2.py +++ b/fennel/gen/schema_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: schema.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/schema_registry_pb2.py b/fennel/gen/schema_registry_pb2.py index 5bfb5b171..f1248b253 100644 --- a/fennel/gen/schema_registry_pb2.py +++ b/fennel/gen/schema_registry_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: schema_registry.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/secret_pb2.py b/fennel/gen/secret_pb2.py new file mode 100644 index 000000000..ae0f9d602 --- /dev/null +++ b/fennel/gen/secret_pb2.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: secret.proto +# Protobuf Python Version: 4.25.4 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0csecret.proto\x12\x13\x66\x65nnel.proto.secret\"Q\n\tSecretRef\x12\x12\n\nsecret_arn\x18\x01 \x01(\t\x12\x15\n\x08role_arn\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x0c\n\x04path\x18\x03 \x03(\tB\x0b\n\t_role_arnb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'secret_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_SECRETREF']._serialized_start=37 + _globals['_SECRETREF']._serialized_end=118 +# @@protoc_insertion_point(module_scope) diff --git a/fennel/gen/secret_pb2.pyi b/fennel/gen/secret_pb2.pyi new file mode 100644 index 000000000..9f6ff1cc7 --- /dev/null +++ b/fennel/gen/secret_pb2.pyi @@ -0,0 +1,41 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class SecretRef(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SECRET_ARN_FIELD_NUMBER: builtins.int + ROLE_ARN_FIELD_NUMBER: builtins.int + PATH_FIELD_NUMBER: builtins.int + secret_arn: builtins.str + role_arn: builtins.str + @property + def path(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + def __init__( + self, + *, + secret_arn: builtins.str = ..., + role_arn: builtins.str | None = ..., + path: collections.abc.Iterable[builtins.str] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_role_arn", b"_role_arn", "role_arn", b"role_arn"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_role_arn", b"_role_arn", "path", b"path", "role_arn", b"role_arn", "secret_arn", b"secret_arn"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_role_arn", b"_role_arn"]) -> typing_extensions.Literal["role_arn"] | None: ... + +global___SecretRef = SecretRef diff --git a/fennel/gen/services_pb2.py b/fennel/gen/services_pb2.py index 00770b71a..0bf595aed 100644 --- a/fennel/gen/services_pb2.py +++ b/fennel/gen/services_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: services.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/spec_pb2.py b/fennel/gen/spec_pb2.py index f2a47489a..44f59b34b 100644 --- a/fennel/gen/spec_pb2.py +++ b/fennel/gen/spec_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: spec.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/status_pb2.py b/fennel/gen/status_pb2.py index cdc144a8b..f24662be1 100644 --- a/fennel/gen/status_pb2.py +++ b/fennel/gen/status_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: status.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/gen/window_pb2.py b/fennel/gen/window_pb2.py index ac2bb8d59..6b51a69dd 100644 --- a/fennel/gen/window_pb2.py +++ b/fennel/gen/window_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: window.proto +# Protobuf Python Version: 4.25.4 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/fennel/integrations/__init__.py b/fennel/integrations/__init__.py new file mode 100644 index 000000000..c1e677e89 --- /dev/null +++ b/fennel/integrations/__init__.py @@ -0,0 +1 @@ +from fennel.integrations.aws import Secret diff --git a/fennel/integrations/aws.py b/fennel/integrations/aws.py new file mode 100644 index 000000000..c12996b94 --- /dev/null +++ b/fennel/integrations/aws.py @@ -0,0 +1,19 @@ +from typing import Optional, List + + +class Secret: + arn: str + role_arn: Optional[str] + path: List[str] + + def __init__(self, arn, role_arn=None, path=None): + self.arn = arn + self.role_arn = role_arn + self.path = path + + def __getitem__(self, key): + if self.path is None: + self.path = [] + return Secret( + arn=self.arn, role_arn=self.role_arn, path=self.path + [key] + ) diff --git a/fennel/internal_lib/to_proto/to_proto.py b/fennel/internal_lib/to_proto/to_proto.py index 070280153..9ffda478f 100644 --- a/fennel/internal_lib/to_proto/to_proto.py +++ b/fennel/internal_lib/to_proto/to_proto.py @@ -14,6 +14,7 @@ Mapping, Set, Callable, + Union, ) import google.protobuf.duration_pb2 as duration_proto # type: ignore @@ -22,6 +23,7 @@ import fennel.connectors as connectors import fennel.gen.connector_pb2 as connector_proto +import fennel.gen.secret_pb2 as secret_proto import fennel.gen.dataset_pb2 as ds_proto import fennel.gen.expectations_pb2 as exp_proto import fennel.gen.featureset_pb2 as fs_proto @@ -43,6 +45,7 @@ from fennel.expr.serializer import ExprSerializer from fennel.expr.visitor import ExprPrinter from fennel.featuresets import Featureset, Feature, Extractor, ExtractorType +from fennel.integrations.aws import Secret from fennel.internal_lib.duration import ( Duration, duration_to_timedelta, @@ -1024,8 +1027,8 @@ def _kafka_to_ext_db_proto( bootstrap_servers: str, security_protocol: str, sasl_mechanism: str, - sasl_plain_username: Optional[str], - sasl_plain_password: Optional[str], + sasl_plain_username: Optional[Union[str, Secret]], + sasl_plain_password: Optional[Union[str, Secret]], ) -> connector_proto.ExtDatabase: return connector_proto.ExtDatabase( name=name, @@ -1033,9 +1036,11 @@ def _kafka_to_ext_db_proto( bootstrap_servers=bootstrap_servers, security_protocol=security_protocol, sasl_mechanism=sasl_mechanism, - sasl_plain_username=sasl_plain_username, - sasl_plain_password=sasl_plain_password, + sasl_plain_username=to_string(sasl_plain_username), + sasl_plain_password=to_string(sasl_plain_password), sasl_jaas_config="", + sasl_password_secret=to_secret_proto(sasl_plain_password), + sasl_username_secret=to_secret_proto(sasl_plain_username), ), ) @@ -1163,16 +1168,18 @@ def _s3_conn_to_sink_proto( def _s3_to_ext_db_proto( name: str, - aws_access_key_id: Optional[str], - aws_secret_access_key: Optional[str], + aws_access_key_id: Optional[Union[str, Secret]], + aws_secret_access_key: Optional[Union[str, Secret]], role_arn: Optional[str], ) -> connector_proto.ExtDatabase: return connector_proto.ExtDatabase( name=name, s3=connector_proto.S3( - aws_access_key_id=aws_access_key_id or "", - aws_secret_access_key=aws_secret_access_key or "", + aws_access_key_id=to_string(aws_access_key_id), + aws_secret_access_key=to_string(aws_secret_access_key), role_arn=role_arn, + aws_access_key_id_secret=to_secret_proto(aws_access_key_id), + aws_secret_access_key_secret=to_secret_proto(aws_secret_access_key), ), ) @@ -1246,12 +1253,15 @@ def _bigquery_conn_to_source_proto( ) -> Tuple[connector_proto.ExtDatabase, connector_proto.Source]: if not connector.cdc: raise ValueError("CDC should always be set for BigQuery source") + service_account_key = connector.data_source.service_account_key + if type(service_account_key) is dict: + # Convert service_account_key to str defined in proto + service_account_key = json.dumps(service_account_key) ext_db = _bigquery_to_ext_db_proto( data_source.name, data_source.project_id, data_source.dataset_id, - # Convert service_account_key to str defined in proto - json.dumps(data_source.service_account_key), + service_account_key, ) ext_table = _bigquery_to_ext_table_proto( ext_db, @@ -1286,14 +1296,15 @@ def _bigquery_to_ext_db_proto( name: str, project_id: str, dataset_id: str, - service_account_key: str, + service_account_key: Union[str, Secret], ) -> connector_proto.ExtDatabase: return connector_proto.ExtDatabase( name=name, bigquery=connector_proto.Bigquery( project_id=project_id, dataset_id=dataset_id, - service_account_key=service_account_key, + service_account_key=to_string(service_account_key), + service_account_key_secret=to_secret_proto(service_account_key), ), ) @@ -1358,8 +1369,8 @@ def _redshift_conn_to_source_proto( def _redshift_to_authentication_proto( s3_access_role_arn: Optional[str], - username: Optional[str], - password: Optional[str], + username: Optional[Union[str, Secret]], + password: Optional[Union[str, Secret]], ): if s3_access_role_arn: return connector_proto.RedshiftAuthentication( @@ -1368,7 +1379,10 @@ def _redshift_to_authentication_proto( else: return connector_proto.RedshiftAuthentication( credentials=connector_proto.Credentials( - username=username, password=password + username=to_string(username), + password=to_string(password), + username_secret=to_secret_proto(username), + password_secret=to_secret_proto(password), ) ) @@ -1376,8 +1390,8 @@ def _redshift_to_authentication_proto( def _redshift_to_ext_db_proto( name: str, s3_access_role_arn: Optional[str], - username: Optional[str], - password: Optional[str], + username: Optional[Union[str, Secret]], + password: Optional[Union[str, Secret]], host: str, port: int, database: str, @@ -1456,16 +1470,18 @@ def _mongo_to_ext_db_proto( name: str, host: str, database: str, - user: str, - password: str, + user: Union[str, Secret], + password: Union[str, Secret], ) -> connector_proto.ExtDatabase: return connector_proto.ExtDatabase( name=name, mongo=connector_proto.Mongo( host=host, database=database, - user=user, - password=password, + user=to_string(user), + password=to_string(password), + username_secret=to_secret_proto(user), + password_secret=to_secret_proto(password), ), ) @@ -1485,15 +1501,19 @@ def _pubsub_conn_to_source_proto( connector: connectors.PubSubConnector, dataset: Dataset, ) -> Tuple[connector_proto.ExtDatabase, connector_proto.Source]: + service_account_key = connector.data_source.service_account_key + data_source = connector.data_source + if type(service_account_key) is dict: + # Convert service_account_key to str defined in proto + service_account_key = json.dumps(service_account_key) if not connector.cdc: raise ValueError("CDC should always be set for PubSub source") - data_source = connector.data_source ext_db = connector_proto.ExtDatabase( name=data_source.name, pubsub=connector_proto.PubSub( project_id=data_source.project_id, - # Convert service_account_key to str defined in proto - service_account_key=json.dumps(data_source.service_account_key), + service_account_key=to_string(service_account_key), + service_account_key_secret=to_secret_proto(service_account_key), ), ) return ( @@ -1567,11 +1587,33 @@ def _snowflake_conn_to_source_proto( ) +def to_secret_proto(secret: Optional[Union[str, Secret]]): + if secret is None or type(secret) is not Secret: + return None + return secret_proto.SecretRef( + secret_arn=secret.arn, + role_arn=secret.role_arn, + path=secret.path, + ) + + +def to_string(s): + if s is not None and type(s) is str: + return s + return None + + +def to_string_value(s): + if to_string(s): + return StringValue(value=s) + return None + + def _snowflake_to_ext_db_proto( name: str, account: str, - user: str, - password: str, + user: Union[str, Secret], + password: Union[str, Secret], schema: str, warehouse: str, role: str, @@ -1581,12 +1623,14 @@ def _snowflake_to_ext_db_proto( name=name, snowflake=connector_proto.Snowflake( account=account, - user=user, - password=password, + user=to_string(user), + password=to_string(password), schema=schema, warehouse=warehouse, role=role, database=database, + username_secret=to_secret_proto(user), + password_secret=to_secret_proto(password), ), ) @@ -1653,8 +1697,8 @@ def _mysql_to_ext_db_proto( name: str, host: str, database: str, - user: str, - password: str, + user: Union[str, Secret], + password: Union[str, Secret], port: int, jdbc_params: Optional[str] = None, ) -> connector_proto.ExtDatabase: @@ -1665,10 +1709,12 @@ def _mysql_to_ext_db_proto( mysql=connector_proto.MySQL( host=host, database=database, - user=user, - password=password, + user=to_string(user), + password=to_string(password), port=port, jdbc_params=jdbc_params, + username_secret=to_secret_proto(user), + password_secret=to_secret_proto(password), ), ) @@ -1744,8 +1790,8 @@ def _pg_to_ext_db_proto( name: str, host: str, database: str, - user: str, - password: str, + user: Union[str, Secret], + password: Union[str, Secret], port: int, jdbc_params: Optional[str] = None, ) -> connector_proto.ExtDatabase: @@ -1757,10 +1803,12 @@ def _pg_to_ext_db_proto( postgres=connector_proto.Postgres( host=host, database=database, - user=user, - password=password, + user=to_string(user), + password=to_string(password), port=port, jdbc_params=jdbc_params, + username_secret=to_secret_proto(user), + password_secret=to_secret_proto(password), ), ) @@ -1947,17 +1995,25 @@ def to_kafka_format_proto( def to_auth_proto( - username: Optional[str], password: Optional[str], token: Optional[str] + username: Optional[Union[str, Secret]], + password: Optional[Union[str, Secret]], + token: Optional[Union[str, Secret]], ) -> Optional[http_auth_proto.HTTPAuthentication]: if username is not None: return http_auth_proto.HTTPAuthentication( basic=http_auth_proto.BasicAuthentication( - username=username, password=StringValue(value=password) + username=to_string(username), + password=to_string_value(password), + username_secret=to_secret_proto(username), + password_secret=to_secret_proto(password), ) ) if token is not None: return http_auth_proto.HTTPAuthentication( - token=http_auth_proto.TokenAuthentication(token=token) + token=http_auth_proto.TokenAuthentication( + token=to_string(token), + token_secret=to_secret_proto(token), + ) ) return None diff --git a/fennel/testing/test_utils.py b/fennel/testing/test_utils.py index f9c6b071b..43cda0e86 100644 --- a/fennel/testing/test_utils.py +++ b/fennel/testing/test_utils.py @@ -44,6 +44,7 @@ def error_message(actual: Any, expected: Any) -> str: # Don't delete the following line. It is used to debug the test in # case of failure. print(actual_dict) + print(expected_dict) return jsondiff.diff(expected_dict, actual_dict, syntax="symmetric") diff --git a/pyproject.toml b/pyproject.toml index 5361c2fe3..5a582dd59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fennel-ai" -version = "1.5.33" +version = "1.5.34" description = "The modern realtime feature engineering platform" authors = ["Fennel AI "] packages = [{ include = "fennel" }]