From e3a8d4985941a55d1c5ed5f106677dba3fadfa48 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:01:36 +0000 Subject: [PATCH 01/20] feat: dlt embedded elt --- .../libraries/dlt_sources/.dlt/.sources | 28 ++ .../dlt_sources/sql_database/__init__.py | 204 +++++++++++++ .../dlt_sources/sql_database/arrow_helpers.py | 139 +++++++++ .../dlt_sources/sql_database/helpers.py | 284 ++++++++++++++++++ .../dlt_sources/sql_database/schema_types.py | 130 ++++++++ 5 files changed, 785 insertions(+) create mode 100644 src/teamster/libraries/dlt_sources/.dlt/.sources create mode 100644 src/teamster/libraries/dlt_sources/sql_database/__init__.py create mode 100644 src/teamster/libraries/dlt_sources/sql_database/arrow_helpers.py create mode 100644 src/teamster/libraries/dlt_sources/sql_database/helpers.py create mode 100644 src/teamster/libraries/dlt_sources/sql_database/schema_types.py diff --git a/src/teamster/libraries/dlt_sources/.dlt/.sources b/src/teamster/libraries/dlt_sources/.dlt/.sources new file mode 100644 index 0000000000..36c60faf33 --- /dev/null +++ b/src/teamster/libraries/dlt_sources/.dlt/.sources @@ -0,0 +1,28 @@ +engine_version: 1 +sources: + sql_database: + is_dirty: false + last_commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 + last_commit_timestamp: '2024-07-23T09:43:26+02:00' + files: + sql_database/README.md: + commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 + git_sha: dfa4b5e161f059b8399db203801ed0d638b7734c + sha3_256: a21100e31c5f4d514b38ba6229735d3c3a9a7f364ef54d2e9d6068c75c8d8e2b + sql_database/helpers.py: + commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 + git_sha: fa054cc90296a5395718659f110e16e8e4c686a3 + sha3_256: a9ce47a86aff20a50fe24a9c88f728b6f28e77972059ca5a5bde9f3e28d766fa + sql_database/schema_types.py: + commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 + git_sha: 12af2d5f39a8f3a753db66bfca72ed9a3f7a0193 + sha3_256: 5c3b3a1e24ed780ca7a6f2c5404008391d6a9e90c0901ea863648c5516338406 + sql_database/__init__.py: + commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 + git_sha: 93b1a0869ad33971f963567cbfecd0049ab401a4 + sha3_256: 8b7e89b223e128bce22e901d63a1bb93b67124540058e17c57253eded000db09 + sql_database/arrow_helpers.py: + commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 + git_sha: bf045e5efed225f6525fcbbfd411395437030471 + sha3_256: 6841f763c5775e0c161df98c14270342328111e8d78ef1b0a21c7a6e08605672 + dlt_version_constraint: '>=0.4.11' diff --git a/src/teamster/libraries/dlt_sources/sql_database/__init__.py b/src/teamster/libraries/dlt_sources/sql_database/__init__.py new file mode 100644 index 0000000000..587e1d1eeb --- /dev/null +++ b/src/teamster/libraries/dlt_sources/sql_database/__init__.py @@ -0,0 +1,204 @@ +"""Source that loads tables form any SQLAlchemy supported database, supports batching requests and incremental loads.""" + +from typing import Any, Callable, Dict, Iterable, List, Optional, Union + +import dlt +from dlt.sources import DltResource +from dlt.sources.credentials import ConnectionStringCredentials +from sqlalchemy import MetaData, Table +from sqlalchemy.engine import Engine + +from .helpers import ( + SqlDatabaseTableConfiguration, + SqlTableResourceConfiguration, + TableBackend, + _detect_precision_hints_deprecated, + engine_from_credentials, + table_rows, +) +from .schema_types import ( + ReflectionLevel, + TTypeAdapter, + get_primary_key, + table_to_columns, +) + + +@dlt.source +def sql_database( + credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value, + schema: Optional[str] = dlt.config.value, + metadata: Optional[MetaData] = None, + table_names: Optional[List[str]] = dlt.config.value, + chunk_size: int = 50000, + backend: TableBackend = "sqlalchemy", + detect_precision_hints: Optional[bool] = False, + reflection_level: Optional[ReflectionLevel] = "full", + defer_table_reflect: Optional[bool] = None, + table_adapter_callback: Callable[[Table], None] = None, + backend_kwargs: Dict[str, Any] = None, + include_views: bool = False, + type_adapter_callback: Optional[TTypeAdapter] = None, +) -> Iterable[DltResource]: + """ + A dlt source which loads data from an SQL database using SQLAlchemy. + Resources are automatically created for each table in the schema or from the given list of tables. + + Args: + credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `sqlalchemy.Engine` instance. + schema (Optional[str]): Name of the database schema to load (if different from default). + metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. `schema` argument is ignored when this is used. + table_names (Optional[List[str]]): A list of table names to load. By default, all tables in the schema are loaded. + chunk_size (int): Number of rows yielded in one batch. SQL Alchemy will create additional internal rows buffer twice the chunk size. + backend (TableBackend): Type of backend to generate table data. One of: "sqlalchemy", "pyarrow", "pandas" and "connectorx". + "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames. + "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types, + "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself. + detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables. + This is disabled by default. + reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema. + "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data. + "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option. + "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types. + defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Requires table_names to be explicitly passed. + Enable this option when running on Airflow. Available on dlt 0.4.4 and later. + table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected. + backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx. + include_views (bool): Reflect views as well as tables. Note view names included in `table_names` are always included regardless of this setting. + type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns. + Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data) + Returns: + + Iterable[DltResource]: A list of DLT resources for each table to be loaded. + """ + # detect precision hints is deprecated + _detect_precision_hints_deprecated(detect_precision_hints) + + if detect_precision_hints: + reflection_level = "full_with_precision" + else: + reflection_level = reflection_level or "minimal" + + # set up alchemy engine + engine = engine_from_credentials(credentials) + engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size) + metadata = metadata or MetaData(schema=schema) + + # use provided tables or all tables + if table_names: + tables = [ + Table(name, metadata, autoload_with=None if defer_table_reflect else engine) + for name in table_names + ] + else: + if defer_table_reflect: + raise ValueError("You must pass table names to defer table reflection") + metadata.reflect(bind=engine, views=include_views) + tables = list(metadata.tables.values()) + + for table in tables: + if table_adapter_callback and not defer_table_reflect: + table_adapter_callback(table) + + yield dlt.resource( + table_rows, + name=table.name, + primary_key=get_primary_key(table), + spec=SqlDatabaseTableConfiguration, + columns=table_to_columns(table, reflection_level, type_adapter_callback), + )( + engine, + table, + chunk_size, + backend, + reflection_level=reflection_level, + defer_table_reflect=defer_table_reflect, + table_adapter_callback=table_adapter_callback, + backend_kwargs=backend_kwargs, + type_adapter_callback=type_adapter_callback, + ) + + +@dlt.resource( + name=lambda args: args["table"], standalone=True, spec=SqlTableResourceConfiguration +) +def sql_table( + credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value, + table: str = dlt.config.value, + schema: Optional[str] = dlt.config.value, + metadata: Optional[MetaData] = None, + incremental: Optional[dlt.sources.incremental[Any]] = None, + chunk_size: int = 50000, + backend: TableBackend = "sqlalchemy", + detect_precision_hints: Optional[bool] = None, + reflection_level: Optional[ReflectionLevel] = "full", + defer_table_reflect: Optional[bool] = None, + table_adapter_callback: Callable[[Table], None] = None, + backend_kwargs: Dict[str, Any] = None, + type_adapter_callback: Optional[TTypeAdapter] = None, +) -> DltResource: + """ + A dlt resource which loads data from an SQL database table using SQLAlchemy. + + Args: + credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `Engine` instance representing the database connection. + table (str): Name of the table or view to load. + schema (Optional[str]): Optional name of the schema the table belongs to. + metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. If provided, the `schema` argument is ignored. + incremental (Optional[dlt.sources.incremental[Any]]): Option to enable incremental loading for the table. + E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))` + chunk_size (int): Number of rows yielded in one batch. SQL Alchemy will create additional internal rows buffer twice the chunk size. + backend (TableBackend): Type of backend to generate table data. One of: "sqlalchemy", "pyarrow", "pandas" and "connectorx". + "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames. + "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types, + "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself. + reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema. + "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data. + "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option. + "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types. + detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables. + This is disabled by default. + defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Enable this option when running on Airflow. Available + on dlt 0.4.4 and later + table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected. + backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx. + type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns. + Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data) + + Returns: + DltResource: The dlt resource for loading data from the SQL database table. + """ + _detect_precision_hints_deprecated(detect_precision_hints) + + if detect_precision_hints: + reflection_level = "full_with_precision" + else: + reflection_level = reflection_level or "minimal" + + engine = engine_from_credentials(credentials, may_dispose_after_use=True) + engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size) + metadata = metadata or MetaData(schema=schema) + + table_obj = Table( + table, metadata, autoload_with=None if defer_table_reflect else engine + ) + if table_adapter_callback and not defer_table_reflect: + table_adapter_callback(table_obj) + + return dlt.resource( + table_rows, + name=table_obj.name, + primary_key=get_primary_key(table_obj), + columns=table_to_columns(table_obj, reflection_level, type_adapter_callback), + )( + engine, + table_obj, + chunk_size, + backend, + incremental=incremental, + reflection_level=reflection_level, + defer_table_reflect=defer_table_reflect, + table_adapter_callback=table_adapter_callback, + backend_kwargs=backend_kwargs, + type_adapter_callback=type_adapter_callback, + ) diff --git a/src/teamster/libraries/dlt_sources/sql_database/arrow_helpers.py b/src/teamster/libraries/dlt_sources/sql_database/arrow_helpers.py new file mode 100644 index 0000000000..62c06bc838 --- /dev/null +++ b/src/teamster/libraries/dlt_sources/sql_database/arrow_helpers.py @@ -0,0 +1,139 @@ +from typing import Any, Optional, Sequence + +from dlt.common import logger +from dlt.common.configuration import with_config +from dlt.common.destination import DestinationCapabilitiesContext +from dlt.common.json import custom_encode, map_nested_in_place +from dlt.common.schema.typing import TTableSchemaColumns + +from .schema_types import RowAny + + +@with_config +def columns_to_arrow( + columns_schema: TTableSchemaColumns, + caps: DestinationCapabilitiesContext = None, + tz: str = "UTC", +) -> Any: + """Converts `column_schema` to arrow schema using `caps` and `tz`. `caps` are injected from the container - which + is always the case if run within the pipeline. This will generate arrow schema compatible with the destination. + Otherwise generic capabilities are used + """ + from dlt.common.destination.capabilities import DestinationCapabilitiesContext + from dlt.common.libs.pyarrow import get_py_arrow_datatype + from dlt.common.libs.pyarrow import pyarrow as pa + + return pa.schema( + [ + pa.field( + name, + get_py_arrow_datatype( + schema_item, + caps or DestinationCapabilitiesContext.generic_capabilities(), + tz, + ), + nullable=schema_item.get("nullable", True), + ) + for name, schema_item in columns_schema.items() + if schema_item.get("data_type") is not None + ] + ) + + +def row_tuples_to_arrow( + rows: Sequence[RowAny], columns: TTableSchemaColumns, tz: str +) -> Any: + """Converts the rows to an arrow table using the columns schema. + Columns missing `data_type` will be inferred from the row data. + Columns with object types not supported by arrow are excluded from the resulting table. + """ + import numpy as np + from dlt.common.libs.pyarrow import pyarrow as pa + + try: + from pandas._libs import lib + + pivoted_rows = lib.to_object_array_tuples(rows).T # type: ignore[attr-defined] + except ImportError: + logger.info( + "Pandas not installed, reverting to numpy.asarray to create a table which is slower" + ) + pivoted_rows = np.asarray(rows, dtype="object", order="k").T # type: ignore[call-overload] + + columnar = { + col: dat.ravel() + for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns))) + } + columnar_known_types = { + col["name"]: columnar[col["name"]] + for col in columns.values() + if col.get("data_type") is not None + } + columnar_unknown_types = { + col["name"]: columnar[col["name"]] + for col in columns.values() + if col.get("data_type") is None + } + + arrow_schema = columns_to_arrow(columns, tz=tz) + + for idx in range(0, len(arrow_schema.names)): + field = arrow_schema.field(idx) + py_type = type(rows[0][idx]) + # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects + if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)): + logger.warning( + f"Field {field.name} was reflected as decimal type, but rows contains {py_type.__name__}. Additional cast is required which may slow down arrow table generation." + ) + float_array = pa.array(columnar_known_types[field.name], type=pa.float64()) + columnar_known_types[field.name] = float_array.cast(field.type, safe=False) + + # If there are unknown type columns, first create a table to infer their types + if columnar_unknown_types: + new_schema_fields = [] + for key in list(columnar_unknown_types): + arrow_col: Optional[pa.Array] = None + try: + arrow_col = pa.array(columnar_unknown_types[key]) + if pa.types.is_null(arrow_col.type): + logger.warning( + f"Column {key} contains only NULL values and data type could not be inferred. This column is removed from a arrow table" + ) + continue + + except pa.ArrowInvalid as e: + # Try coercing types not supported by arrow to a json friendly format + # E.g. dataclasses -> dict, UUID -> str + try: + arrow_col = pa.array( + map_nested_in_place( + custom_encode, list(columnar_unknown_types[key]) + ) + ) + logger.warning( + f"Column {key} contains a data type which is not supported by pyarrow and got converted into {arrow_col.type}. This slows down arrow table generation." + ) + except (pa.ArrowInvalid, TypeError): + logger.warning( + f"Column {key} contains a data type which is not supported by pyarrow. This column will be ignored. Error: {e}" + ) + if arrow_col is not None: + columnar_known_types[key] = arrow_col + new_schema_fields.append( + pa.field( + key, + arrow_col.type, + nullable=columns[key]["nullable"], + ) + ) + + # New schema + column_order = {name: idx for idx, name in enumerate(columns)} + arrow_schema = pa.schema( + sorted( + list(arrow_schema) + new_schema_fields, + key=lambda x: column_order[x.name], + ) + ) + + return pa.Table.from_pydict(columnar_known_types, schema=arrow_schema) diff --git a/src/teamster/libraries/dlt_sources/sql_database/helpers.py b/src/teamster/libraries/dlt_sources/sql_database/helpers.py new file mode 100644 index 0000000000..f2c9a1a4ab --- /dev/null +++ b/src/teamster/libraries/dlt_sources/sql_database/helpers.py @@ -0,0 +1,284 @@ +"""SQL database source helpers""" + +import operator +import warnings +from typing import Any, Callable, Dict, Iterator, Literal, Optional, Union + +import dlt +from dlt.common.configuration.specs import BaseConfiguration, configspec +from dlt.common.exceptions import MissingDependencyException +from dlt.common.schema import TTableSchemaColumns +from dlt.common.typing import TDataItem, TSortOrder +from dlt.sources.credentials import ConnectionStringCredentials +from sqlalchemy import Table, create_engine +from sqlalchemy.engine import Engine +from sqlalchemy.exc import CompileError + +from .arrow_helpers import row_tuples_to_arrow +from .schema_types import ( + ReflectionLevel, + SelectAny, + TTypeAdapter, + get_primary_key, + table_to_columns, +) + +TableBackend = Literal["sqlalchemy", "pyarrow", "pandas", "connectorx"] + + +class TableLoader: + def __init__( + self, + engine: Engine, + backend: TableBackend, + table: Table, + columns: TTableSchemaColumns, + chunk_size: int = 1000, + incremental: Optional[dlt.sources.incremental[Any]] = None, + ) -> None: + self.engine = engine + self.backend = backend + self.table = table + self.columns = columns + self.chunk_size = chunk_size + self.incremental = incremental + if incremental: + try: + self.cursor_column = table.c[incremental.cursor_path] + except KeyError as e: + raise KeyError( + f"Cursor column '{incremental.cursor_path}' does not exist in table '{table.name}'" + ) from e + self.last_value = incremental.last_value + self.end_value = incremental.end_value + self.row_order: TSortOrder = self.incremental.row_order + else: + self.cursor_column = None + self.last_value = None + self.end_value = None + self.row_order = None + + def make_query(self) -> SelectAny: + table = self.table + query = table.select() + if not self.incremental: + return query + last_value_func = self.incremental.last_value_func + + # generate where + if ( + last_value_func is max + ): # Query ordered and filtered according to last_value function + filter_op = operator.ge + filter_op_end = operator.lt + elif last_value_func is min: + filter_op = operator.le + filter_op_end = operator.gt + else: # Custom last_value, load everything and let incremental handle filtering + return query + + if self.last_value is not None: + query = query.where(filter_op(self.cursor_column, self.last_value)) + if self.end_value is not None: + query = query.where(filter_op_end(self.cursor_column, self.end_value)) + + # generate order by from declared row order + order_by = None + if (self.row_order == "asc" and last_value_func is max) or ( + self.row_order == "desc" and last_value_func is min + ): + order_by = self.cursor_column.asc() + elif (self.row_order == "asc" and last_value_func is min) or ( + self.row_order == "desc" and last_value_func is max + ): + order_by = self.cursor_column.desc() + if order_by is not None: + query = query.order_by(order_by) + + return query + + def load_rows(self, backend_kwargs: Dict[str, Any] = None) -> Iterator[TDataItem]: + # make copy of kwargs + backend_kwargs = dict(backend_kwargs or {}) + query = self.make_query() + if self.backend == "connectorx": + yield from self._load_rows_connectorx(query, backend_kwargs) + else: + yield from self._load_rows(query, backend_kwargs) + + def _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem: + with self.engine.connect() as conn: + result = conn.execution_options(yield_per=self.chunk_size).execute(query) + # NOTE: cursor returns not normalized column names! may be quite useful in case of Oracle dialect + # that normalizes columns + # columns = [c[0] for c in result.cursor.description] + columns = list(result.keys()) + for partition in result.partitions(size=self.chunk_size): + if self.backend == "sqlalchemy": + yield [dict(row._mapping) for row in partition] + elif self.backend == "pandas": + from dlt.common.libs.pandas_sql import _wrap_result + + df = _wrap_result( + partition, + columns, + **{"dtype_backend": "pyarrow", **backend_kwargs}, + ) + yield df + elif self.backend == "pyarrow": + yield row_tuples_to_arrow( + partition, self.columns, tz=backend_kwargs.get("tz", "UTC") + ) + + def _load_rows_connectorx( + self, query: SelectAny, backend_kwargs: Dict[str, Any] + ) -> Iterator[TDataItem]: + try: + import connectorx as cx # type: ignore + except ImportError: + # trunk-ignore(ruff/B904) + raise MissingDependencyException( + "Connector X table backend", ["connectorx"] + ) + + # default settings + backend_kwargs = { + "return_type": "arrow2", + "protocol": "binary", + **backend_kwargs, + } + conn = backend_kwargs.pop( + "conn", + self.engine.url._replace( + drivername=self.engine.url.get_backend_name() + ).render_as_string(hide_password=False), + ) + try: + query_str = str( + query.compile(self.engine, compile_kwargs={"literal_binds": True}) + ) + except CompileError as ex: + raise NotImplementedError( + f"Query for table {self.table.name} could not be compiled to string to execute it on ConnectorX. If you are on SQLAlchemy 1.4.x the causing exception is due to literals that cannot be rendered, upgrade to 2.x: {str(ex)}" + ) from ex + df = cx.read_sql(conn, query_str, **backend_kwargs) + yield df + + +def table_rows( + engine: Engine, + table: Table, + chunk_size: int, + backend: TableBackend, + incremental: Optional[dlt.sources.incremental[Any]] = None, + defer_table_reflect: bool = False, + table_adapter_callback: Callable[[Table], None] = None, + reflection_level: ReflectionLevel = "minimal", + backend_kwargs: Dict[str, Any] = None, + type_adapter_callback: Optional[TTypeAdapter] = None, +) -> Iterator[TDataItem]: + columns: TTableSchemaColumns = None + if defer_table_reflect: + table = Table( + table.name, table.metadata, autoload_with=engine, extend_existing=True + ) + if table_adapter_callback: + table_adapter_callback(table) + columns = table_to_columns(table, reflection_level, type_adapter_callback) + + # set the primary_key in the incremental + if incremental and incremental.primary_key is None: + primary_key = get_primary_key(table) + if primary_key is not None: + incremental.primary_key = primary_key + # yield empty record to set hints + yield dlt.mark.with_hints( + [], + dlt.mark.make_hints( + primary_key=get_primary_key(table), + columns=columns, + ), + ) + else: + # table was already reflected + columns = table_to_columns(table, reflection_level, type_adapter_callback) + + loader = TableLoader( + engine, backend, table, columns, incremental=incremental, chunk_size=chunk_size + ) + try: + yield from loader.load_rows(backend_kwargs) + finally: + # dispose the engine if created for this particular table + # NOTE: database wide engines are not disposed, not externally provided + if getattr(engine, "may_dispose_after_use", False): + engine.dispose() + + +def engine_from_credentials( + credentials: Union[ConnectionStringCredentials, Engine, str], + may_dispose_after_use: bool = False, + **backend_kwargs: Any, +) -> Engine: + if isinstance(credentials, Engine): + return credentials + if isinstance(credentials, ConnectionStringCredentials): + credentials = credentials.to_native_representation() + engine = create_engine(credentials, **backend_kwargs) + setattr(engine, "may_dispose_after_use", may_dispose_after_use) # noqa + return engine + + +def unwrap_json_connector_x(field: str) -> TDataItem: + """Creates a transform function to be added with `add_map` that will unwrap JSON columns + ingested via connectorx. Such columns are additionally quoted and translate SQL NULL to json "null" + """ + import pyarrow as pa + import pyarrow.compute as pc + + def _unwrap(table: TDataItem) -> TDataItem: + col_index = table.column_names.index(field) + # remove quotes + column = pc.replace_substring_regex(table[field], '"(.*)"', "\\1") + # convert json null to null + column = pc.replace_with_mask( + column, + pc.equal(column, "null").combine_chunks(), + pa.scalar(None, pa.large_string()), + ) + return table.set_column(col_index, table.schema.field(col_index), column) + + return _unwrap + + +def _detect_precision_hints_deprecated(value: Optional[bool]) -> None: + if value is None: + return + + msg = "`detect_precision_hints` argument is deprecated and will be removed in a future release. " + if value: + msg += "Use `reflection_level='full_with_precision'` which has the same effect instead." + + # trunk-ignore(ruff/B028) + warnings.warn( + msg, + DeprecationWarning, + ) + + +@configspec +class SqlDatabaseTableConfiguration(BaseConfiguration): + incremental: Optional[dlt.sources.incremental] = None # type: ignore[type-arg] + + +@configspec +class SqlTableResourceConfiguration(BaseConfiguration): + credentials: Union[ConnectionStringCredentials, Engine, str] = None + table: str = None + schema: Optional[str] = None + incremental: Optional[dlt.sources.incremental] = None # type: ignore[type-arg] + chunk_size: int = 50000 + backend: TableBackend = "sqlalchemy" + detect_precision_hints: Optional[bool] = None + defer_table_reflect: Optional[bool] = False + reflection_level: Optional[ReflectionLevel] = "full" diff --git a/src/teamster/libraries/dlt_sources/sql_database/schema_types.py b/src/teamster/libraries/dlt_sources/sql_database/schema_types.py new file mode 100644 index 0000000000..ecf540b232 --- /dev/null +++ b/src/teamster/libraries/dlt_sources/sql_database/schema_types.py @@ -0,0 +1,130 @@ +from typing import TYPE_CHECKING, Any, Callable, List, Literal, Optional, Type, Union + +from dlt.common import logger +from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns +from sqlalchemy import Column, Table +from sqlalchemy.engine import Row +from sqlalchemy.sql import Select, sqltypes +from sqlalchemy.sql.sqltypes import TypeEngine +from typing_extensions import TypeAlias + +ReflectionLevel = Literal["minimal", "full", "full_with_precision"] + + +# optionally create generics with any so they can be imported by dlt importer +if TYPE_CHECKING: + SelectAny: TypeAlias = Select[Any] + ColumnAny: TypeAlias = Column[Any] + RowAny: TypeAlias = Row[Any] + TypeEngineAny = TypeEngine[Any] +else: + SelectAny: TypeAlias = Type[Any] + ColumnAny: TypeAlias = Type[Any] + RowAny: TypeAlias = Type[Any] + TypeEngineAny = Type[Any] + + +TTypeAdapter = Callable[ + [TypeEngineAny], Optional[Union[TypeEngineAny, Type[TypeEngineAny]]] +] + + +def sqla_col_to_column_schema( + sql_col: ColumnAny, + reflection_level: ReflectionLevel, + type_adapter_callback: Optional[TTypeAdapter] = None, +) -> Optional[TColumnSchema]: + """Infer dlt schema column type from an sqlalchemy type. + + If `add_precision` is set, precision and scale is inferred from that types that support it, + such as numeric, varchar, int, bigint. Numeric (decimal) types have always precision added. + """ + col: TColumnSchema = { + "name": sql_col.name, + "nullable": sql_col.nullable, + } + if reflection_level == "minimal": + return col + + sql_t = sql_col.type + + if type_adapter_callback: + sql_t = type_adapter_callback(sql_t) # type: ignore[assignment] + # Check if sqla type class rather than instance is returned + if sql_t is not None and isinstance(sql_t, type): + sql_t = sql_t() + + if sql_t is None: + # Column ignored by callback + return col + + add_precision = reflection_level == "full_with_precision" + + if isinstance(sql_t, sqltypes.Numeric): + # check for Numeric type first and integer later, some numeric types (ie. Oracle) + # derive from both + # all Numeric types that are returned as floats will assume "double" type + # and returned as decimals will assume "decimal" type + if sql_t.asdecimal is False: + col["data_type"] = "double" + else: + col["data_type"] = "decimal" + if sql_t.precision is not None: + col["precision"] = sql_t.precision + # must have a precision for any meaningful scale + if sql_t.scale is not None: + col["scale"] = sql_t.scale + elif sql_t.decimal_return_scale is not None: + col["scale"] = sql_t.decimal_return_scale + elif isinstance(sql_t, sqltypes.SmallInteger): + col["data_type"] = "bigint" + if add_precision: + col["precision"] = 32 + elif isinstance(sql_t, sqltypes.Integer): + col["data_type"] = "bigint" + elif isinstance(sql_t, sqltypes.String): + col["data_type"] = "text" + if add_precision and sql_t.length: + col["precision"] = sql_t.length + elif isinstance(sql_t, sqltypes._Binary): + col["data_type"] = "binary" + if add_precision and sql_t.length: + col["precision"] = sql_t.length + elif isinstance(sql_t, sqltypes.DateTime): + col["data_type"] = "timestamp" + elif isinstance(sql_t, sqltypes.Date): + col["data_type"] = "date" + elif isinstance(sql_t, sqltypes.Time): + col["data_type"] = "time" + elif isinstance(sql_t, sqltypes.JSON): + col["data_type"] = "complex" + elif isinstance(sql_t, sqltypes.Boolean): + col["data_type"] = "bool" + else: + logger.warning( + f"A column with name {sql_col.name} contains unknown data type {sql_t} which cannot be mapped to `dlt` data type. When using sqlalchemy backend such data will be passed to the normalizer. In case of `pyarrow` and `pandas` backend, data types are detected from numpy ndarrays. In case of other backends, the behavior is backend-specific." + ) + + return {key: value for key, value in col.items() if value is not None} # type: ignore[return-value] + + +def get_primary_key(table: Table) -> Optional[List[str]]: + """Create primary key or return None if no key defined""" + primary_key = [c.name for c in table.primary_key] + return primary_key if len(primary_key) > 0 else None + + +def table_to_columns( + table: Table, + reflection_level: ReflectionLevel = "full", + type_conversion_fallback: Optional[TTypeAdapter] = None, +) -> TTableSchemaColumns: + """Convert an sqlalchemy table to a dlt table schema.""" + return { + col["name"]: col + for col in ( + sqla_col_to_column_schema(c, reflection_level, type_conversion_fallback) + for c in table.columns + ) + if col is not None + } From d7efba84118c10ce7483620c5c7ace6fd1f4ec5d Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:03:19 +0000 Subject: [PATCH 02/20] style: trunk --- .devcontainer/tpl/.env.tpl | 9 + .k8s/1password/items.yaml | 16 + .trunk/trunk.yaml | 9 +- pdm.lock | 504 ++++++++++++++---- pyproject.toml | 2 + requirements.txt | 243 ++++++--- .../code_locations/kipptaf/dagster-cloud.yaml | 90 ++++ .../code_locations/kipptaf/definitions.py | 3 + .../kipptaf/illuminate/__init__.py | 5 + .../kipptaf/illuminate/assets.py | 25 + .../code_locations/kipptaf/resources.py | 3 + 11 files changed, 726 insertions(+), 183 deletions(-) create mode 100644 src/teamster/code_locations/kipptaf/illuminate/__init__.py create mode 100644 src/teamster/code_locations/kipptaf/illuminate/assets.py diff --git a/.devcontainer/tpl/.env.tpl b/.devcontainer/tpl/.env.tpl index ba49d6c2da..46d04427fb 100644 --- a/.devcontainer/tpl/.env.tpl +++ b/.devcontainer/tpl/.env.tpl @@ -28,6 +28,9 @@ DEANSLIST_SFTP_HOST=op://Data Team/DeansList SFTP/host DEANSLIST_SFTP_PASSWORD=op://Data Team/DeansList SFTP/password DEANSLIST_SFTP_USERNAME=op://Data Team/DeansList SFTP/username DEANSLIST_SUBDOMAIN=op://Data Team/DeansList API/subdomain +DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL=op://Data Team/Teamster Service Account - dlt/username +DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY=op://Data Team/Teamster Service Account - dlt/password +DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID=op://Data Team/Teamster Service Account - dlt/project id EDPLAN_SFTP_HOST=op://Data Team/edplan SFTP - Camden/host EDPLAN_SFTP_PASSWORD_KIPPCAMDEN=op://Data Team/edplan SFTP - Camden/password EDPLAN_SFTP_PASSWORD_KIPPNEWARK=op://Data Team/edplan SFTP - Newark/password @@ -82,6 +85,12 @@ SCHOOLMINT_GROW_CLIENT_ID=op://Data Team/SchooMint Grow API/client id SCHOOLMINT_GROW_CLIENT_SECRET=op://Data Team/SchooMint Grow API/client secret SCHOOLMINT_GROW_DISTRICT_ID=op://Data Team/SchooMint Grow API/district id SMARTRECRUITERS_SMARTTOKEN=op://Data Team/SmartRecruiters API/smart token +SOURCES__SQL_DATABASE__CREDENTIALS__DATABASE=op://Data Team/Illuminate ODBC/database +SOURCES__SQL_DATABASE__CREDENTIALS__DRIVERNAME=op://Data Team/Illuminate ODBC/driver +SOURCES__SQL_DATABASE__CREDENTIALS__HOST=op://Data Team/Illuminate ODBC/ip +SOURCES__SQL_DATABASE__CREDENTIALS__PASSWORD=op://Data Team/Illuminate ODBC/password +SOURCES__SQL_DATABASE__CREDENTIALS__PORT=op://Data Team/Illuminate ODBC/port +SOURCES__SQL_DATABASE__CREDENTIALS__USERNAME=op://Data Team/Illuminate ODBC/username TABLEAU_PERSONAL_ACCESS_TOKEN=op://Data Team/Tableau Server PAT - Dagster/credential TABLEAU_SERVER_ADDRESS=op://Data Team/Tableau Server PAT - Dagster/hostname TABLEAU_SITE_ID=op://Data Team/Tableau Server PAT - Dagster/site id diff --git a/.k8s/1password/items.yaml b/.k8s/1password/items.yaml index 4010004f1d..dc8c3b004d 100644 --- a/.k8s/1password/items.yaml +++ b/.k8s/1password/items.yaml @@ -334,3 +334,19 @@ metadata: namespace: dagster-cloud spec: itemPath: vaults/Data Team/items/Overgrad API +--- +apiVersion: onepassword.com/v1 +kind: OnePasswordItem +metadata: + name: op-illuminate-db + namespace: dagster-cloud +spec: + itemPath: vaults/Data Team/items/Illuminate ODBC +--- +apiVersion: onepassword.com/v1 +kind: OnePasswordItem +metadata: + name: op-gcp-service-account-dlt + namespace: dagster-cloud +spec: + itemPath: vaults/Data Team/items/Teamster Service Account - dlt diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index a54cb03bba..235c6c2314 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -8,7 +8,6 @@ plugins: uri: https://github.com/trunk-io/plugins lint: enabled: - - pyright@1.1.372 - actionlint@1.7.1 - bandit@1.7.9 - git-diff-check @@ -19,6 +18,7 @@ lint: - osv-scanner@1.8.2 - oxipng@9.1.2 - prettier@3.3.3 + - pyright@1.1.372 - ruff@0.5.4 - shellcheck@0.10.0 - shfmt@3.6.0 @@ -41,6 +41,12 @@ lint: linters: - sqlfluff - sqlfmt + - paths: + - src/teamster/libraries/dlt_sources/** + linters: + - pyright + - sqlfluff + - sqlfmt - paths: - src/dbt/** linters: @@ -52,6 +58,7 @@ lint: - markdownlint - osv-scanner - oxipng + - pyright - ruff - shellcheck - shfmt diff --git a/pdm.lock b/pdm.lock index e5410033ca..60175ef96f 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:07cc097fd0facd0519ce5a4a40f0aa04eb1608e9b35f268c114a4ef538d5eff3" +content_hash = "sha256:2b4b1bb6960e6d64ea76fdbba4a3e8983a112653be52f8115f1e00d81866756a" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -99,6 +99,20 @@ files = [ {file = "argcomplete-3.4.0.tar.gz", hash = "sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f"}, ] +[[package]] +name = "astunparse" +version = "1.6.3" +summary = "An AST unparser for Python" +groups = ["default"] +dependencies = [ + "six<2.0,>=1.6.1", + "wheel<1.0,>=0.23.0", +] +files = [ + {file = "astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8"}, + {file = "astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872"}, +] + [[package]] name = "attrs" version = "23.2.0" @@ -153,34 +167,34 @@ files = [ [[package]] name = "bcrypt" -version = "4.1.3" +version = "4.2.0" requires_python = ">=3.7" summary = "Modern password hashing for your software and your servers" groups = ["default"] files = [ - {file = "bcrypt-4.1.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:48429c83292b57bf4af6ab75809f8f4daf52aa5d480632e53707805cc1ce9b74"}, - {file = "bcrypt-4.1.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a8bea4c152b91fd8319fef4c6a790da5c07840421c2b785084989bf8bbb7455"}, - {file = "bcrypt-4.1.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d3b317050a9a711a5c7214bf04e28333cf528e0ed0ec9a4e55ba628d0f07c1a"}, - {file = "bcrypt-4.1.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:094fd31e08c2b102a14880ee5b3d09913ecf334cd604af27e1013c76831f7b05"}, - {file = "bcrypt-4.1.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4fb253d65da30d9269e0a6f4b0de32bd657a0208a6f4e43d3e645774fb5457f3"}, - {file = "bcrypt-4.1.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:193bb49eeeb9c1e2db9ba65d09dc6384edd5608d9d672b4125e9320af9153a15"}, - {file = "bcrypt-4.1.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8cbb119267068c2581ae38790e0d1fbae65d0725247a930fc9900c285d95725d"}, - {file = "bcrypt-4.1.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6cac78a8d42f9d120b3987f82252bdbeb7e6e900a5e1ba37f6be6fe4e3848286"}, - {file = "bcrypt-4.1.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:01746eb2c4299dd0ae1670234bf77704f581dd72cc180f444bfe74eb80495b64"}, - {file = "bcrypt-4.1.3-cp37-abi3-win32.whl", hash = "sha256:037c5bf7c196a63dcce75545c8874610c600809d5d82c305dd327cd4969995bf"}, - {file = "bcrypt-4.1.3-cp37-abi3-win_amd64.whl", hash = "sha256:8a893d192dfb7c8e883c4576813bf18bb9d59e2cfd88b68b725990f033f1b978"}, - {file = "bcrypt-4.1.3-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0d4cf6ef1525f79255ef048b3489602868c47aea61f375377f0d00514fe4a78c"}, - {file = "bcrypt-4.1.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5698ce5292a4e4b9e5861f7e53b1d89242ad39d54c3da451a93cac17b61921a"}, - {file = "bcrypt-4.1.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec3c2e1ca3e5c4b9edb94290b356d082b721f3f50758bce7cce11d8a7c89ce84"}, - {file = "bcrypt-4.1.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3a5be252fef513363fe281bafc596c31b552cf81d04c5085bc5dac29670faa08"}, - {file = "bcrypt-4.1.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5f7cd3399fbc4ec290378b541b0cf3d4398e4737a65d0f938c7c0f9d5e686611"}, - {file = "bcrypt-4.1.3-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:c4c8d9b3e97209dd7111bf726e79f638ad9224b4691d1c7cfefa571a09b1b2d6"}, - {file = "bcrypt-4.1.3-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:31adb9cbb8737a581a843e13df22ffb7c84638342de3708a98d5c986770f2834"}, - {file = "bcrypt-4.1.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:551b320396e1d05e49cc18dd77d970accd52b322441628aca04801bbd1d52a73"}, - {file = "bcrypt-4.1.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6717543d2c110a155e6821ce5670c1f512f602eabb77dba95717ca76af79867d"}, - {file = "bcrypt-4.1.3-cp39-abi3-win32.whl", hash = "sha256:6004f5229b50f8493c49232b8e75726b568535fd300e5039e255d919fc3a07f2"}, - {file = "bcrypt-4.1.3-cp39-abi3-win_amd64.whl", hash = "sha256:2505b54afb074627111b5a8dc9b6ae69d0f01fea65c2fcaea403448c503d3991"}, - {file = "bcrypt-4.1.3.tar.gz", hash = "sha256:2ee15dd749f5952fe3f0430d0ff6b74082e159c50332a1413d51b5689cf06623"}, + {file = "bcrypt-4.2.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7"}, + {file = "bcrypt-4.2.0-cp37-abi3-win32.whl", hash = "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458"}, + {file = "bcrypt-4.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5"}, + {file = "bcrypt-4.2.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8"}, + {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"}, + {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"}, + {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"}, ] [[package]] @@ -309,7 +323,7 @@ files = [ [[package]] name = "cmake" -version = "3.30.0" +version = "3.30.1" requires_python = ">=3.7" summary = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" groups = ["default"] @@ -317,23 +331,23 @@ dependencies = [ "importlib-metadata>=1.4; python_version < \"3.8\"", ] files = [ - {file = "cmake-3.30.0-py3-none-macosx_10_10_x86_64.macosx_11_0_universal2.macosx_11_0_arm64.whl", hash = "sha256:9caf5839d041f3276596abf564267f7bbaf4b36731ad1f574f3d4c04d7f8c26b"}, - {file = "cmake-3.30.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2c19c50ee12fb1fddb636401b60f301e873b1f0bc726968509556450496c26fb"}, - {file = "cmake-3.30.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cc343a5fd4b3013e313083fd3226f4599210560e4d72743faa98057e9f41ccea"}, - {file = "cmake-3.30.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbe32916158e6ca2f45f6e1dc4578a99f5c9ab6cfc7e4f812fae284d54c4749d"}, - {file = "cmake-3.30.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a981336efd0d97a02bab4aba90f989077516a42c2510a1ba216f1a5cc00656f"}, - {file = "cmake-3.30.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59b8491d54064bf734e709001b1f79b1356a4c6c016f78445d5c0516785d096b"}, - {file = "cmake-3.30.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968e00571f6c07f36b2226a8dbd63eeba4888bcc2f9f30b1dbd2673f75b98564"}, - {file = "cmake-3.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e123afb34f08e38e76cd3303d1cea166f15ec7acd48353b6fe9d1175b10b4553"}, - {file = "cmake-3.30.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:d7c6265b3d066b25eaf07fc69b8672c28f531b59403cbabb864219f84098b378"}, - {file = "cmake-3.30.0-py3-none-musllinux_1_1_i686.whl", hash = "sha256:a6960b4b9e91bbcd68fc1a0395306a0eab68981752e667d4dc1721d9ad895358"}, - {file = "cmake-3.30.0-py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:100da4b77c2133a426ec6bffc01efcbdd9c212665c0b9acaa20bcaf98dc75097"}, - {file = "cmake-3.30.0-py3-none-musllinux_1_1_s390x.whl", hash = "sha256:e6e3ab9d48d5bf5564840e8152bcfe41a9318b1fe95b1410f8cc1f15800ff2bf"}, - {file = "cmake-3.30.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:bfb761c3dc275034d251494503e643dc8f23d15e8e6284eca1b2bfbde4634851"}, - {file = "cmake-3.30.0-py3-none-win32.whl", hash = "sha256:23253f76f44f0f69cf18c8343e56184ea3ab51e837198db691fbdef1bf986455"}, - {file = "cmake-3.30.0-py3-none-win_amd64.whl", hash = "sha256:aa9b483ff53804566909ec7ef8c25eaf4226c224756d731cb3dd28d9be2dea46"}, - {file = "cmake-3.30.0-py3-none-win_arm64.whl", hash = "sha256:fc9aba5cc8a631cbbe7a6b4b6b1f981346e70af35900459b4ac6a1b18f489568"}, - {file = "cmake-3.30.0.tar.gz", hash = "sha256:b6b9b584ce226dfde4d419578a2ae542e72409655c0ea2c989d5f9bb688cf024"}, + {file = "cmake-3.30.1-py3-none-macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:d56c1d96c4f8277bebbef768ad008a15d8b20b5946c87d888b85251d00b7509d"}, + {file = "cmake-3.30.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:15796c4ca5f32207d315a402604785e3288a9ca8bcf3a59427af31c21a09df50"}, + {file = "cmake-3.30.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c94ce1df31a0e9244e1ec00b1efc2c4df2cbb9450d640087bacb46dc99a90abd"}, + {file = "cmake-3.30.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8ec762c5364a4d33cbc395c435a0afbf706cc623f55d7c51166d6c48e745dfd"}, + {file = "cmake-3.30.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:065d68e35f6fa7973982f2d725ee8662b7e94cb5fd6856787608be7d62f64e30"}, + {file = "cmake-3.30.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:88c561e29af6a21fb4dc80f9438767af8ba5081d2c58cfc2a16298076d731539"}, + {file = "cmake-3.30.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77577bdc99c6597da9674d788f23421c0417c598b411d6b8ada64d0c70ff32a5"}, + {file = "cmake-3.30.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7743a2ba38edf56701ad3a40fac09ea3bba0538c6843fbc29cfccdbfc567873c"}, + {file = "cmake-3.30.1-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:b6cd7e8b854e4bd366632317ea3a8d7554fd5de8f5056ba13fab78576b31a2f8"}, + {file = "cmake-3.30.1-py3-none-musllinux_1_1_i686.whl", hash = "sha256:21c9db134fb859bbf163431f13c38c10bbcbc9a93287f6df61a305fe80c030b1"}, + {file = "cmake-3.30.1-py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:6d86335029ca716bad3c6fbcb83eb14acb0f70daa961cdf229a349057c7f1df4"}, + {file = "cmake-3.30.1-py3-none-musllinux_1_1_s390x.whl", hash = "sha256:212a6061ea724dfe89225005303f9f5ec804f46338338e9061381c22aca990ae"}, + {file = "cmake-3.30.1-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:3f6b8f12be57e8246f553ff1b081d2d02dc0b6194565e92ff08eb7159eceef24"}, + {file = "cmake-3.30.1-py3-none-win32.whl", hash = "sha256:8b15804f28dd3c22798c93e38be4d328e2aca00cc852a5afd72ca2332e28a021"}, + {file = "cmake-3.30.1-py3-none-win_amd64.whl", hash = "sha256:b512dfdbfe99d608aa22a152dac614fa00456b6adc2a24f4e586ab781b1c573a"}, + {file = "cmake-3.30.1-py3-none-win_arm64.whl", hash = "sha256:5b2556b2e999169121a7720f4e597848391d174dde05de9dfeec43c29565c97f"}, + {file = "cmake-3.30.1.tar.gz", hash = "sha256:51f01ce429a55acbfe1f1baf507f0fe6916243a9f3e5868a928d073ae4b18ef9"}, ] [[package]] @@ -387,7 +401,7 @@ files = [ [[package]] name = "croniter" -version = "2.0.7" +version = "3.0.0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.6" summary = "croniter provides iteration for datetime object with cron like format" groups = ["default", "dev"] @@ -396,13 +410,13 @@ dependencies = [ "pytz>2021.1", ] files = [ - {file = "croniter-2.0.7-py2.py3-none-any.whl", hash = "sha256:f15e80828d23920c4bb7f4d9340b932c9dcabecafc7775703c8b36d1253ed526"}, - {file = "croniter-2.0.7.tar.gz", hash = "sha256:1041b912b4b1e03751a0993531becf77851ae6e8b334c9c76ffeffb8f055f53f"}, + {file = "croniter-3.0.0-py2.py3-none-any.whl", hash = "sha256:92fcad6a4eb9749fa9cad2bba18916520b49408f6282a02aeac0dcda0586f7df"}, + {file = "croniter-3.0.0.tar.gz", hash = "sha256:745b22960bb21ead6d79b25c0e849fffd2fe65215a944def6b235d4c72688b98"}, ] [[package]] name = "cryptography" -version = "42.0.8" +version = "43.0.0" requires_python = ">=3.7" summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." groups = ["default"] @@ -410,30 +424,25 @@ dependencies = [ "cffi>=1.12; platform_python_implementation != \"PyPy\"", ] files = [ - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, - {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, - {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, - {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, - {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, - {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, - {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, + {file = "cryptography-43.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ea9e57f8ea880eeea38ab5abf9fbe39f923544d7884228ec67d666abd60f5a47"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9a8d6802e0825767476f62aafed40532bd435e8a5f7d23bd8b4f5fd04cc80ecf"}, + {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cc70b4b581f28d0a254d006f26949245e3657d40d8857066c2ae22a61222ef55"}, + {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a997df8c1c2aae1e1e5ac49c2e4f610ad037fc5a3aadc7b64e39dea42249431"}, + {file = "cryptography-43.0.0-cp37-abi3-win32.whl", hash = "sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc"}, + {file = "cryptography-43.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778"}, + {file = "cryptography-43.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb013933d4c127349b3948aa8aaf2f12c0353ad0eccd715ca789c8a0f671646f"}, + {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0"}, + {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2905ccf93a8a2a416f3ec01b1a7911c3fe4073ef35640e7ee5296754e30b762b"}, + {file = "cryptography-43.0.0-cp39-abi3-win32.whl", hash = "sha256:47ca71115e545954e6c1d207dd13461ab81f4eccfcb1345eac874828b5e3eaaf"}, + {file = "cryptography-43.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709"}, + {file = "cryptography-43.0.0.tar.gz", hash = "sha256:b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e"}, ] [[package]] @@ -568,6 +577,22 @@ files = [ {file = "dagster_dbt-0.23.14-py3-none-any.whl", hash = "sha256:ad287426de2786e0179d076ce193759c6e4e555b984aa380a5be9a20296f52f5"}, ] +[[package]] +name = "dagster-embedded-elt" +version = "0.23.14" +requires_python = "<3.13,>=3.8" +summary = "Package for performing ETL/ELT tasks with Dagster." +groups = ["default"] +dependencies = [ + "dagster==1.7.14", + "dlt>=0.4", + "sling>=1.1.5", +] +files = [ + {file = "dagster-embedded-elt-0.23.14.tar.gz", hash = "sha256:ce2d7e4182681682c68a04522834812a9972d2c5986b523c864bea898da6ffc8"}, + {file = "dagster_embedded_elt-0.23.14-py3-none-any.whl", hash = "sha256:df9a9acc623a85bd5a2d3808bb4f3dba98bd50c7858a248d32ec527414abd25f"}, +] + [[package]] name = "dagster-fivetran" version = "0.23.14" @@ -909,6 +934,44 @@ files = [ {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] +[[package]] +name = "dlt" +version = "0.5.1" +requires_python = "<3.13,>=3.8.1" +summary = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." +groups = ["default"] +dependencies = [ + "PyYAML>=5.4.1", + "astunparse>=1.6.3", + "click>=7.1", + "fsspec>=2022.4.0", + "gitpython>=3.1.29", + "giturlparse>=0.10.0", + "hexbytes>=0.2.2", + "humanize>=4.4.0", + "jsonpath-ng>=1.5.3", + "makefun>=1.15.0", + "orjson!=3.10.1,!=3.9.11,!=3.9.12,!=3.9.13,!=3.9.14,<4,>=3.6.7; platform_python_implementation != \"PyPy\"", + "packaging>=21.1", + "pathvalidate>=2.5.2", + "pendulum>=2.1.2", + "pytz>=2022.6", + "requests>=2.26.0", + "requirements-parser>=0.5.0", + "semver>=2.13.0", + "setuptools>=65.6.0", + "simplejson>=3.17.5", + "tenacity>=8.0.2", + "tomlkit>=0.11.3", + "typing-extensions>=4.0.0", + "tzdata>=2022.1", + "win-precise-time>=1.4.2; os_name == \"nt\"", +] +files = [ + {file = "dlt-0.5.1-py3-none-any.whl", hash = "sha256:19d7920816fadd049a1a92c4ecc4e740bf1cd5f6484fce535715f8c214a835ce"}, + {file = "dlt-0.5.1.tar.gz", hash = "sha256:dfa11e498feec3aca0022541850b6fa7dd3fddce4c2e6fef195530638269fec4"}, +] + [[package]] name = "dnspython" version = "2.6.1" @@ -979,7 +1042,6 @@ version = "2024.6.1" requires_python = ">=3.8" summary = "File-system specification" groups = ["default", "dev"] -marker = "python_version >= \"3.12\"" files = [ {file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"}, {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"}, @@ -995,6 +1057,20 @@ files = [ {file = "genson-1.3.0.tar.gz", hash = "sha256:e02db9ac2e3fd29e65b5286f7135762e2cd8a986537c075b06fc5f1517308e37"}, ] +[[package]] +name = "gitdb" +version = "4.0.11" +requires_python = ">=3.7" +summary = "Git Object Database" +groups = ["default"] +dependencies = [ + "smmap<6,>=3.0.1", +] +files = [ + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, +] + [[package]] name = "github3-py" version = "4.0.1" @@ -1012,6 +1088,32 @@ files = [ {file = "github3.py-4.0.1.tar.gz", hash = "sha256:30d571076753efc389edc7f9aaef338a4fcb24b54d8968d5f39b1342f45ddd36"}, ] +[[package]] +name = "gitpython" +version = "3.1.43" +requires_python = ">=3.7" +summary = "GitPython is a Python library used to interact with Git repositories" +groups = ["default"] +dependencies = [ + "gitdb<5,>=4.0.1", + "typing-extensions>=3.7.4.3; python_version < \"3.8\"", +] +files = [ + {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, + {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, +] + +[[package]] +name = "giturlparse" +version = "0.12.0" +requires_python = ">=3.8" +summary = "A Git URL parsing module (supports parsing and rewriting)" +groups = ["default"] +files = [ + {file = "giturlparse-0.12.0-py2.py3-none-any.whl", hash = "sha256:412b74f2855f1da2fefa89fd8dde62df48476077a72fc19b62039554d27360eb"}, + {file = "giturlparse-0.12.0.tar.gz", hash = "sha256:c0fff7c21acc435491b1779566e038757a205c1ffdcb47e4f81ea52ad8c3859a"}, +] + [[package]] name = "google-api-core" version = "2.19.1" @@ -1187,7 +1289,7 @@ files = [ [[package]] name = "google-cloud-storage" -version = "2.17.0" +version = "2.18.0" requires_python = ">=3.7" summary = "Google Cloud Storage API client library" groups = ["default"] @@ -1200,8 +1302,8 @@ dependencies = [ "requests<3.0.0dev,>=2.18.0", ] files = [ - {file = "google-cloud-storage-2.17.0.tar.gz", hash = "sha256:49378abff54ef656b52dca5ef0f2eba9aa83dc2b2c72c78714b03a1a95fe9388"}, - {file = "google_cloud_storage-2.17.0-py2.py3-none-any.whl", hash = "sha256:5b393bc766b7a3bc6f5407b9e665b2450d36282614b7945e570b3480a456d1e1"}, + {file = "google_cloud_storage-2.18.0-py2.py3-none-any.whl", hash = "sha256:e8e1a9577952143c3fca8163005ecfadd2d70ec080fa158a8b305000e2c22fbb"}, + {file = "google_cloud_storage-2.18.0.tar.gz", hash = "sha256:0aa3f7c57f3632f81b455d91558d2b27ada96eee2de3aaa17f689db1470d9578"}, ] [[package]] @@ -1340,7 +1442,7 @@ version = "3.0.3" requires_python = ">=3.7" summary = "Lightweight in-process concurrent programming" groups = ["default", "dev"] -marker = "(platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"" +marker = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"" files = [ {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, @@ -1449,6 +1551,17 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] +[[package]] +name = "hexbytes" +version = "1.2.1" +requires_python = "<4,>=3.8" +summary = "hexbytes: Python `bytes` subclass that decodes hex, with a readable console output" +groups = ["default"] +files = [ + {file = "hexbytes-1.2.1-py3-none-any.whl", hash = "sha256:e64890b203a31f4a23ef11470ecfcca565beaee9198df623047df322b757471a"}, + {file = "hexbytes-1.2.1.tar.gz", hash = "sha256:515f00dddf31053db4d0d7636dd16061c1d896c3109b8e751005db4ca46bcca7"}, +] + [[package]] name = "httplib2" version = "0.22.0" @@ -1497,6 +1610,17 @@ files = [ {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, ] +[[package]] +name = "humanize" +version = "4.10.0" +requires_python = ">=3.8" +summary = "Python humanize utilities" +groups = ["default"] +files = [ + {file = "humanize-4.10.0-py3-none-any.whl", hash = "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6"}, + {file = "humanize-4.10.0.tar.gz", hash = "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978"}, +] + [[package]] name = "idna" version = "3.7" @@ -1594,6 +1718,19 @@ files = [ {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, ] +[[package]] +name = "jsonpath-ng" +version = "1.6.1" +summary = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." +groups = ["default"] +dependencies = [ + "ply", +] +files = [ + {file = "jsonpath-ng-1.6.1.tar.gz", hash = "sha256:086c37ba4917304850bd837aeab806670224d3f038fe2833ff593a672ef0a5fa"}, + {file = "jsonpath_ng-1.6.1-py3-none-any.whl", hash = "sha256:8f22cd8273d7772eea9aaa84d922e0841aa36fdb8a2c6b7f6c3791a16a9bc0be"}, +] + [[package]] name = "jsonschema" version = "4.23.0" @@ -1683,6 +1820,19 @@ files = [ {file = "Logbook-1.5.3.tar.gz", hash = "sha256:66f454ada0f56eae43066f604a222b09893f98c1adc18df169710761b8f32fe8"}, ] +[[package]] +name = "makefun" +version = "1.15.4" +summary = "Small library to dynamically create python functions." +groups = ["default"] +dependencies = [ + "funcsigs; python_version < \"3.3\"", +] +files = [ + {file = "makefun-1.15.4-py2.py3-none-any.whl", hash = "sha256:945d078a7e01a903f2cbef738b33e0ebc52b8d35fb7e20c528ed87b5c80db5b7"}, + {file = "makefun-1.15.4.tar.gz", hash = "sha256:9f9b9904e7c397759374a88f4c57781fbab2a458dec78df4b3ee6272cd9fb010"}, +] + [[package]] name = "mako" version = "1.3.5" @@ -2043,13 +2193,23 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] +[[package]] +name = "pathvalidate" +version = "3.2.0" +requires_python = ">=3.7" +summary = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." +groups = ["default"] +files = [ + {file = "pathvalidate-3.2.0-py3-none-any.whl", hash = "sha256:cc593caa6299b22b37f228148257997e2fa850eea2daf7e4cc9205cef6908dee"}, + {file = "pathvalidate-3.2.0.tar.gz", hash = "sha256:5e8378cf6712bff67fbe7a8307d99fa8c1a0cb28aa477056f8fc374f0dff24ad"}, +] + [[package]] name = "pendulum" version = "3.0.0" requires_python = ">=3.8" summary = "Python datetimes made easy" groups = ["default", "dev"] -marker = "python_version >= \"3.12\"" dependencies = [ "backports-zoneinfo>=0.2.1; python_version < \"3.9\"", "importlib-resources>=5.9.0; python_version < \"3.9\"", @@ -2104,6 +2264,16 @@ files = [ {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] +[[package]] +name = "ply" +version = "3.11" +summary = "Python Lex & Yacc" +groups = ["default"] +files = [ + {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, + {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, +] + [[package]] name = "prompt-toolkit" version = "3.0.36" @@ -2166,6 +2336,18 @@ files = [ {file = "psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"}, ] +[[package]] +name = "psycopg2" +version = "2.9.9" +requires_python = ">=3.7" +summary = "psycopg2 - Python-PostgreSQL Database Adapter" +groups = ["default"] +files = [ + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, + {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, +] + [[package]] name = "py-avro-schema" version = "3.7.1" @@ -2308,7 +2490,6 @@ extras = ["email"] requires_python = ">=3.8" summary = "Data validation using Python type hints" groups = ["dev"] -marker = "python_version ~= \"3.11\"" dependencies = [ "email-validator>=2.0.0", "pydantic==2.8.2", @@ -2387,7 +2568,6 @@ version = "3.1.2" requires_python = ">=3.6.8" summary = "pyparsing module - Classes and methods to define and execute parsing grammars" groups = ["default"] -marker = "python_version > \"3.0\"" files = [ {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, @@ -2398,7 +2578,7 @@ name = "pyreadline3" version = "3.4.1" summary = "A python implementation of GNU readline." groups = ["default", "dev"] -marker = "sys_platform == \"win32\" and python_version >= \"3.8\"" +marker = "sys_platform == \"win32\"" files = [ {file = "pyreadline3-3.4.1-py3-none-any.whl", hash = "sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb"}, {file = "pyreadline3-3.4.1.tar.gz", hash = "sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae"}, @@ -2406,7 +2586,7 @@ files = [ [[package]] name = "pytest" -version = "8.2.2" +version = "8.3.1" requires_python = ">=3.8" summary = "pytest: simple powerful testing with Python" groups = ["dev"] @@ -2415,12 +2595,12 @@ dependencies = [ "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", "iniconfig", "packaging", - "pluggy<2.0,>=1.5", + "pluggy<2,>=1.5", "tomli>=1; python_version < \"3.11\"", ] files = [ - {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, - {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, + {file = "pytest-8.3.1-py3-none-any.whl", hash = "sha256:e9600ccf4f563976e2c99fa02c7624ab938296551f280835ee6516df8bc4ae8c"}, + {file = "pytest-8.3.1.tar.gz", hash = "sha256:7e8e5c5abd6e93cb1cc151f23e57adc31fcf8cfd2a3ff2da63e23f732de35db6"}, ] [[package]] @@ -2586,6 +2766,20 @@ files = [ {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, ] +[[package]] +name = "requirements-parser" +version = "0.10.1" +requires_python = "<4.0,>=3.8" +summary = "This is a small Python module for parsing Pip requirement files." +groups = ["default"] +dependencies = [ + "types-setuptools>=69.1.0", +] +files = [ + {file = "requirements_parser-0.10.1-py3-none-any.whl", hash = "sha256:bb6b6e84d54ed4fee5153d0f2889615f791d8eba4521961a075f6d3f1fb2974d"}, + {file = "requirements_parser-0.10.1.tar.gz", hash = "sha256:cf208b8cb23d9b05b35ab0238db416a4b751441a41744c9e8b619362ee534d17"}, +] + [[package]] name = "rich" version = "13.7.1" @@ -2681,15 +2875,26 @@ files = [ {file = "scipy-1.14.0.tar.gz", hash = "sha256:b5923f48cb840380f9854339176ef21763118a7300a88203ccd0bdd26e58527b"}, ] +[[package]] +name = "semver" +version = "3.0.2" +requires_python = ">=3.7" +summary = "Python helper for Semantic Versioning (https://semver.org)" +groups = ["default"] +files = [ + {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"}, + {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"}, +] + [[package]] name = "setuptools" -version = "71.0.3" +version = "71.1.0" requires_python = ">=3.8" summary = "Easily download, build, install, upgrade, and uninstall Python packages" groups = ["default", "dev"] files = [ - {file = "setuptools-71.0.3-py3-none-any.whl", hash = "sha256:f501b6e6db709818dc76882582d9c516bf3b67b948864c5fa1d1624c09a49207"}, - {file = "setuptools-71.0.3.tar.gz", hash = "sha256:3d8531791a27056f4a38cd3e54084d8b1c4228ff9cf3f2d7dd075ec99f9fd70d"}, + {file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"}, + {file = "setuptools-71.1.0.tar.gz", hash = "sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936"}, ] [[package]] @@ -2703,6 +2908,30 @@ files = [ {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] +[[package]] +name = "simplejson" +version = "3.19.2" +requires_python = ">=2.5, !=3.0.*, !=3.1.*, !=3.2.*" +summary = "Simple, fast, extensible JSON encoder/decoder for Python" +groups = ["default"] +files = [ + {file = "simplejson-3.19.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b8d940fd28eb34a7084877747a60873956893e377f15a32ad445fe66c972c3b8"}, + {file = "simplejson-3.19.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4969d974d9db826a2c07671273e6b27bc48e940738d768fa8f33b577f0978378"}, + {file = "simplejson-3.19.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c594642d6b13d225e10df5c16ee15b3398e21a35ecd6aee824f107a625690374"}, + {file = "simplejson-3.19.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2f5a398b5e77bb01b23d92872255e1bcb3c0c719a3be40b8df146570fe7781a"}, + {file = "simplejson-3.19.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:176a1b524a3bd3314ed47029a86d02d5a95cc0bee15bd3063a1e1ec62b947de6"}, + {file = "simplejson-3.19.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3c7363a8cb8c5238878ec96c5eb0fc5ca2cb11fc0c7d2379863d342c6ee367a"}, + {file = "simplejson-3.19.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:346820ae96aa90c7d52653539a57766f10f33dd4be609206c001432b59ddf89f"}, + {file = "simplejson-3.19.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de9a2792612ec6def556d1dc621fd6b2073aff015d64fba9f3e53349ad292734"}, + {file = "simplejson-3.19.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1c768e7584c45094dca4b334af361e43b0aaa4844c04945ac7d43379eeda9bc2"}, + {file = "simplejson-3.19.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:9652e59c022e62a5b58a6f9948b104e5bb96d3b06940c6482588176f40f4914b"}, + {file = "simplejson-3.19.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9c1a4393242e321e344213a90a1e3bf35d2f624aa8b8f6174d43e3c6b0e8f6eb"}, + {file = "simplejson-3.19.2-cp312-cp312-win32.whl", hash = "sha256:7cb98be113911cb0ad09e5523d0e2a926c09a465c9abb0784c9269efe4f95917"}, + {file = "simplejson-3.19.2-cp312-cp312-win_amd64.whl", hash = "sha256:6779105d2fcb7fcf794a6a2a233787f6bbd4731227333a072d8513b252ed374f"}, + {file = "simplejson-3.19.2-py3-none-any.whl", hash = "sha256:bcedf4cae0d47839fee7de344f96b5694ca53c786f28b5f773d4f0b265a159eb"}, + {file = "simplejson-3.19.2.tar.gz", hash = "sha256:9eb442a2442ce417801c912df68e1f6ccfcd41577ae7274953ab3ad24ef7d82c"}, +] + [[package]] name = "six" version = "1.16.0" @@ -2714,6 +2943,38 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "sling" +version = "1.2.13" +summary = "Slings data from a source to a target" +groups = ["default"] +dependencies = [ + "sling-linux-amd64==1.2.13", +] +files = [ + {file = "sling-1.2.13.tar.gz", hash = "sha256:40c77d0cf4ade41618b33c46704fbe85f6ebec0c1119decfee43125591d8bce0"}, +] + +[[package]] +name = "sling-linux-amd64" +version = "1.2.13" +summary = "Sling Binary for Linux (AMD64)" +groups = ["default"] +files = [ + {file = "sling-linux-amd64-1.2.13.tar.gz", hash = "sha256:ee1b674091d50ba3014b05944401df34248cbf28ba6a8074bc1e2fbfee5a8b1d"}, +] + +[[package]] +name = "smmap" +version = "5.0.1" +requires_python = ">=3.7" +summary = "A pure Python implementation of a sliding window memory map manager" +groups = ["default"] +files = [ + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -2833,7 +3094,7 @@ files = [ [[package]] name = "starlette" -version = "0.37.2" +version = "0.38.1" requires_python = ">=3.8" summary = "The little ASGI library that shines." groups = ["dev"] @@ -2842,8 +3103,8 @@ dependencies = [ "typing-extensions>=3.10.0; python_version < \"3.10\"", ] files = [ - {file = "starlette-0.37.2-py3-none-any.whl", hash = "sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee"}, - {file = "starlette-0.37.2.tar.gz", hash = "sha256:9af890290133b79fc3db55474ade20f6220a364a0402e0b556e7cd5e1e093823"}, + {file = "starlette-0.38.1-py3-none-any.whl", hash = "sha256:42688a287165bd6acc53068457b5cbf28a3204110b704930411728a633f0f7b8"}, + {file = "starlette-0.38.1.tar.gz", hash = "sha256:890e89d9b1b367bad6fd11a6e191f40d03504df1ff55fd15674a3b35a122990e"}, ] [[package]] @@ -2924,7 +3185,7 @@ version = "2.14.2" requires_python = ">=3.8" summary = "Travel through time in your tests." groups = ["default", "dev"] -marker = "implementation_name != \"pypy\" and python_version >= \"3.12\"" +marker = "implementation_name != \"pypy\"" dependencies = [ "python-dateutil", ] @@ -2954,6 +3215,17 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tomlkit" +version = "0.13.0" +requires_python = ">=3.8" +summary = "Style preserving TOML library" +groups = ["default"] +files = [ + {file = "tomlkit-0.13.0-py3-none-any.whl", hash = "sha256:7075d3042d03b80f603482d69bf0c8f345c2b30e41699fd8883227f89972b264"}, + {file = "tomlkit-0.13.0.tar.gz", hash = "sha256:08ad192699734149f5b97b45f1f18dad7eb1b6d16bc72ad0c2335772650d7b72"}, +] + [[package]] name = "toposort" version = "1.10" @@ -3010,6 +3282,17 @@ files = [ {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, ] +[[package]] +name = "types-setuptools" +version = "71.1.0.20240723" +requires_python = ">=3.8" +summary = "Typing stubs for setuptools" +groups = ["default"] +files = [ + {file = "types-setuptools-71.1.0.20240723.tar.gz", hash = "sha256:8a9349038c7e22d88e6c5d9c6705b347b22930424114a452c1712899e85131ff"}, + {file = "types_setuptools-71.1.0.20240723-py3-none-any.whl", hash = "sha256:ac9fc263f59d1e02bca49cb7270a12c47ab80b3b911fb4d92f1fecf978bfe88a"}, +] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -3038,7 +3321,6 @@ version = "0.2.2" requires_python = ">=3.8" summary = "pathlib api extended to use fsspec backends" groups = ["default", "dev"] -marker = "python_version >= \"3.12\"" dependencies = [ "fsspec>=2022.1.0", ] @@ -3071,7 +3353,7 @@ files = [ [[package]] name = "uvicorn" -version = "0.30.1" +version = "0.30.3" requires_python = ">=3.8" summary = "The lightning-fast ASGI server." groups = ["dev"] @@ -3081,13 +3363,13 @@ dependencies = [ "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ - {file = "uvicorn-0.30.1-py3-none-any.whl", hash = "sha256:cd17daa7f3b9d7a24de3617820e634d0933b69eed8e33a516071174427238c81"}, - {file = "uvicorn-0.30.1.tar.gz", hash = "sha256:d46cd8e0fd80240baffbcd9ec1012a712938754afcf81bce56c024c1656aece8"}, + {file = "uvicorn-0.30.3-py3-none-any.whl", hash = "sha256:94a3608da0e530cea8f69683aa4126364ac18e3826b6630d1a65f4638aade503"}, + {file = "uvicorn-0.30.3.tar.gz", hash = "sha256:0d114d0831ff1adbf231d358cbf42f17333413042552a624ea6a9b4c33dcfd81"}, ] [[package]] name = "uvicorn" -version = "0.30.1" +version = "0.30.3" extras = ["standard"] requires_python = ">=3.8" summary = "The lightning-fast ASGI server." @@ -3097,14 +3379,14 @@ dependencies = [ "httptools>=0.5.0", "python-dotenv>=0.13", "pyyaml>=5.1", - "uvicorn==0.30.1", + "uvicorn==0.30.3", "uvloop!=0.15.0,!=0.15.1,>=0.14.0; (sys_platform != \"cygwin\" and sys_platform != \"win32\") and platform_python_implementation != \"PyPy\"", "watchfiles>=0.13", "websockets>=10.4", ] files = [ - {file = "uvicorn-0.30.1-py3-none-any.whl", hash = "sha256:cd17daa7f3b9d7a24de3617820e634d0933b69eed8e33a516071174427238c81"}, - {file = "uvicorn-0.30.1.tar.gz", hash = "sha256:d46cd8e0fd80240baffbcd9ec1012a712938754afcf81bce56c024c1656aece8"}, + {file = "uvicorn-0.30.3-py3-none-any.whl", hash = "sha256:94a3608da0e530cea8f69683aa4126364ac18e3826b6630d1a65f4638aade503"}, + {file = "uvicorn-0.30.3.tar.gz", hash = "sha256:0d114d0831ff1adbf231d358cbf42f17333413042552a624ea6a9b4c33dcfd81"}, ] [[package]] @@ -3219,6 +3501,30 @@ files = [ {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, ] +[[package]] +name = "wheel" +version = "0.43.0" +requires_python = ">=3.8" +summary = "A built-package format for Python" +groups = ["default"] +files = [ + {file = "wheel-0.43.0-py3-none-any.whl", hash = "sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81"}, + {file = "wheel-0.43.0.tar.gz", hash = "sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85"}, +] + +[[package]] +name = "win-precise-time" +version = "1.4.2" +requires_python = ">=3.7" +summary = "" +groups = ["default"] +marker = "os_name == \"nt\"" +files = [ + {file = "win-precise-time-1.4.2.tar.gz", hash = "sha256:89274785cbc5f2997e01675206da3203835a442c60fd97798415c6b3c179c0b9"}, + {file = "win_precise_time-1.4.2-cp312-cp312-win32.whl", hash = "sha256:0210dcea88a520c91de1708ae4c881e3c0ddc956daa08b9eabf2b7c35f3109f5"}, + {file = "win_precise_time-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:85670f77cc8accd8f1e6d05073999f77561c23012a9ee988cbd44bb7ce655062"}, +] + [[package]] name = "yarl" version = "1.9.4" diff --git a/pyproject.toml b/pyproject.toml index 9b04aee2f3..7ab207bc61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ dependencies = [ "scikit-learn>=1.4.0", "tableauserverclient>=0.25", "tenacity>=8.2.3", + "dagster-embedded-elt>=0.23.14", + "psycopg2>=2.9.9", ] requires-python = ">=3.12,<3.13" license = { text = "GPL-3.0-or-later" } diff --git a/requirements.txt b/requirements.txt index a80eb69203..54f39f0437 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,9 @@ alembic==1.13.2 \ annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 +astunparse==1.6.3 \ + --hash=sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872 \ + --hash=sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8 attrs==23.2.0 \ --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 @@ -18,30 +21,30 @@ avro==1.11.3 \ babel==2.15.0 \ --hash=sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb \ --hash=sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413 -bcrypt==4.1.3 \ - --hash=sha256:01746eb2c4299dd0ae1670234bf77704f581dd72cc180f444bfe74eb80495b64 \ - --hash=sha256:037c5bf7c196a63dcce75545c8874610c600809d5d82c305dd327cd4969995bf \ - --hash=sha256:094fd31e08c2b102a14880ee5b3d09913ecf334cd604af27e1013c76831f7b05 \ - --hash=sha256:0d4cf6ef1525f79255ef048b3489602868c47aea61f375377f0d00514fe4a78c \ - --hash=sha256:193bb49eeeb9c1e2db9ba65d09dc6384edd5608d9d672b4125e9320af9153a15 \ - --hash=sha256:2505b54afb074627111b5a8dc9b6ae69d0f01fea65c2fcaea403448c503d3991 \ - --hash=sha256:2ee15dd749f5952fe3f0430d0ff6b74082e159c50332a1413d51b5689cf06623 \ - --hash=sha256:31adb9cbb8737a581a843e13df22ffb7c84638342de3708a98d5c986770f2834 \ - --hash=sha256:3a5be252fef513363fe281bafc596c31b552cf81d04c5085bc5dac29670faa08 \ - --hash=sha256:3d3b317050a9a711a5c7214bf04e28333cf528e0ed0ec9a4e55ba628d0f07c1a \ - --hash=sha256:48429c83292b57bf4af6ab75809f8f4daf52aa5d480632e53707805cc1ce9b74 \ - --hash=sha256:4a8bea4c152b91fd8319fef4c6a790da5c07840421c2b785084989bf8bbb7455 \ - --hash=sha256:4fb253d65da30d9269e0a6f4b0de32bd657a0208a6f4e43d3e645774fb5457f3 \ - --hash=sha256:551b320396e1d05e49cc18dd77d970accd52b322441628aca04801bbd1d52a73 \ - --hash=sha256:5f7cd3399fbc4ec290378b541b0cf3d4398e4737a65d0f938c7c0f9d5e686611 \ - --hash=sha256:6004f5229b50f8493c49232b8e75726b568535fd300e5039e255d919fc3a07f2 \ - --hash=sha256:6717543d2c110a155e6821ce5670c1f512f602eabb77dba95717ca76af79867d \ - --hash=sha256:6cac78a8d42f9d120b3987f82252bdbeb7e6e900a5e1ba37f6be6fe4e3848286 \ - --hash=sha256:8a893d192dfb7c8e883c4576813bf18bb9d59e2cfd88b68b725990f033f1b978 \ - --hash=sha256:8cbb119267068c2581ae38790e0d1fbae65d0725247a930fc9900c285d95725d \ - --hash=sha256:c4c8d9b3e97209dd7111bf726e79f638ad9224b4691d1c7cfefa571a09b1b2d6 \ - --hash=sha256:ec3c2e1ca3e5c4b9edb94290b356d082b721f3f50758bce7cce11d8a7c89ce84 \ - --hash=sha256:f5698ce5292a4e4b9e5861f7e53b1d89242ad39d54c3da451a93cac17b61921a +bcrypt==4.2.0 \ + --hash=sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb \ + --hash=sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399 \ + --hash=sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291 \ + --hash=sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d \ + --hash=sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7 \ + --hash=sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d \ + --hash=sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe \ + --hash=sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060 \ + --hash=sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68 \ + --hash=sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c \ + --hash=sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458 \ + --hash=sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9 \ + --hash=sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328 \ + --hash=sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7 \ + --hash=sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34 \ + --hash=sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e \ + --hash=sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2 \ + --hash=sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5 \ + --hash=sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae \ + --hash=sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00 \ + --hash=sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841 \ + --hash=sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8 \ + --hash=sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221 beautifulsoup4==4.12.3 \ --hash=sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051 \ --hash=sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed @@ -84,24 +87,24 @@ charset-normalizer==3.3.2 \ click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de -cmake==3.30.0 \ - --hash=sha256:100da4b77c2133a426ec6bffc01efcbdd9c212665c0b9acaa20bcaf98dc75097 \ - --hash=sha256:23253f76f44f0f69cf18c8343e56184ea3ab51e837198db691fbdef1bf986455 \ - --hash=sha256:2c19c50ee12fb1fddb636401b60f301e873b1f0bc726968509556450496c26fb \ - --hash=sha256:4a981336efd0d97a02bab4aba90f989077516a42c2510a1ba216f1a5cc00656f \ - --hash=sha256:59b8491d54064bf734e709001b1f79b1356a4c6c016f78445d5c0516785d096b \ - --hash=sha256:968e00571f6c07f36b2226a8dbd63eeba4888bcc2f9f30b1dbd2673f75b98564 \ - --hash=sha256:9caf5839d041f3276596abf564267f7bbaf4b36731ad1f574f3d4c04d7f8c26b \ - --hash=sha256:a6960b4b9e91bbcd68fc1a0395306a0eab68981752e667d4dc1721d9ad895358 \ - --hash=sha256:aa9b483ff53804566909ec7ef8c25eaf4226c224756d731cb3dd28d9be2dea46 \ - --hash=sha256:b6b9b584ce226dfde4d419578a2ae542e72409655c0ea2c989d5f9bb688cf024 \ - --hash=sha256:bfb761c3dc275034d251494503e643dc8f23d15e8e6284eca1b2bfbde4634851 \ - --hash=sha256:cbe32916158e6ca2f45f6e1dc4578a99f5c9ab6cfc7e4f812fae284d54c4749d \ - --hash=sha256:cc343a5fd4b3013e313083fd3226f4599210560e4d72743faa98057e9f41ccea \ - --hash=sha256:d7c6265b3d066b25eaf07fc69b8672c28f531b59403cbabb864219f84098b378 \ - --hash=sha256:e123afb34f08e38e76cd3303d1cea166f15ec7acd48353b6fe9d1175b10b4553 \ - --hash=sha256:e6e3ab9d48d5bf5564840e8152bcfe41a9318b1fe95b1410f8cc1f15800ff2bf \ - --hash=sha256:fc9aba5cc8a631cbbe7a6b4b6b1f981346e70af35900459b4ac6a1b18f489568 +cmake==3.30.1 \ + --hash=sha256:065d68e35f6fa7973982f2d725ee8662b7e94cb5fd6856787608be7d62f64e30 \ + --hash=sha256:15796c4ca5f32207d315a402604785e3288a9ca8bcf3a59427af31c21a09df50 \ + --hash=sha256:212a6061ea724dfe89225005303f9f5ec804f46338338e9061381c22aca990ae \ + --hash=sha256:21c9db134fb859bbf163431f13c38c10bbcbc9a93287f6df61a305fe80c030b1 \ + --hash=sha256:3f6b8f12be57e8246f553ff1b081d2d02dc0b6194565e92ff08eb7159eceef24 \ + --hash=sha256:51f01ce429a55acbfe1f1baf507f0fe6916243a9f3e5868a928d073ae4b18ef9 \ + --hash=sha256:5b2556b2e999169121a7720f4e597848391d174dde05de9dfeec43c29565c97f \ + --hash=sha256:6d86335029ca716bad3c6fbcb83eb14acb0f70daa961cdf229a349057c7f1df4 \ + --hash=sha256:7743a2ba38edf56701ad3a40fac09ea3bba0538c6843fbc29cfccdbfc567873c \ + --hash=sha256:77577bdc99c6597da9674d788f23421c0417c598b411d6b8ada64d0c70ff32a5 \ + --hash=sha256:88c561e29af6a21fb4dc80f9438767af8ba5081d2c58cfc2a16298076d731539 \ + --hash=sha256:8b15804f28dd3c22798c93e38be4d328e2aca00cc852a5afd72ca2332e28a021 \ + --hash=sha256:b512dfdbfe99d608aa22a152dac614fa00456b6adc2a24f4e586ab781b1c573a \ + --hash=sha256:b6cd7e8b854e4bd366632317ea3a8d7554fd5de8f5056ba13fab78576b31a2f8 \ + --hash=sha256:c94ce1df31a0e9244e1ec00b1efc2c4df2cbb9450d640087bacb46dc99a90abd \ + --hash=sha256:d56c1d96c4f8277bebbef768ad008a15d8b20b5946c87d888b85251d00b7509d \ + --hash=sha256:e8ec762c5364a4d33cbc395c435a0afbf706cc623f55d7c51166d6c48e745dfd colorama==0.4.6 \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 @@ -124,34 +127,29 @@ cramjam==2.8.3 \ --hash=sha256:c14728e3360cd212d5b606ca703c3bd1c8912efcdbc1aa032c81c2882509ebd5 \ --hash=sha256:d93b42d22bf3e17290c5e4cf58e715a419330bb5255c35933c14db82ecf3872c \ --hash=sha256:fe84440100e7045190da7f80219be9989b0b6db6acadb3ae9cfe0935d93ebf8c -croniter==2.0.7 \ - --hash=sha256:1041b912b4b1e03751a0993531becf77851ae6e8b334c9c76ffeffb8f055f53f \ - --hash=sha256:f15e80828d23920c4bb7f4d9340b932c9dcabecafc7775703c8b36d1253ed526 -cryptography==42.0.8 \ - --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ - --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ - --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ - --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ - --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ - --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ - --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ - --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ - --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ - --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ - --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ - --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ - --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ - --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ - --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ - --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ - --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ - --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ - --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ - --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ - --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ - --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ - --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ - --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e +croniter==3.0.0 \ + --hash=sha256:745b22960bb21ead6d79b25c0e849fffd2fe65215a944def6b235d4c72688b98 \ + --hash=sha256:92fcad6a4eb9749fa9cad2bba18916520b49408f6282a02aeac0dcda0586f7df +cryptography==43.0.0 \ + --hash=sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709 \ + --hash=sha256:2905ccf93a8a2a416f3ec01b1a7911c3fe4073ef35640e7ee5296754e30b762b \ + --hash=sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e \ + --hash=sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778 \ + --hash=sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22 \ + --hash=sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895 \ + --hash=sha256:47ca71115e545954e6c1d207dd13461ab81f4eccfcb1345eac874828b5e3eaaf \ + --hash=sha256:4a997df8c1c2aae1e1e5ac49c2e4f610ad037fc5a3aadc7b64e39dea42249431 \ + --hash=sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74 \ + --hash=sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc \ + --hash=sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66 \ + --hash=sha256:9a8d6802e0825767476f62aafed40532bd435e8a5f7d23bd8b4f5fd04cc80ecf \ + --hash=sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5 \ + --hash=sha256:b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e \ + --hash=sha256:cb013933d4c127349b3948aa8aaf2f12c0353ad0eccd715ca789c8a0f671646f \ + --hash=sha256:cc70b4b581f28d0a254d006f26949245e3657d40d8857066c2ae22a61222ef55 \ + --hash=sha256:ea9e57f8ea880eeea38ab5abf9fbe39f923544d7884228ec67d666abd60f5a47 \ + --hash=sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5 \ + --hash=sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0 daff==1.3.46 \ --hash=sha256:22d0da9fd6a3275b54c926a9c97b180f9258aad65113ea18f3fec52cbadcd818 dagster==1.7.14 \ @@ -169,6 +167,9 @@ dagster-cloud-cli==1.7.14 \ dagster-dbt==0.23.14 \ --hash=sha256:0f912d900d1ac91d841b6ca5c65f643ac2d819764a9b3b4de5df211e2664ecc5 \ --hash=sha256:ad287426de2786e0179d076ce193759c6e4e555b984aa380a5be9a20296f52f5 +dagster-embedded-elt==0.23.14 \ + --hash=sha256:ce2d7e4182681682c68a04522834812a9972d2c5986b523c864bea898da6ffc8 \ + --hash=sha256:df9a9acc623a85bd5a2d3808bb4f3dba98bd50c7858a248d32ec527414abd25f dagster-fivetran==0.23.14 \ --hash=sha256:84451fcb724ff4a6fa9693311d4daada100a5d605176a72f32373f0be5170d90 \ --hash=sha256:b17e819430f752e4de41a267291402805b2c5110892d01f74166b6d015a1172b @@ -228,6 +229,9 @@ deepdiff==7.0.1 \ defusedxml==0.7.1 \ --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 +dlt==0.5.1 \ + --hash=sha256:19d7920816fadd049a1a92c4ecc4e740bf1cd5f6484fce535715f8c214a835ce \ + --hash=sha256:dfa11e498feec3aca0022541850b6fa7dd3fddce4c2e6fef195530638269fec4 docstring-parser==0.16 \ --hash=sha256:538beabd0af1e2db0146b6bd3caa526c35a34d61af9fd2887f3a8a27a739aa6e \ --hash=sha256:bf0a1387354d3691d102edef7ec124f219ef639982d096e26e3b60aeffa90637 @@ -242,12 +246,21 @@ fastavro==1.9.5 \ filelock==3.15.4 \ --hash=sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb \ --hash=sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7 -fsspec==2024.6.1; python_version >= "3.12" \ +fsspec==2024.6.1 \ --hash=sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e \ --hash=sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49 +gitdb==4.0.11 \ + --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ + --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b github3-py==4.0.1 \ --hash=sha256:30d571076753efc389edc7f9aaef338a4fcb24b54d8968d5f39b1342f45ddd36 \ --hash=sha256:a89af7de25650612d1da2f0609622bcdeb07ee8a45a1c06b2d16a05e4234e753 +gitpython==3.1.43 \ + --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ + --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff +giturlparse==0.12.0 \ + --hash=sha256:412b74f2855f1da2fefa89fd8dde62df48476077a72fc19b62039554d27360eb \ + --hash=sha256:c0fff7c21acc435491b1779566e038757a205c1ffdcb47e4f81ea52ad8c3859a google-api-core==2.19.1 \ --hash=sha256:f12a9b8309b5e21d92483bbd47ce2c445861ec7d269ef6784ecc0ea8c1fa6125 \ --hash=sha256:f4695f1e3650b316a795108a76a1c416e6afb036199d1c1f1f110916df479ffd @@ -278,9 +291,9 @@ google-cloud-core==2.4.1 \ google-cloud-dataproc==5.10.1 \ --hash=sha256:28b763c9b019ca7d7c3e917ade04647c00494e77d4e682ca221d53e8d36f70af \ --hash=sha256:f3f0f0f3933328e80273774540368432550e296c255928657069a31a2de01c39 -google-cloud-storage==2.17.0 \ - --hash=sha256:49378abff54ef656b52dca5ef0f2eba9aa83dc2b2c72c78714b03a1a95fe9388 \ - --hash=sha256:5b393bc766b7a3bc6f5407b9e665b2450d36282614b7945e570b3480a456d1e1 +google-cloud-storage==2.18.0 \ + --hash=sha256:0aa3f7c57f3632f81b455d91558d2b27ada96eee2de3aaa17f689db1470d9578 \ + --hash=sha256:e8e1a9577952143c3fca8163005ecfadd2d70ec080fa158a8b305000e2c22fbb google-crc32c==1.5.0 \ --hash=sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7 google-resumable-media==2.7.1 \ @@ -292,7 +305,7 @@ googleapis-common-protos==1.63.2 \ googleapis-common-protos[grpc]==1.63.2 \ --hash=sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945 \ --hash=sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87 -greenlet==3.0.3; (platform_machine == "win32" or platform_machine == "WIN32" or platform_machine == "AMD64" or platform_machine == "amd64" or platform_machine == "x86_64" or platform_machine == "ppc64le" or platform_machine == "aarch64") and python_version < "3.13" \ +greenlet==3.0.3; platform_machine == "win32" or platform_machine == "WIN32" or platform_machine == "AMD64" or platform_machine == "amd64" or platform_machine == "x86_64" or platform_machine == "ppc64le" or platform_machine == "aarch64" \ --hash=sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676 \ --hash=sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc \ --hash=sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305 \ @@ -326,12 +339,18 @@ grpcio-status==1.62.2 \ gspread==6.1.2 \ --hash=sha256:345996fbb74051ee574e3d330a375ac625774f289459f73cb1f8b6fb3cf4cac5 \ --hash=sha256:b147688b8c7a18c9835d5f998997ec17c97c0470babcab17f65ac2b3a32402b7 +hexbytes==1.2.1 \ + --hash=sha256:515f00dddf31053db4d0d7636dd16061c1d896c3109b8e751005db4ca46bcca7 \ + --hash=sha256:e64890b203a31f4a23ef11470ecfcca565beaee9198df623047df322b757471a httplib2==0.22.0 \ --hash=sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc \ --hash=sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81 humanfriendly==10.0 \ --hash=sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477 \ --hash=sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc +humanize==4.10.0 \ + --hash=sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978 \ + --hash=sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6 idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 @@ -347,6 +366,9 @@ jinja2==3.1.4 \ joblib==1.4.2 \ --hash=sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6 \ --hash=sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e +jsonpath-ng==1.6.1 \ + --hash=sha256:086c37ba4917304850bd837aeab806670224d3f038fe2833ff593a672ef0a5fa \ + --hash=sha256:8f22cd8273d7772eea9aaa84d922e0841aa36fdb8a2c6b7f6c3791a16a9bc0be jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 @@ -364,6 +386,9 @@ leather==0.4.0 \ --hash=sha256:f964bec2086f3153a6c16e707f20cb718f811f57af116075f4c0f4805c608b95 logbook==1.5.3 \ --hash=sha256:66f454ada0f56eae43066f604a222b09893f98c1adc18df169710761b8f32fe8 +makefun==1.15.4 \ + --hash=sha256:945d078a7e01a903f2cbef738b33e0ebc52b8d35fb7e20c528ed87b5c80db5b7 \ + --hash=sha256:9f9b9904e7c397759374a88f4c57781fbab2a458dec78df4b3ee6272cd9fb010 mako==1.3.5 \ --hash=sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a \ --hash=sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc @@ -473,7 +498,10 @@ parsedatetime==2.6 \ pathspec==0.12.1 \ --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \ --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712 -pendulum==3.0.0; python_version >= "3.12" \ +pathvalidate==3.2.0 \ + --hash=sha256:5e8378cf6712bff67fbe7a8307d99fa8c1a0cb28aa477056f8fc374f0dff24ad \ + --hash=sha256:cc593caa6299b22b37f228148257997e2fa850eea2daf7e4cc9205cef6908dee +pendulum==3.0.0 \ --hash=sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9 \ --hash=sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f \ --hash=sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f \ @@ -488,6 +516,9 @@ pendulum==3.0.0; python_version >= "3.12" \ pex==2.11.0 \ --hash=sha256:22285310b06dedb45c5ce19d21fa0a5e102caad0030458e53b0cda4975c9a4ef \ --hash=sha256:9f3394da2c651d4d0adebf0a8e419abf1e37b6302fcbf66de54882281538e29c +ply==3.11 \ + --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3 \ + --hash=sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce prompt-toolkit==3.0.36 \ --hash=sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63 \ --hash=sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305 @@ -511,6 +542,10 @@ psutil==6.0.0; platform_system == "Windows" \ --hash=sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0 \ --hash=sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132 \ --hash=sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0 +psycopg2==2.9.9 \ + --hash=sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693 \ + --hash=sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156 \ + --hash=sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024 py-avro-schema==3.7.1 \ --hash=sha256:575f8a825e955ca48f6e2b7115a92ef2e07dbc9f49c784d0a8b2ffc7994e63fe \ --hash=sha256:a2ecf48e7ae226ecc5626b113dd51b68e7615bb92544d74d553a256e835b106c @@ -581,10 +616,10 @@ pynacl==1.5.0 \ --hash=sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394 \ --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \ --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543 -pyparsing==3.1.2; python_version > "3.0" \ +pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 -pyreadline3==3.4.1; sys_platform == "win32" and python_version >= "3.8" \ +pyreadline3==3.4.1; sys_platform == "win32" \ --hash=sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae \ --hash=sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb python-dateutil==2.9.0.post0 \ @@ -627,6 +662,9 @@ requests==2.32.3 \ requests-oauthlib==2.0.0 \ --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 +requirements-parser==0.10.1 \ + --hash=sha256:bb6b6e84d54ed4fee5153d0f2889615f791d8eba4521961a075f6d3f1fb2974d \ + --hash=sha256:cf208b8cb23d9b05b35ab0238db416a4b751441a41744c9e8b619362ee534d17 rich==13.7.1 \ --hash=sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222 \ --hash=sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432 @@ -665,15 +703,41 @@ scipy==1.14.0 \ --hash=sha256:bbc0471b5f22c11c389075d091d3885693fd3f5e9a54ce051b46308bc787e5d4 \ --hash=sha256:bff2438ea1330e06e53c424893ec0072640dac00f29c6a43a575cbae4c99b2b9 \ --hash=sha256:d91db2c41dd6c20646af280355d41dfa1ec7eead235642178bd57635a3f82209 -setuptools==71.0.3 \ - --hash=sha256:3d8531791a27056f4a38cd3e54084d8b1c4228ff9cf3f2d7dd075ec99f9fd70d \ - --hash=sha256:f501b6e6db709818dc76882582d9c516bf3b67b948864c5fa1d1624c09a49207 +semver==3.0.2 \ + --hash=sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc \ + --hash=sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4 +setuptools==71.1.0 \ + --hash=sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936 \ + --hash=sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855 shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de +simplejson==3.19.2 \ + --hash=sha256:176a1b524a3bd3314ed47029a86d02d5a95cc0bee15bd3063a1e1ec62b947de6 \ + --hash=sha256:1c768e7584c45094dca4b334af361e43b0aaa4844c04945ac7d43379eeda9bc2 \ + --hash=sha256:346820ae96aa90c7d52653539a57766f10f33dd4be609206c001432b59ddf89f \ + --hash=sha256:4969d974d9db826a2c07671273e6b27bc48e940738d768fa8f33b577f0978378 \ + --hash=sha256:6779105d2fcb7fcf794a6a2a233787f6bbd4731227333a072d8513b252ed374f \ + --hash=sha256:7cb98be113911cb0ad09e5523d0e2a926c09a465c9abb0784c9269efe4f95917 \ + --hash=sha256:9652e59c022e62a5b58a6f9948b104e5bb96d3b06940c6482588176f40f4914b \ + --hash=sha256:9c1a4393242e321e344213a90a1e3bf35d2f624aa8b8f6174d43e3c6b0e8f6eb \ + --hash=sha256:9eb442a2442ce417801c912df68e1f6ccfcd41577ae7274953ab3ad24ef7d82c \ + --hash=sha256:b8d940fd28eb34a7084877747a60873956893e377f15a32ad445fe66c972c3b8 \ + --hash=sha256:bcedf4cae0d47839fee7de344f96b5694ca53c786f28b5f773d4f0b265a159eb \ + --hash=sha256:c594642d6b13d225e10df5c16ee15b3398e21a35ecd6aee824f107a625690374 \ + --hash=sha256:de9a2792612ec6def556d1dc621fd6b2073aff015d64fba9f3e53349ad292734 \ + --hash=sha256:e2f5a398b5e77bb01b23d92872255e1bcb3c0c719a3be40b8df146570fe7781a \ + --hash=sha256:f3c7363a8cb8c5238878ec96c5eb0fc5ca2cb11fc0c7d2379863d342c6ee367a six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +sling==1.2.13 \ + --hash=sha256:40c77d0cf4ade41618b33c46704fbe85f6ebec0c1119decfee43125591d8bce0 +sling-linux-amd64==1.2.13 \ + --hash=sha256:ee1b674091d50ba3014b05944401df34248cbf28ba6a8074bc1e2fbfee5a8b1d +smmap==5.0.1 \ + --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ + --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da soupsieve==2.5 \ --hash=sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690 \ --hash=sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7 @@ -730,7 +794,7 @@ text-unidecode==1.3 \ threadpoolctl==3.5.0 \ --hash=sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107 \ --hash=sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467 -time-machine==2.14.2; implementation_name != "pypy" and python_version >= "3.12" \ +time-machine==2.14.2; implementation_name != "pypy" \ --hash=sha256:10c7cf6134e32e1074d37319f8b7662cc200ee9dd813a48b7520dd4aa49131a9 \ --hash=sha256:146aee86d237aa3a0ad1287718f1228107d21f3cd775c40f121a4670b3dee02c \ --hash=sha256:1ea4319010914c8d69bd16d9839a5c2f1df104b5a4704882bc44599d81611582 \ @@ -746,6 +810,9 @@ time-machine==2.14.2; implementation_name != "pypy" and python_version >= "3.12" tomli==2.0.1 \ --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f +tomlkit==0.13.0 \ + --hash=sha256:08ad192699734149f5b97b45f1f18dad7eb1b6d16bc72ad0c2335772650d7b72 \ + --hash=sha256:7075d3042d03b80f603482d69bf0c8f345c2b30e41699fd8883227f89972b264 toposort==1.10 \ --hash=sha256:bfbb479c53d0a696ea7402601f4e693c97b0367837c8898bc6471adfca37a6bd \ --hash=sha256:cbdbc0d0bee4d2695ab2ceec97fe0679e9c10eab4b2a87a9372b929e70563a87 @@ -758,13 +825,16 @@ typeguard==4.3.0 \ typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 +types-setuptools==71.1.0.20240723 \ + --hash=sha256:8a9349038c7e22d88e6c5d9c6705b347b22930424114a452c1712899e85131ff \ + --hash=sha256:ac9fc263f59d1e02bca49cb7270a12c47ab80b3b911fb4d92f1fecf978bfe88a typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 tzdata==2024.1 \ --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \ --hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252 -universal-pathlib==0.2.2; python_version >= "3.12" \ +universal-pathlib==0.2.2 \ --hash=sha256:6bc215548792ad5db3553708b1c19bafd9e2fa1667dc925ed404c95e52ae2f13 \ --hash=sha256:9bc176112d593348bb29806a47e409eda78dff8d95391d66dd6f85e443aaa75d uritemplate==4.1.1 \ @@ -794,6 +864,13 @@ wcwidth==0.2.13 \ websocket-client==1.8.0 \ --hash=sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 \ --hash=sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da +wheel==0.43.0 \ + --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \ + --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81 +win-precise-time==1.4.2; os_name == "nt" \ + --hash=sha256:0210dcea88a520c91de1708ae4c881e3c0ddc956daa08b9eabf2b7c35f3109f5 \ + --hash=sha256:85670f77cc8accd8f1e6d05073999f77561c23012a9ee988cbd44bb7ce655062 \ + --hash=sha256:89274785cbc5f2997e01675206da3203835a442c60fd97798415c6b3c179c0b9 zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index 37d6eb6db5..852ba6f736 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -361,6 +361,51 @@ locations: secretKeyRef: name: op-overgrad-api key: credential + - name: SOURCES__SQL_DATABASE__CREDENTIALS__DATABASE + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: database + - name: SOURCES__SQL_DATABASE__CREDENTIALS__DRIVERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: driver + - name: SOURCES__SQL_DATABASE__CREDENTIALS__HOST + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: ip + - name: SOURCES__SQL_DATABASE__CREDENTIALS__PASSWORD + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: password + - name: SOURCES__SQL_DATABASE__CREDENTIALS__PORT + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: port + - name: SOURCES__SQL_DATABASE__CREDENTIALS__USERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: username + - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: username + - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: password + - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: project-id run_k8s_config: container_config: resources: @@ -679,3 +724,48 @@ locations: secretKeyRef: name: op-overgrad-api key: credential + - name: SOURCES__SQL_DATABASE__CREDENTIALS__DATABASE + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: database + - name: SOURCES__SQL_DATABASE__CREDENTIALS__DRIVERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: driver + - name: SOURCES__SQL_DATABASE__CREDENTIALS__HOST + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: ip + - name: SOURCES__SQL_DATABASE__CREDENTIALS__PASSWORD + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: password + - name: SOURCES__SQL_DATABASE__CREDENTIALS__PORT + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: port + - name: SOURCES__SQL_DATABASE__CREDENTIALS__USERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: username + - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: username + - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: password + - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: project-id diff --git a/src/teamster/code_locations/kipptaf/definitions.py b/src/teamster/code_locations/kipptaf/definitions.py index 78544fe7b6..2de89c78c0 100644 --- a/src/teamster/code_locations/kipptaf/definitions.py +++ b/src/teamster/code_locations/kipptaf/definitions.py @@ -16,6 +16,7 @@ datagun, deanslist, fivetran, + illuminate, ldap, overgrad, performance_management, @@ -47,6 +48,7 @@ datagun, deanslist, fivetran, + illuminate, ldap, overgrad, performance_management, @@ -89,6 +91,7 @@ "adp_wfm": resources.ADP_WORKFORCE_MANAGER_RESOURCE, "adp_wfn": resources.ADP_WORKFORCE_NOW_RESOURCE, "airbyte": resources.AIRBYTE_CLOUD_RESOURCE, + "dlt": resources.DLT_RESOURCE, "fivetran": resources.FIVETRAN_RESOURCE, "google_directory": resources.GOOGLE_DIRECTORY_RESOURCE, "google_drive": resources.GOOGLE_DRIVE_RESOURCE, diff --git a/src/teamster/code_locations/kipptaf/illuminate/__init__.py b/src/teamster/code_locations/kipptaf/illuminate/__init__.py new file mode 100644 index 0000000000..25c0efc0aa --- /dev/null +++ b/src/teamster/code_locations/kipptaf/illuminate/__init__.py @@ -0,0 +1,5 @@ +from teamster.code_locations.kipptaf.illuminate.assets import assets + +__all__ = [ + "assets", +] diff --git a/src/teamster/code_locations/kipptaf/illuminate/assets.py b/src/teamster/code_locations/kipptaf/illuminate/assets.py new file mode 100644 index 0000000000..42ffb01d6a --- /dev/null +++ b/src/teamster/code_locations/kipptaf/illuminate/assets.py @@ -0,0 +1,25 @@ +from dagster import AssetExecutionContext +from dagster_embedded_elt.dlt import DagsterDltResource, dlt_assets +from dlt import pipeline + +from teamster.libraries.dlt_sources.sql_database import sql_database + + +@dlt_assets( + dlt_source=sql_database( + schema="dna_assessments", table_names=["assessments", "students_assessments"] + ), + dlt_pipeline=pipeline( + pipeline_name="illuminate", + destination="bigquery", + dataset_name="dev_illuminate", + progress="log", + ), +) +def illuminate_assets(context: AssetExecutionContext, dlt: DagsterDltResource): + yield from dlt.run(context=context) + + +assets = [ + illuminate_assets, +] diff --git a/src/teamster/code_locations/kipptaf/resources.py b/src/teamster/code_locations/kipptaf/resources.py index 27b8fe2917..0b6c05385a 100644 --- a/src/teamster/code_locations/kipptaf/resources.py +++ b/src/teamster/code_locations/kipptaf/resources.py @@ -1,5 +1,6 @@ from dagster import EnvVar from dagster_airbyte import AirbyteCloudResource +from dagster_embedded_elt.dlt import DagsterDltResource from dagster_fivetran import FivetranResource from teamster.libraries.adp.workforce_manager.resources import ( @@ -51,6 +52,8 @@ username=EnvVar("AMPLIFY_DDS_USERNAME"), password=EnvVar("AMPLIFY_DDS_PASSWORD") ) +DLT_RESOURCE = DagsterDltResource() + FIVETRAN_RESOURCE = FivetranResource( api_key=EnvVar("FIVETRAN_API_KEY"), api_secret=EnvVar("FIVETRAN_API_SECRET"), From b5e99e9f8afb4649520b17c7c3b17873f8c46e55 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:09:29 +0000 Subject: [PATCH 03/20] build: fix psycopg2-binary --- pdm.lock | 20 +++++++++++++++----- pyproject.toml | 2 +- requirements.txt | 18 ++++++++++++++---- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/pdm.lock b/pdm.lock index 60175ef96f..e5f7239a3a 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:2b4b1bb6960e6d64ea76fdbba4a3e8983a112653be52f8115f1e00d81866756a" +content_hash = "sha256:b7f298a74b8a9ab16927a8eabd63a31195640e15b3001f20cbb74adcf525591c" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -2337,15 +2337,25 @@ files = [ ] [[package]] -name = "psycopg2" +name = "psycopg2-binary" version = "2.9.9" requires_python = ">=3.7" summary = "psycopg2 - Python-PostgreSQL Database Adapter" groups = ["default"] files = [ - {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, - {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, - {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, + {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 7ab207bc61..f2adb606f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "tableauserverclient>=0.25", "tenacity>=8.2.3", "dagster-embedded-elt>=0.23.14", - "psycopg2>=2.9.9", + "psycopg2-binary>=2.9.9", ] requires-python = ">=3.12,<3.13" license = { text = "GPL-3.0-or-later" } diff --git a/requirements.txt b/requirements.txt index 54f39f0437..75f5c4a692 100644 --- a/requirements.txt +++ b/requirements.txt @@ -542,10 +542,20 @@ psutil==6.0.0; platform_system == "Windows" \ --hash=sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0 \ --hash=sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132 \ --hash=sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0 -psycopg2==2.9.9 \ - --hash=sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693 \ - --hash=sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156 \ - --hash=sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024 +psycopg2-binary==2.9.9 \ + --hash=sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493 \ + --hash=sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996 \ + --hash=sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe \ + --hash=sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93 \ + --hash=sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119 \ + --hash=sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c \ + --hash=sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab \ + --hash=sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf \ + --hash=sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212 \ + --hash=sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb \ + --hash=sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d \ + --hash=sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba \ + --hash=sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07 py-avro-schema==3.7.1 \ --hash=sha256:575f8a825e955ca48f6e2b7115a92ef2e07dbc9f49c784d0a8b2ffc7994e63fe \ --hash=sha256:a2ecf48e7ae226ecc5626b113dd51b68e7615bb92544d74d553a256e835b106c From 29fb4f877b69795da0c483bf600a551751a4b352 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:45:31 +0000 Subject: [PATCH 04/20] build: rm bq creds --- .../code_locations/kipptaf/dagster-cloud.yaml | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index 852ba6f736..e4c24f934c 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -391,21 +391,21 @@ locations: secretKeyRef: name: op-illuminate-db key: username - - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: username - - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: password - - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: project-id + # - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: username + # - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: password + # - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: project-id run_k8s_config: container_config: resources: @@ -754,18 +754,18 @@ locations: secretKeyRef: name: op-illuminate-db key: username - - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: username - - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: password - - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: project-id + # - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: username + # - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: password + # - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: project-id From 63ac5df72289afb034b18ee8737e7b16c78ef4d9 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:39:21 +0000 Subject: [PATCH 05/20] feat: sling resource --- .../code_locations/kipptaf/resources.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/teamster/code_locations/kipptaf/resources.py b/src/teamster/code_locations/kipptaf/resources.py index 0b6c05385a..a625e700f0 100644 --- a/src/teamster/code_locations/kipptaf/resources.py +++ b/src/teamster/code_locations/kipptaf/resources.py @@ -1,6 +1,7 @@ from dagster import EnvVar from dagster_airbyte import AirbyteCloudResource from dagster_embedded_elt.dlt import DagsterDltResource +from dagster_embedded_elt.sling.resources import SlingConnectionResource, SlingResource from dagster_fivetran import FivetranResource from teamster.libraries.adp.workforce_manager.resources import ( @@ -165,3 +166,20 @@ username=EnvVar("LITTLESIS_SFTP_USERNAME"), password=EnvVar("LITTLESIS_SFTP_PASSWORD"), ) + +SLING_RESOURCE = SlingResource( + connections=[ + SlingConnectionResource( + name="ILLUMINATE", + type="postgres", + host=EnvVar("SNOWFLAKE_HOST"), + user=EnvVar("SNOWFLAKE_USER"), + ), + SlingConnectionResource( + name="BIGQUERY", + type="bigquery", + host=EnvVar("SNOWFLAKE_HOST"), + user=EnvVar("SNOWFLAKE_USER"), + ), + ] +) From 443bf3ddb698aa418543df193e402252eb5f7808 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Fri, 8 Nov 2024 22:49:26 +0000 Subject: [PATCH 06/20] fix: lint --- .trunk/trunk.yaml | 2 +- pdm.lock | 330 ++++++------------ pyproject.toml | 3 +- requirements.txt | 25 +- .../code_locations/kipptaf/definitions.py | 1 + .../code_locations/kipptaf/resources.py | 4 + 6 files changed, 143 insertions(+), 222 deletions(-) diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 036d2e33af..6a84010738 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -83,4 +83,4 @@ actions: - trunk-announce - trunk-check-pre-push - trunk-fmt-pre-commit - - trunk-upgrade-available \ No newline at end of file + - trunk-upgrade-available diff --git a/pdm.lock b/pdm.lock index 9cd9407f0b..7f94ac56e0 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,8 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:853ce1ab8fc1945aa1d8cc54631705063453009f77d7c6204801ffb03307cd5d" -content_hash = "sha256:b7f298a74b8a9ab16927a8eabd63a31195640e15b3001f20cbb74adcf525591c" +content_hash = "sha256:7550121135ad9170f6eba2cc38681d7e0a4b54cf75791735336b40c6bd48bf77" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -86,8 +85,8 @@ requires_python = ">=3.8" summary = "Bash tab completion for argparse" groups = ["dev"] files = [ - {file = "argcomplete-3.4.0-py3-none-any.whl", hash = "sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5"}, - {file = "argcomplete-3.4.0.tar.gz", hash = "sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f"}, + {file = "argcomplete-3.5.1-py3-none-any.whl", hash = "sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363"}, + {file = "argcomplete-3.5.1.tar.gz", hash = "sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4"}, ] [[package]] @@ -160,7 +159,6 @@ files = [ [[package]] name = "bcrypt" version = "4.2.0" -version = "4.2.0" requires_python = ">=3.7" summary = "Modern password hashing for your software and your servers" groups = ["default"] @@ -188,29 +186,6 @@ files = [ {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"}, {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"}, {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"}, - {file = "bcrypt-4.2.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7"}, - {file = "bcrypt-4.2.0-cp37-abi3-win32.whl", hash = "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458"}, - {file = "bcrypt-4.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5"}, - {file = "bcrypt-4.2.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8"}, - {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"}, - {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"}, - {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"}, ] [[package]] @@ -341,7 +316,6 @@ files = [ [[package]] name = "cmake" version = "3.30.5" -version = "3.30.1" requires_python = ">=3.7" summary = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" groups = ["default"] @@ -367,23 +341,6 @@ files = [ {file = "cmake-3.30.5-py3-none-win_amd64.whl", hash = "sha256:dddb1c82e69c8cf79a314f57c2cbb1b3c152758fa0997a5f5ddf8fc8207c994a"}, {file = "cmake-3.30.5-py3-none-win_arm64.whl", hash = "sha256:ef454176664c941da91197feb91b68cab91ef0354ab0c4d7bfb70e069cb700f1"}, {file = "cmake-3.30.5.tar.gz", hash = "sha256:2815a2b4dde61a78648a2ff4318d2d303eb41600c4bdb32fcb237b4746444ed2"}, - {file = "cmake-3.30.1-py3-none-macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:d56c1d96c4f8277bebbef768ad008a15d8b20b5946c87d888b85251d00b7509d"}, - {file = "cmake-3.30.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:15796c4ca5f32207d315a402604785e3288a9ca8bcf3a59427af31c21a09df50"}, - {file = "cmake-3.30.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c94ce1df31a0e9244e1ec00b1efc2c4df2cbb9450d640087bacb46dc99a90abd"}, - {file = "cmake-3.30.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8ec762c5364a4d33cbc395c435a0afbf706cc623f55d7c51166d6c48e745dfd"}, - {file = "cmake-3.30.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:065d68e35f6fa7973982f2d725ee8662b7e94cb5fd6856787608be7d62f64e30"}, - {file = "cmake-3.30.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:88c561e29af6a21fb4dc80f9438767af8ba5081d2c58cfc2a16298076d731539"}, - {file = "cmake-3.30.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77577bdc99c6597da9674d788f23421c0417c598b411d6b8ada64d0c70ff32a5"}, - {file = "cmake-3.30.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7743a2ba38edf56701ad3a40fac09ea3bba0538c6843fbc29cfccdbfc567873c"}, - {file = "cmake-3.30.1-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:b6cd7e8b854e4bd366632317ea3a8d7554fd5de8f5056ba13fab78576b31a2f8"}, - {file = "cmake-3.30.1-py3-none-musllinux_1_1_i686.whl", hash = "sha256:21c9db134fb859bbf163431f13c38c10bbcbc9a93287f6df61a305fe80c030b1"}, - {file = "cmake-3.30.1-py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:6d86335029ca716bad3c6fbcb83eb14acb0f70daa961cdf229a349057c7f1df4"}, - {file = "cmake-3.30.1-py3-none-musllinux_1_1_s390x.whl", hash = "sha256:212a6061ea724dfe89225005303f9f5ec804f46338338e9061381c22aca990ae"}, - {file = "cmake-3.30.1-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:3f6b8f12be57e8246f553ff1b081d2d02dc0b6194565e92ff08eb7159eceef24"}, - {file = "cmake-3.30.1-py3-none-win32.whl", hash = "sha256:8b15804f28dd3c22798c93e38be4d328e2aca00cc852a5afd72ca2332e28a021"}, - {file = "cmake-3.30.1-py3-none-win_amd64.whl", hash = "sha256:b512dfdbfe99d608aa22a152dac614fa00456b6adc2a24f4e586ab781b1c573a"}, - {file = "cmake-3.30.1-py3-none-win_arm64.whl", hash = "sha256:5b2556b2e999169121a7720f4e597848391d174dde05de9dfeec43c29565c97f"}, - {file = "cmake-3.30.1.tar.gz", hash = "sha256:51f01ce429a55acbfe1f1baf507f0fe6916243a9f3e5868a928d073ae4b18ef9"}, ] [[package]] @@ -438,7 +395,6 @@ files = [ [[package]] name = "croniter" version = "3.0.4" -version = "3.0.0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.6" summary = "croniter provides iteration for datetime object with cron like format" groups = ["default", "dev"] @@ -449,14 +405,11 @@ dependencies = [ files = [ {file = "croniter-3.0.4-py2.py3-none-any.whl", hash = "sha256:96e14cdd5dcb479dd48d7db14b53d8434b188dfb9210448bef6f65663524a6f0"}, {file = "croniter-3.0.4.tar.gz", hash = "sha256:f9dcd4bdb6c97abedb6f09d6ed3495b13ede4d4544503fa580b6372a56a0c520"}, - {file = "croniter-3.0.0-py2.py3-none-any.whl", hash = "sha256:92fcad6a4eb9749fa9cad2bba18916520b49408f6282a02aeac0dcda0586f7df"}, - {file = "croniter-3.0.0.tar.gz", hash = "sha256:745b22960bb21ead6d79b25c0e849fffd2fe65215a944def6b235d4c72688b98"}, ] [[package]] name = "cryptography" version = "43.0.3" -version = "43.0.0" requires_python = ">=3.7" summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." groups = ["default"] @@ -483,25 +436,6 @@ files = [ {file = "cryptography-43.0.3-cp39-abi3-win32.whl", hash = "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995"}, {file = "cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362"}, {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, - {file = "cryptography-43.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74"}, - {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895"}, - {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22"}, - {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ea9e57f8ea880eeea38ab5abf9fbe39f923544d7884228ec67d666abd60f5a47"}, - {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9a8d6802e0825767476f62aafed40532bd435e8a5f7d23bd8b4f5fd04cc80ecf"}, - {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cc70b4b581f28d0a254d006f26949245e3657d40d8857066c2ae22a61222ef55"}, - {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a997df8c1c2aae1e1e5ac49c2e4f610ad037fc5a3aadc7b64e39dea42249431"}, - {file = "cryptography-43.0.0-cp37-abi3-win32.whl", hash = "sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc"}, - {file = "cryptography-43.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778"}, - {file = "cryptography-43.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66"}, - {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5"}, - {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e"}, - {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5"}, - {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb013933d4c127349b3948aa8aaf2f12c0353ad0eccd715ca789c8a0f671646f"}, - {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0"}, - {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2905ccf93a8a2a416f3ec01b1a7911c3fe4073ef35640e7ee5296754e30b762b"}, - {file = "cryptography-43.0.0-cp39-abi3-win32.whl", hash = "sha256:47ca71115e545954e6c1d207dd13461ab81f4eccfcb1345eac874828b5e3eaaf"}, - {file = "cryptography-43.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709"}, - {file = "cryptography-43.0.0.tar.gz", hash = "sha256:b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e"}, ] [[package]] @@ -638,32 +572,32 @@ files = [ [[package]] name = "dagster-embedded-elt" -version = "0.23.14" -requires_python = "<3.13,>=3.8" +version = "0.25.1" +requires_python = "<3.13,>=3.9" summary = "Package for performing ETL/ELT tasks with Dagster." groups = ["default"] dependencies = [ - "dagster==1.7.14", + "dagster==1.9.1", "dlt>=0.4", "sling>=1.1.5", ] files = [ - {file = "dagster-embedded-elt-0.23.14.tar.gz", hash = "sha256:ce2d7e4182681682c68a04522834812a9972d2c5986b523c864bea898da6ffc8"}, - {file = "dagster_embedded_elt-0.23.14-py3-none-any.whl", hash = "sha256:df9a9acc623a85bd5a2d3808bb4f3dba98bd50c7858a248d32ec527414abd25f"}, + {file = "dagster-embedded-elt-0.25.1.tar.gz", hash = "sha256:41eba4c82814d8b7c1fb0dd02fb0113b7a32d9b3d1abfb94a2ce8bed45223893"}, + {file = "dagster_embedded_elt-0.25.1-py3-none-any.whl", hash = "sha256:b7eb014bf2fe93cab717ef5564b2596b0b0bf83c6f9397e4a36fe580e2a2980c"}, ] [[package]] name = "dagster-fivetran" -version = "0.23.14" -requires_python = "<3.13,>=3.8" +version = "0.25.1" +requires_python = "<3.13,>=3.9" summary = "Package for integrating Fivetran with Dagster." groups = ["default"] dependencies = [ - "dagster==1.7.14", + "dagster==1.9.1", ] files = [ - {file = "dagster-fivetran-0.23.14.tar.gz", hash = "sha256:b17e819430f752e4de41a267291402805b2c5110892d01f74166b6d015a1172b"}, - {file = "dagster_fivetran-0.23.14-py3-none-any.whl", hash = "sha256:84451fcb724ff4a6fa9693311d4daada100a5d605176a72f32373f0be5170d90"}, + {file = "dagster-fivetran-0.25.1.tar.gz", hash = "sha256:0ef431df843b4410ba173646eceee6dd1c3a25ac4af597efeaf266f5ffbf8af2"}, + {file = "dagster_fivetran-0.25.1-py3-none-any.whl", hash = "sha256:4d67152dc6ed86e92f8bd0140b3816504bec06acac10ad729621d7e7fa395218"}, ] [[package]] @@ -1009,7 +943,7 @@ files = [ [[package]] name = "dlt" -version = "0.5.1" +version = "1.3.0" requires_python = "<3.13,>=3.8.1" summary = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." groups = ["default"] @@ -1020,6 +954,7 @@ dependencies = [ "fsspec>=2022.4.0", "gitpython>=3.1.29", "giturlparse>=0.10.0", + "graphlib-backport; python_version < \"3.9\"", "hexbytes>=0.2.2", "humanize>=4.4.0", "jsonpath-ng>=1.5.3", @@ -1028,6 +963,7 @@ dependencies = [ "packaging>=21.1", "pathvalidate>=2.5.2", "pendulum>=2.1.2", + "pluggy>=1.3.0", "pytz>=2022.6", "requests>=2.26.0", "requirements-parser>=0.5.0", @@ -1041,8 +977,8 @@ dependencies = [ "win-precise-time>=1.4.2; os_name == \"nt\"", ] files = [ - {file = "dlt-0.5.1-py3-none-any.whl", hash = "sha256:19d7920816fadd049a1a92c4ecc4e740bf1cd5f6484fce535715f8c214a835ce"}, - {file = "dlt-0.5.1.tar.gz", hash = "sha256:dfa11e498feec3aca0022541850b6fa7dd3fddce4c2e6fef195530638269fec4"}, + {file = "dlt-1.3.0-py3-none-any.whl", hash = "sha256:e2583ed0ad4a0d9941b8f9cb0e078f4443bcbeb0e1cf1cce586cf35107ccf266"}, + {file = "dlt-1.3.0.tar.gz", hash = "sha256:57eecee99ace25b6d37027a78f59f8c735d1913cc81f1101e1b47bf96fc544b8"}, ] [[package]] @@ -1377,7 +1313,6 @@ files = [ [[package]] name = "google-cloud-storage" version = "2.18.2" -version = "2.18.0" requires_python = ">=3.7" summary = "Google Cloud Storage API client library" groups = ["default"] @@ -1392,8 +1327,6 @@ dependencies = [ files = [ {file = "google_cloud_storage-2.18.2-py2.py3-none-any.whl", hash = "sha256:97a4d45c368b7d401ed48c4fdfe86e1e1cb96401c9e199e419d289e2c0370166"}, {file = "google_cloud_storage-2.18.2.tar.gz", hash = "sha256:aaf7acd70cdad9f274d29332673fcab98708d0e1f4dceb5a5356aaef06af4d99"}, - {file = "google_cloud_storage-2.18.0-py2.py3-none-any.whl", hash = "sha256:e8e1a9577952143c3fca8163005ecfadd2d70ec080fa158a8b305000e2c22fbb"}, - {file = "google_cloud_storage-2.18.0.tar.gz", hash = "sha256:0aa3f7c57f3632f81b455d91558d2b27ada96eee2de3aaa17f689db1470d9578"}, ] [[package]] @@ -1541,7 +1474,7 @@ version = "3.1.1" requires_python = ">=3.7" summary = "Lightweight in-process concurrent programming" groups = ["default", "dev"] -marker = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"" +marker = "(platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"" files = [ {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"}, {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"}, @@ -1678,19 +1611,19 @@ files = [ [[package]] name = "httptools" -version = "0.6.1" +version = "0.6.4" requires_python = ">=3.8.0" summary = "A collection of framework independent HTTP protocol utils." groups = ["dev"] files = [ - {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75c8022dca7935cba14741a42744eee13ba05db00b27a4b940f0d646bd4d56d0"}, - {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:48ed8129cd9a0d62cf4d1575fcf90fb37e3ff7d5654d3a5814eb3d55f36478c2"}, - {file = "httptools-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f58e335a1402fb5a650e271e8c2d03cfa7cea46ae124649346d17bd30d59c90"}, - {file = "httptools-0.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ad80d7176aa5788902f207a4e79885f0576134695dfb0fefc15b7a4648d503"}, - {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9bb68d3a085c2174c2477eb3ffe84ae9fb4fde8792edb7bcd09a1d8467e30a84"}, - {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b512aa728bc02354e5ac086ce76c3ce635b62f5fbc32ab7082b5e582d27867bb"}, - {file = "httptools-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:97662ce7fb196c785344d00d638fc9ad69e18ee4bfb4000b35a52efe5adcc949"}, - {file = "httptools-0.6.1.tar.gz", hash = "sha256:c6e26c30455600b95d94b1b836085138e82f177351454ee841c148f93a9bad5a"}, + {file = "httptools-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2"}, + {file = "httptools-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44"}, + {file = "httptools-0.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1"}, + {file = "httptools-0.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2"}, + {file = "httptools-0.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81"}, + {file = "httptools-0.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f"}, + {file = "httptools-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970"}, + {file = "httptools-0.6.4.tar.gz", hash = "sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c"}, ] [[package]] @@ -1711,13 +1644,13 @@ files = [ [[package]] name = "humanize" -version = "4.10.0" -requires_python = ">=3.8" +version = "4.11.0" +requires_python = ">=3.9" summary = "Python humanize utilities" groups = ["default"] files = [ - {file = "humanize-4.10.0-py3-none-any.whl", hash = "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6"}, - {file = "humanize-4.10.0.tar.gz", hash = "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978"}, + {file = "humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0"}, + {file = "humanize-4.11.0.tar.gz", hash = "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be"}, ] [[package]] @@ -1819,15 +1752,14 @@ files = [ [[package]] name = "jsonpath-ng" -version = "1.6.1" +version = "1.7.0" summary = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." groups = ["default"] dependencies = [ "ply", ] files = [ - {file = "jsonpath-ng-1.6.1.tar.gz", hash = "sha256:086c37ba4917304850bd837aeab806670224d3f038fe2833ff593a672ef0a5fa"}, - {file = "jsonpath_ng-1.6.1-py3-none-any.whl", hash = "sha256:8f22cd8273d7772eea9aaa84d922e0841aa36fdb8a2c6b7f6c3791a16a9bc0be"}, + {file = "jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c"}, ] [[package]] @@ -1921,15 +1853,15 @@ files = [ [[package]] name = "makefun" -version = "1.15.4" +version = "1.15.6" summary = "Small library to dynamically create python functions." groups = ["default"] dependencies = [ "funcsigs; python_version < \"3.3\"", ] files = [ - {file = "makefun-1.15.4-py2.py3-none-any.whl", hash = "sha256:945d078a7e01a903f2cbef738b33e0ebc52b8d35fb7e20c528ed87b5c80db5b7"}, - {file = "makefun-1.15.4.tar.gz", hash = "sha256:9f9b9904e7c397759374a88f4c57781fbab2a458dec78df4b3ee6272cd9fb010"}, + {file = "makefun-1.15.6-py2.py3-none-any.whl", hash = "sha256:e69b870f0bb60304765b1e3db576aaecf2f9b3e5105afe8cfeff8f2afe6ad067"}, + {file = "makefun-1.15.6.tar.gz", hash = "sha256:26bc63442a6182fb75efed8b51741dd2d1db2f176bec8c64e20a586256b8f149"}, ] [[package]] @@ -2299,13 +2231,13 @@ files = [ [[package]] name = "pathvalidate" -version = "3.2.0" +version = "3.2.1" requires_python = ">=3.7" summary = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." groups = ["default"] files = [ - {file = "pathvalidate-3.2.0-py3-none-any.whl", hash = "sha256:cc593caa6299b22b37f228148257997e2fa850eea2daf7e4cc9205cef6908dee"}, - {file = "pathvalidate-3.2.0.tar.gz", hash = "sha256:5e8378cf6712bff67fbe7a8307d99fa8c1a0cb28aa477056f8fc374f0dff24ad"}, + {file = "pathvalidate-3.2.1-py3-none-any.whl", hash = "sha256:9a6255eb8f63c9e2135b9be97a5ce08f10230128c4ae7b3e935378b82b22c4c9"}, + {file = "pathvalidate-3.2.1.tar.gz", hash = "sha256:f5d07b1e2374187040612a1fcd2bcb2919f8db180df254c9581bb90bf903377d"}, ] [[package]] @@ -2314,7 +2246,6 @@ version = "3.0.0" requires_python = ">=3.8" summary = "Python datetimes made easy" groups = ["default"] -groups = ["default", "dev"] dependencies = [ "backports-zoneinfo>=0.2.1; python_version < \"3.9\"", "importlib-resources>=5.9.0; python_version < \"3.9\"", @@ -2363,7 +2294,7 @@ name = "pluggy" version = "1.5.0" requires_python = ">=3.8" summary = "plugin and hook calling mechanisms for python" -groups = ["dev"] +groups = ["default", "dev"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -2458,36 +2389,14 @@ summary = "Cross-platform lib for process and system monitoring in Python." groups = ["default", "dev"] marker = "platform_system == \"Windows\"" files = [ - {file = "psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"}, - {file = "psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"}, - {file = "psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"}, - {file = "psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"}, - {file = "psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"}, -] - -[[package]] -name = "psycopg2-binary" -version = "2.9.9" -requires_python = ">=3.7" -summary = "psycopg2 - Python-PostgreSQL Database Adapter" -groups = ["default"] -files = [ - {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"}, + {file = "psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"}, + {file = "psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"}, + {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, ] [[package]] @@ -2630,6 +2539,7 @@ extras = ["email"] requires_python = ">=3.8" summary = "Data validation using Python type hints" groups = ["dev"] +marker = "python_version ~= \"3.11\"" dependencies = [ "email-validator>=2.0.0", "pydantic==2.9.2", @@ -2705,6 +2615,7 @@ version = "3.2.0" requires_python = ">=3.9" summary = "pyparsing module - Classes and methods to define and execute parsing grammars" groups = ["default"] +marker = "python_version > \"3.0\"" files = [ {file = "pyparsing-3.2.0-py3-none-any.whl", hash = "sha256:93d9577b88da0bbea8cc8334ee8b918ed014968fd2ec383e868fb8afb1ccef84"}, {file = "pyparsing-3.2.0.tar.gz", hash = "sha256:cbf74e27246d595d9a74b186b810f6fbb86726dbf3b9532efb343f6d7294fe9c"}, @@ -2730,7 +2641,7 @@ version = "3.5.4" requires_python = ">=3.8" summary = "A python implementation of GNU readline." groups = ["default", "dev"] -marker = "sys_platform == \"win32\"" +marker = "sys_platform == \"win32\" and python_version >= \"3.8\"" files = [ {file = "pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6"}, {file = "pyreadline3-3.5.4.tar.gz", hash = "sha256:8d57d53039a1c75adba8e50dd3d992b28143480816187ea5efbd5c78e6c885b7"}, @@ -2739,7 +2650,6 @@ files = [ [[package]] name = "pytest" version = "8.3.3" -version = "8.3.1" requires_python = ">=3.8" summary = "pytest: simple powerful testing with Python" groups = ["dev"] @@ -2749,14 +2659,11 @@ dependencies = [ "iniconfig", "packaging", "pluggy<2,>=1.5", - "pluggy<2,>=1.5", "tomli>=1; python_version < \"3.11\"", ] files = [ {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, - {file = "pytest-8.3.1-py3-none-any.whl", hash = "sha256:e9600ccf4f563976e2c99fa02c7624ab938296551f280835ee6516df8bc4ae8c"}, - {file = "pytest-8.3.1.tar.gz", hash = "sha256:7e8e5c5abd6e93cb1cc151f23e57adc31fcf8cfd2a3ff2da63e23f732de35db6"}, ] [[package]] @@ -2926,16 +2833,17 @@ files = [ [[package]] name = "requirements-parser" -version = "0.10.1" +version = "0.11.0" requires_python = "<4.0,>=3.8" summary = "This is a small Python module for parsing Pip requirement files." groups = ["default"] dependencies = [ + "packaging>=23.2", "types-setuptools>=69.1.0", ] files = [ - {file = "requirements_parser-0.10.1-py3-none-any.whl", hash = "sha256:bb6b6e84d54ed4fee5153d0f2889615f791d8eba4521961a075f6d3f1fb2974d"}, - {file = "requirements_parser-0.10.1.tar.gz", hash = "sha256:cf208b8cb23d9b05b35ab0238db416a4b751441a41744c9e8b619362ee534d17"}, + {file = "requirements_parser-0.11.0-py3-none-any.whl", hash = "sha256:50379eb50311834386c2568263ae5225d7b9d0867fb55cf4ecc93959de2c2684"}, + {file = "requirements_parser-0.11.0.tar.gz", hash = "sha256:35f36dc969d14830bf459803da84f314dc3d17c802592e9e970f63d0359e5920"}, ] [[package]] @@ -3022,15 +2930,15 @@ dependencies = [ "numpy<2.3,>=1.23.5", ] files = [ - {file = "scipy-1.14.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff2438ea1330e06e53c424893ec0072640dac00f29c6a43a575cbae4c99b2b9"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bbc0471b5f22c11c389075d091d3885693fd3f5e9a54ce051b46308bc787e5d4"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:64b2ff514a98cf2bb734a9f90d32dc89dc6ad4a4a36a312cd0d6327170339eb0"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:7d3da42fbbbb860211a811782504f38ae7aaec9de8764a9bef6b262de7a2b50f"}, - {file = "scipy-1.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d91db2c41dd6c20646af280355d41dfa1ec7eead235642178bd57635a3f82209"}, - {file = "scipy-1.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a01cc03bcdc777c9da3cfdcc74b5a75caffb48a6c39c8450a9a05f82c4250a14"}, - {file = "scipy-1.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:65df4da3c12a2bb9ad52b86b4dcf46813e869afb006e58be0f516bc370165159"}, - {file = "scipy-1.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:4c4161597c75043f7154238ef419c29a64ac4a7c889d588ea77690ac4d0d9b20"}, - {file = "scipy-1.14.0.tar.gz", hash = "sha256:b5923f48cb840380f9854339176ef21763118a7300a88203ccd0bdd26e58527b"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:af29a935803cc707ab2ed7791c44288a682f9c8107bc00f0eccc4f92c08d6e07"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2843f2d527d9eebec9a43e6b406fb7266f3af25a751aa91d62ff416f54170bc5"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:eb58ca0abd96911932f688528977858681a59d61a7ce908ffd355957f7025cfc"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30ac8812c1d2aab7131a79ba62933a2a76f582d5dbbc695192453dae67ad6310"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9ea80f2e65bdaa0b7627fb00cbeb2daf163caa015e59b7516395fe3bd1e066"}, + {file = "scipy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:edaf02b82cd7639db00dbff629995ef185c8df4c3ffa71a5562a595765a06ce1"}, + {file = "scipy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f"}, + {file = "scipy-1.14.1.tar.gz", hash = "sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417"}, ] [[package]] @@ -3047,15 +2955,12 @@ files = [ [[package]] name = "setuptools" version = "75.3.0" -version = "71.1.0" requires_python = ">=3.8" summary = "Easily download, build, install, upgrade, and uninstall Python packages" groups = ["default", "dev"] files = [ {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, - {file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"}, - {file = "setuptools-71.1.0.tar.gz", hash = "sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936"}, ] [[package]] @@ -3071,26 +2976,26 @@ files = [ [[package]] name = "simplejson" -version = "3.19.2" -requires_python = ">=2.5, !=3.0.*, !=3.1.*, !=3.2.*" +version = "3.19.3" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.5" summary = "Simple, fast, extensible JSON encoder/decoder for Python" groups = ["default"] files = [ - {file = "simplejson-3.19.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b8d940fd28eb34a7084877747a60873956893e377f15a32ad445fe66c972c3b8"}, - {file = "simplejson-3.19.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4969d974d9db826a2c07671273e6b27bc48e940738d768fa8f33b577f0978378"}, - {file = "simplejson-3.19.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c594642d6b13d225e10df5c16ee15b3398e21a35ecd6aee824f107a625690374"}, - {file = "simplejson-3.19.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2f5a398b5e77bb01b23d92872255e1bcb3c0c719a3be40b8df146570fe7781a"}, - {file = "simplejson-3.19.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:176a1b524a3bd3314ed47029a86d02d5a95cc0bee15bd3063a1e1ec62b947de6"}, - {file = "simplejson-3.19.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3c7363a8cb8c5238878ec96c5eb0fc5ca2cb11fc0c7d2379863d342c6ee367a"}, - {file = "simplejson-3.19.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:346820ae96aa90c7d52653539a57766f10f33dd4be609206c001432b59ddf89f"}, - {file = "simplejson-3.19.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de9a2792612ec6def556d1dc621fd6b2073aff015d64fba9f3e53349ad292734"}, - {file = "simplejson-3.19.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1c768e7584c45094dca4b334af361e43b0aaa4844c04945ac7d43379eeda9bc2"}, - {file = "simplejson-3.19.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:9652e59c022e62a5b58a6f9948b104e5bb96d3b06940c6482588176f40f4914b"}, - {file = "simplejson-3.19.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9c1a4393242e321e344213a90a1e3bf35d2f624aa8b8f6174d43e3c6b0e8f6eb"}, - {file = "simplejson-3.19.2-cp312-cp312-win32.whl", hash = "sha256:7cb98be113911cb0ad09e5523d0e2a926c09a465c9abb0784c9269efe4f95917"}, - {file = "simplejson-3.19.2-cp312-cp312-win_amd64.whl", hash = "sha256:6779105d2fcb7fcf794a6a2a233787f6bbd4731227333a072d8513b252ed374f"}, - {file = "simplejson-3.19.2-py3-none-any.whl", hash = "sha256:bcedf4cae0d47839fee7de344f96b5694ca53c786f28b5f773d4f0b265a159eb"}, - {file = "simplejson-3.19.2.tar.gz", hash = "sha256:9eb442a2442ce417801c912df68e1f6ccfcd41577ae7274953ab3ad24ef7d82c"}, + {file = "simplejson-3.19.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:66a0399e21c2112acacfebf3d832ebe2884f823b1c7e6d1363f2944f1db31a99"}, + {file = "simplejson-3.19.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6ef9383c5e05f445be60f1735c1816163c874c0b1ede8bb4390aff2ced34f333"}, + {file = "simplejson-3.19.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42e5acf80d4d971238d4df97811286a044d720693092b20a56d5e56b7dcc5d09"}, + {file = "simplejson-3.19.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0b0efc7279d768db7c74d3d07f0b5c81280d16ae3fb14e9081dc903e8360771"}, + {file = "simplejson-3.19.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0552eb06e7234da892e1d02365cd2b7b2b1f8233aa5aabdb2981587b7cc92ea0"}, + {file = "simplejson-3.19.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5bf6a3b9a7d7191471b464fe38f684df10eb491ec9ea454003edb45a011ab187"}, + {file = "simplejson-3.19.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7017329ca8d4dca94ad5e59f496e5fc77630aecfc39df381ffc1d37fb6b25832"}, + {file = "simplejson-3.19.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:67a20641afebf4cfbcff50061f07daad1eace6e7b31d7622b6fa2c40d43900ba"}, + {file = "simplejson-3.19.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:dd6a7dabcc4c32daf601bc45e01b79175dde4b52548becea4f9545b0a4428169"}, + {file = "simplejson-3.19.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:08f9b443a94e72dd02c87098c96886d35790e79e46b24e67accafbf13b73d43b"}, + {file = "simplejson-3.19.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa97278ae6614346b5ca41a45a911f37a3261b57dbe4a00602048652c862c28b"}, + {file = "simplejson-3.19.3-cp312-cp312-win32.whl", hash = "sha256:ef28c3b328d29b5e2756903aed888960bc5df39b4c2eab157ae212f70ed5bf74"}, + {file = "simplejson-3.19.3-cp312-cp312-win_amd64.whl", hash = "sha256:1e662336db50ad665777e6548b5076329a94a0c3d4a0472971c588b3ef27de3a"}, + {file = "simplejson-3.19.3-py3-none-any.whl", hash = "sha256:49cc4c7b940d43bd12bf87ec63f28cbc4964fc4e12c031cc8cd01650f43eb94e"}, + {file = "simplejson-3.19.3.tar.gz", hash = "sha256:8e086896c36210ab6050f2f9f095a5f1e03c83fa0e7f296d6cba425411364680"}, ] [[package]] @@ -3117,23 +3022,23 @@ files = [ [[package]] name = "sling" -version = "1.2.13" +version = "1.2.22" summary = "Slings data from a source to a target" groups = ["default"] dependencies = [ - "sling-linux-amd64==1.2.13", + "sling-linux-amd64==1.2.22", ] files = [ - {file = "sling-1.2.13.tar.gz", hash = "sha256:40c77d0cf4ade41618b33c46704fbe85f6ebec0c1119decfee43125591d8bce0"}, + {file = "sling-1.2.22.tar.gz", hash = "sha256:13bdf70a0f974b5ad417ec73ff35d39ae5460e48d302e79d0d104208b36d8e1a"}, ] [[package]] name = "sling-linux-amd64" -version = "1.2.13" +version = "1.2.22" summary = "Sling Binary for Linux (AMD64)" groups = ["default"] files = [ - {file = "sling-linux-amd64-1.2.13.tar.gz", hash = "sha256:ee1b674091d50ba3014b05944401df34248cbf28ba6a8074bc1e2fbfee5a8b1d"}, + {file = "sling-linux-amd64-1.2.22.tar.gz", hash = "sha256:c83c3da996bab735e8003e60ea922ab11ada61f2b7960f10df2a40df39e4edb3"}, ] [[package]] @@ -3267,7 +3172,6 @@ files = [ [[package]] name = "starlette" version = "0.41.2" -version = "0.38.1" requires_python = ">=3.8" summary = "The little ASGI library that shines." groups = ["dev"] @@ -3278,8 +3182,6 @@ dependencies = [ files = [ {file = "starlette-0.41.2-py3-none-any.whl", hash = "sha256:fbc189474b4731cf30fcef52f18a8d070e3f3b46c6a04c97579e85e6ffca942d"}, {file = "starlette-0.41.2.tar.gz", hash = "sha256:9834fd799d1a87fd346deb76158668cfa0b0d56f85caefe8268e2d97c3468b62"}, - {file = "starlette-0.38.1-py3-none-any.whl", hash = "sha256:42688a287165bd6acc53068457b5cbf28a3204110b704930411728a633f0f7b8"}, - {file = "starlette-0.38.1.tar.gz", hash = "sha256:890e89d9b1b367bad6fd11a6e191f40d03504df1ff55fd15674a3b35a122990e"}, ] [[package]] @@ -3361,8 +3263,6 @@ requires_python = ">=3.9" summary = "Travel through time in your tests." groups = ["default"] marker = "implementation_name != \"pypy\"" -groups = ["default", "dev"] -marker = "implementation_name != \"pypy\"" dependencies = [ "python-dateutil", ] @@ -3388,19 +3288,19 @@ requires_python = ">=3.8" summary = "A lil' TOML parser" groups = ["default", "dev"] files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] [[package]] name = "tomlkit" -version = "0.13.0" +version = "0.13.2" requires_python = ">=3.8" summary = "Style preserving TOML library" groups = ["default"] files = [ - {file = "tomlkit-0.13.0-py3-none-any.whl", hash = "sha256:7075d3042d03b80f603482d69bf0c8f345c2b30e41699fd8883227f89972b264"}, - {file = "tomlkit-0.13.0.tar.gz", hash = "sha256:08ad192699734149f5b97b45f1f18dad7eb1b6d16bc72ad0c2335772650d7b72"}, + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] [[package]] @@ -3455,19 +3355,19 @@ dependencies = [ "typing-extensions>=3.7.4.3", ] files = [ - {file = "typer-0.12.3-py3-none-any.whl", hash = "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914"}, - {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, + {file = "typer-0.13.0-py3-none-any.whl", hash = "sha256:d85fe0b777b2517cc99c8055ed735452f2659cd45e451507c76f48ce5c1d00e2"}, + {file = "typer-0.13.0.tar.gz", hash = "sha256:f1c7198347939361eec90139ffa0fd8b3df3a2259d5852a0f7400e476d95985c"}, ] [[package]] name = "types-setuptools" -version = "71.1.0.20240723" +version = "75.3.0.20241107" requires_python = ">=3.8" summary = "Typing stubs for setuptools" groups = ["default"] files = [ - {file = "types-setuptools-71.1.0.20240723.tar.gz", hash = "sha256:8a9349038c7e22d88e6c5d9c6705b347b22930424114a452c1712899e85131ff"}, - {file = "types_setuptools-71.1.0.20240723-py3-none-any.whl", hash = "sha256:ac9fc263f59d1e02bca49cb7270a12c47ab80b3b911fb4d92f1fecf978bfe88a"}, + {file = "types-setuptools-75.3.0.20241107.tar.gz", hash = "sha256:f66710e1cd4a936e5fcc12d4e49be1a67c34372cf753e87ebe704426451b4012"}, + {file = "types_setuptools-75.3.0.20241107-py3-none-any.whl", hash = "sha256:bc6de6e2bcb6d610556304d0a69fe4ca208ac4896162647314ecfd9fd73d8550"}, ] [[package]] @@ -3498,6 +3398,7 @@ version = "0.2.5" requires_python = ">=3.8" summary = "pathlib api extended to use fsspec backends" groups = ["default", "dev"] +marker = "python_version >= \"3.12\"" dependencies = [ "fsspec!=2024.3.1,>=2022.1.0", ] @@ -3531,7 +3432,6 @@ files = [ [[package]] name = "uvicorn" version = "0.32.0" -version = "0.30.3" requires_python = ">=3.8" summary = "The lightning-fast ASGI server." groups = ["dev"] @@ -3543,14 +3443,11 @@ dependencies = [ files = [ {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, {file = "uvicorn-0.32.0.tar.gz", hash = "sha256:f78b36b143c16f54ccdb8190d0a26b5f1901fe5a3c777e1ab29f26391af8551e"}, - {file = "uvicorn-0.30.3-py3-none-any.whl", hash = "sha256:94a3608da0e530cea8f69683aa4126364ac18e3826b6630d1a65f4638aade503"}, - {file = "uvicorn-0.30.3.tar.gz", hash = "sha256:0d114d0831ff1adbf231d358cbf42f17333413042552a624ea6a9b4c33dcfd81"}, ] [[package]] name = "uvicorn" version = "0.32.0" -version = "0.30.3" extras = ["standard"] requires_python = ">=3.8" summary = "The lightning-fast ASGI server." @@ -3561,7 +3458,6 @@ dependencies = [ "python-dotenv>=0.13", "pyyaml>=5.1", "uvicorn==0.32.0", - "uvicorn==0.30.3", "uvloop!=0.15.0,!=0.15.1,>=0.14.0; (sys_platform != \"cygwin\" and sys_platform != \"win32\") and platform_python_implementation != \"PyPy\"", "watchfiles>=0.13", "websockets>=10.4", @@ -3569,25 +3465,23 @@ dependencies = [ files = [ {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, {file = "uvicorn-0.32.0.tar.gz", hash = "sha256:f78b36b143c16f54ccdb8190d0a26b5f1901fe5a3c777e1ab29f26391af8551e"}, - {file = "uvicorn-0.30.3-py3-none-any.whl", hash = "sha256:94a3608da0e530cea8f69683aa4126364ac18e3826b6630d1a65f4638aade503"}, - {file = "uvicorn-0.30.3.tar.gz", hash = "sha256:0d114d0831ff1adbf231d358cbf42f17333413042552a624ea6a9b4c33dcfd81"}, ] [[package]] name = "uvloop" -version = "0.20.0" +version = "0.21.0" requires_python = ">=3.8.0" summary = "Fast implementation of asyncio event loop on top of libuv" groups = ["dev"] marker = "(sys_platform != \"cygwin\" and sys_platform != \"win32\") and platform_python_implementation != \"PyPy\"" files = [ - {file = "uvloop-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4b75f2950ddb6feed85336412b9a0c310a2edbcf4cf931aa5cfe29034829676d"}, - {file = "uvloop-0.20.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:77fbc69c287596880ecec2d4c7a62346bef08b6209749bf6ce8c22bbaca0239e"}, - {file = "uvloop-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6462c95f48e2d8d4c993a2950cd3d31ab061864d1c226bbf0ee2f1a8f36674b9"}, - {file = "uvloop-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:649c33034979273fa71aa25d0fe120ad1777c551d8c4cd2c0c9851d88fcb13ab"}, - {file = "uvloop-0.20.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a609780e942d43a275a617c0839d85f95c334bad29c4c0918252085113285b5"}, - {file = "uvloop-0.20.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aea15c78e0d9ad6555ed201344ae36db5c63d428818b4b2a42842b3870127c00"}, - {file = "uvloop-0.20.0.tar.gz", hash = "sha256:4603ca714a754fc8d9b197e325db25b2ea045385e8a3ad05d3463de725fdf469"}, + {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c"}, + {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2"}, + {file = "uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d"}, + {file = "uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc"}, + {file = "uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb"}, + {file = "uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f"}, + {file = "uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3"}, ] [[package]] @@ -3687,13 +3581,13 @@ files = [ [[package]] name = "wheel" -version = "0.43.0" +version = "0.45.0" requires_python = ">=3.8" summary = "A built-package format for Python" groups = ["default"] files = [ - {file = "wheel-0.43.0-py3-none-any.whl", hash = "sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81"}, - {file = "wheel-0.43.0.tar.gz", hash = "sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85"}, + {file = "wheel-0.45.0-py3-none-any.whl", hash = "sha256:52f0baa5e6522155090a09c6bd95718cc46956d1b51d537ea5454249edb671c7"}, + {file = "wheel-0.45.0.tar.gz", hash = "sha256:a57353941a3183b3d5365346b567a260a0602a0f8a635926a7dede41b94c674a"}, ] [[package]] @@ -3711,8 +3605,8 @@ files = [ [[package]] name = "yarl" -version = "1.9.4" -requires_python = ">=3.7" +version = "1.17.1" +requires_python = ">=3.9" summary = "Yet another URL library" groups = ["dev"] dependencies = [ @@ -3750,4 +3644,4 @@ groups = ["default"] files = [ {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, -] \ No newline at end of file +] diff --git a/pyproject.toml b/pyproject.toml index 1dfc718902..786af88248 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "dagster-k8s", "dagster-slack", "dagster-ssh", + "dagster-embedded-elt", "dbt-bigquery>=1.8,<1.9", "dbt-core>=1.8,<1.9", "beautifulsoup4>=4.12.2", @@ -30,8 +31,6 @@ dependencies = [ "scikit-learn>=1.4.0", "tableauserverclient>=0.25", "tenacity>=8.2.3", - "dagster-embedded-elt>=0.23.14", - "psycopg2-binary>=2.9.9", "pypdf>=5.0.0", ] requires-python = ">=3.12,<3.13" diff --git a/requirements.txt b/requirements.txt index 02b7f5aae3..79886416b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ agate==1.9.1 alembic==1.14.0 annotated-types==0.7.0 +astunparse==1.6.3 attrs==24.2.0 avro==1.12.0 babel==2.16.0 @@ -26,6 +27,7 @@ dagster-airbyte==0.25.1 dagster-cloud==1.9.1 dagster-cloud-cli==1.9.1 dagster-dbt==0.25.1 +dagster-embedded-elt==0.25.1 dagster-fivetran==0.25.1 dagster-gcp==0.25.1 dagster-k8s==0.25.1 @@ -42,12 +44,16 @@ dbt-extractor==0.5.1 dbt-semantic-interfaces==0.5.1 deepdiff==7.0.1 defusedxml==0.7.1 +dlt==1.3.0 docstring-parser==0.16 durationpy==0.9 fastavro==1.9.7 filelock==3.16.1 -fsspec==2024.10.0; python_version >= "3.12" +fsspec==2024.10.0 +gitdb==4.0.11 github3-py==4.0.1 +gitpython==3.1.43 +giturlparse==0.12.0 google-api-core[grpc]==2.22.0 google-api-python-client==2.151.0 google-auth==2.36.0 @@ -66,19 +72,23 @@ grpcio==1.67.1 grpcio-health-checking==1.62.3 grpcio-status==1.62.3 gspread==6.1.4 +hexbytes==1.2.1 httplib2==0.22.0 humanfriendly==10.0 +humanize==4.11.0 idna==3.10 importlib-metadata==6.11.0 isodate==0.6.1 jinja2==3.1.4 joblib==1.4.2 +jsonpath-ng==1.7.0 jsonschema==4.23.0 jsonschema-specifications==2024.10.1 kubernetes==31.0.0 ldap3==2.9.1 leather==0.4.0 logbook==1.5.3 +makefun==1.15.6 mako==1.3.6 markdown-it-py==3.0.0 markupsafe==3.0.2 @@ -100,8 +110,11 @@ pandas==2.2.3 paramiko==3.5.0 parsedatetime==2.6 pathspec==0.12.1 +pathvalidate==3.2.1 pendulum==3.0.0 pex==2.24.1 +pluggy==1.5.0 +ply==3.11 prompt-toolkit==3.0.36 proto-plus==1.25.0 protobuf==4.25.5 @@ -131,15 +144,21 @@ questionary==2.0.1 referencing==0.35.1 requests==2.32.3 requests-oauthlib==2.0.0 +requirements-parser==0.11.0 rich==13.9.4 rpds-py==0.21.0 rsa==4.9 scikit-learn==1.5.2 scipy==1.14.1 +semver==3.0.2 setuptools==75.3.0 shellingham==1.5.4 +simplejson==3.19.3 six==1.16.0 slack-sdk==3.33.3 +sling==1.2.22 +sling-linux-amd64==1.2.22 +smmap==5.0.1 soupsieve==2.6 sqlalchemy==2.0.36 sqlglot[rs]==25.29.0 @@ -154,10 +173,12 @@ text-unidecode==1.3 threadpoolctl==3.5.0 time-machine==2.16.0; implementation_name != "pypy" tomli==2.0.2 +tomlkit==0.13.2 toposort==1.10 tqdm==4.67.0 typeguard==4.4.1 typer==0.13.0 +types-setuptools==75.3.0.20241107 typing-extensions==4.12.2 tzdata==2024.2 universal-pathlib==0.2.5; python_version >= "3.12" @@ -166,4 +187,6 @@ urllib3==2.2.3 watchdog==5.0.3 wcwidth==0.2.13 websocket-client==1.8.0 +wheel==0.45.0 +win-precise-time==1.4.2; os_name == "nt" zipp==3.20.2 diff --git a/src/teamster/code_locations/kipptaf/definitions.py b/src/teamster/code_locations/kipptaf/definitions.py index cb1d66407d..b7bce4aa1c 100644 --- a/src/teamster/code_locations/kipptaf/definitions.py +++ b/src/teamster/code_locations/kipptaf/definitions.py @@ -55,6 +55,7 @@ amplify, extracts, deanslist, + illuminate, ldap, overgrad, performance_management, diff --git a/src/teamster/code_locations/kipptaf/resources.py b/src/teamster/code_locations/kipptaf/resources.py index d9ea979faa..2d5de133d5 100644 --- a/src/teamster/code_locations/kipptaf/resources.py +++ b/src/teamster/code_locations/kipptaf/resources.py @@ -169,14 +169,18 @@ SlingConnectionResource( name="ILLUMINATE", type="postgres", + # trunk-ignore-begin(pyright/reportCallIssue) host=EnvVar("SNOWFLAKE_HOST"), user=EnvVar("SNOWFLAKE_USER"), + # trunk-ignore-end(pyright/reportCallIssue) ), SlingConnectionResource( name="BIGQUERY", type="bigquery", + # trunk-ignore-begin(pyright/reportCallIssue) host=EnvVar("SNOWFLAKE_HOST"), user=EnvVar("SNOWFLAKE_USER"), + # trunk-ignore-end(pyright/reportCallIssue) ), ] ) From c39099c01b1ba80ae8639f080ecb3c29e81ddcd1 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Fri, 8 Nov 2024 22:56:09 +0000 Subject: [PATCH 07/20] feat: change table --- .../kipptaf/illuminate/assets.py | 2 +- .../libraries/dlt_sources/.dlt/.sources | 28 ------------------- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 src/teamster/libraries/dlt_sources/.dlt/.sources diff --git a/src/teamster/code_locations/kipptaf/illuminate/assets.py b/src/teamster/code_locations/kipptaf/illuminate/assets.py index 42ffb01d6a..10a8644e02 100644 --- a/src/teamster/code_locations/kipptaf/illuminate/assets.py +++ b/src/teamster/code_locations/kipptaf/illuminate/assets.py @@ -7,7 +7,7 @@ @dlt_assets( dlt_source=sql_database( - schema="dna_assessments", table_names=["assessments", "students_assessments"] + schema="dna_assessments", table_names=["agg_student_responses_standard"] ), dlt_pipeline=pipeline( pipeline_name="illuminate", diff --git a/src/teamster/libraries/dlt_sources/.dlt/.sources b/src/teamster/libraries/dlt_sources/.dlt/.sources deleted file mode 100644 index 36c60faf33..0000000000 --- a/src/teamster/libraries/dlt_sources/.dlt/.sources +++ /dev/null @@ -1,28 +0,0 @@ -engine_version: 1 -sources: - sql_database: - is_dirty: false - last_commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 - last_commit_timestamp: '2024-07-23T09:43:26+02:00' - files: - sql_database/README.md: - commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 - git_sha: dfa4b5e161f059b8399db203801ed0d638b7734c - sha3_256: a21100e31c5f4d514b38ba6229735d3c3a9a7f364ef54d2e9d6068c75c8d8e2b - sql_database/helpers.py: - commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 - git_sha: fa054cc90296a5395718659f110e16e8e4c686a3 - sha3_256: a9ce47a86aff20a50fe24a9c88f728b6f28e77972059ca5a5bde9f3e28d766fa - sql_database/schema_types.py: - commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 - git_sha: 12af2d5f39a8f3a753db66bfca72ed9a3f7a0193 - sha3_256: 5c3b3a1e24ed780ca7a6f2c5404008391d6a9e90c0901ea863648c5516338406 - sql_database/__init__.py: - commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 - git_sha: 93b1a0869ad33971f963567cbfecd0049ab401a4 - sha3_256: 8b7e89b223e128bce22e901d63a1bb93b67124540058e17c57253eded000db09 - sql_database/arrow_helpers.py: - commit_sha: 7e888b30abd8a7985d964e5daceba6f0679b95e4 - git_sha: bf045e5efed225f6525fcbbfd411395437030471 - sha3_256: 6841f763c5775e0c161df98c14270342328111e8d78ef1b0a21c7a6e08605672 - dlt_version_constraint: '>=0.4.11' From aa69efeaaaa07b40894acd6e40d4dd90042c2365 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Fri, 8 Nov 2024 23:55:42 +0000 Subject: [PATCH 08/20] reset dlt project --- .../dlt_sources/sql_database/__init__.py | 204 ---------- .../dlt_sources/sql_database/arrow_helpers.py | 139 ------- .../dlt_sources/sql_database/helpers.py | 284 -------------- .../dlt_sources/sql_database/schema_types.py | 130 ------- .../dlt_sources/sql_database_pipeline.py | 367 ++++++++++++++++++ 5 files changed, 367 insertions(+), 757 deletions(-) delete mode 100644 src/teamster/libraries/dlt_sources/sql_database/__init__.py delete mode 100644 src/teamster/libraries/dlt_sources/sql_database/arrow_helpers.py delete mode 100644 src/teamster/libraries/dlt_sources/sql_database/helpers.py delete mode 100644 src/teamster/libraries/dlt_sources/sql_database/schema_types.py create mode 100644 src/teamster/libraries/dlt_sources/sql_database_pipeline.py diff --git a/src/teamster/libraries/dlt_sources/sql_database/__init__.py b/src/teamster/libraries/dlt_sources/sql_database/__init__.py deleted file mode 100644 index 587e1d1eeb..0000000000 --- a/src/teamster/libraries/dlt_sources/sql_database/__init__.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Source that loads tables form any SQLAlchemy supported database, supports batching requests and incremental loads.""" - -from typing import Any, Callable, Dict, Iterable, List, Optional, Union - -import dlt -from dlt.sources import DltResource -from dlt.sources.credentials import ConnectionStringCredentials -from sqlalchemy import MetaData, Table -from sqlalchemy.engine import Engine - -from .helpers import ( - SqlDatabaseTableConfiguration, - SqlTableResourceConfiguration, - TableBackend, - _detect_precision_hints_deprecated, - engine_from_credentials, - table_rows, -) -from .schema_types import ( - ReflectionLevel, - TTypeAdapter, - get_primary_key, - table_to_columns, -) - - -@dlt.source -def sql_database( - credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value, - schema: Optional[str] = dlt.config.value, - metadata: Optional[MetaData] = None, - table_names: Optional[List[str]] = dlt.config.value, - chunk_size: int = 50000, - backend: TableBackend = "sqlalchemy", - detect_precision_hints: Optional[bool] = False, - reflection_level: Optional[ReflectionLevel] = "full", - defer_table_reflect: Optional[bool] = None, - table_adapter_callback: Callable[[Table], None] = None, - backend_kwargs: Dict[str, Any] = None, - include_views: bool = False, - type_adapter_callback: Optional[TTypeAdapter] = None, -) -> Iterable[DltResource]: - """ - A dlt source which loads data from an SQL database using SQLAlchemy. - Resources are automatically created for each table in the schema or from the given list of tables. - - Args: - credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `sqlalchemy.Engine` instance. - schema (Optional[str]): Name of the database schema to load (if different from default). - metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. `schema` argument is ignored when this is used. - table_names (Optional[List[str]]): A list of table names to load. By default, all tables in the schema are loaded. - chunk_size (int): Number of rows yielded in one batch. SQL Alchemy will create additional internal rows buffer twice the chunk size. - backend (TableBackend): Type of backend to generate table data. One of: "sqlalchemy", "pyarrow", "pandas" and "connectorx". - "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames. - "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types, - "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself. - detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables. - This is disabled by default. - reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema. - "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data. - "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option. - "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types. - defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Requires table_names to be explicitly passed. - Enable this option when running on Airflow. Available on dlt 0.4.4 and later. - table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected. - backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx. - include_views (bool): Reflect views as well as tables. Note view names included in `table_names` are always included regardless of this setting. - type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns. - Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data) - Returns: - - Iterable[DltResource]: A list of DLT resources for each table to be loaded. - """ - # detect precision hints is deprecated - _detect_precision_hints_deprecated(detect_precision_hints) - - if detect_precision_hints: - reflection_level = "full_with_precision" - else: - reflection_level = reflection_level or "minimal" - - # set up alchemy engine - engine = engine_from_credentials(credentials) - engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size) - metadata = metadata or MetaData(schema=schema) - - # use provided tables or all tables - if table_names: - tables = [ - Table(name, metadata, autoload_with=None if defer_table_reflect else engine) - for name in table_names - ] - else: - if defer_table_reflect: - raise ValueError("You must pass table names to defer table reflection") - metadata.reflect(bind=engine, views=include_views) - tables = list(metadata.tables.values()) - - for table in tables: - if table_adapter_callback and not defer_table_reflect: - table_adapter_callback(table) - - yield dlt.resource( - table_rows, - name=table.name, - primary_key=get_primary_key(table), - spec=SqlDatabaseTableConfiguration, - columns=table_to_columns(table, reflection_level, type_adapter_callback), - )( - engine, - table, - chunk_size, - backend, - reflection_level=reflection_level, - defer_table_reflect=defer_table_reflect, - table_adapter_callback=table_adapter_callback, - backend_kwargs=backend_kwargs, - type_adapter_callback=type_adapter_callback, - ) - - -@dlt.resource( - name=lambda args: args["table"], standalone=True, spec=SqlTableResourceConfiguration -) -def sql_table( - credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value, - table: str = dlt.config.value, - schema: Optional[str] = dlt.config.value, - metadata: Optional[MetaData] = None, - incremental: Optional[dlt.sources.incremental[Any]] = None, - chunk_size: int = 50000, - backend: TableBackend = "sqlalchemy", - detect_precision_hints: Optional[bool] = None, - reflection_level: Optional[ReflectionLevel] = "full", - defer_table_reflect: Optional[bool] = None, - table_adapter_callback: Callable[[Table], None] = None, - backend_kwargs: Dict[str, Any] = None, - type_adapter_callback: Optional[TTypeAdapter] = None, -) -> DltResource: - """ - A dlt resource which loads data from an SQL database table using SQLAlchemy. - - Args: - credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `Engine` instance representing the database connection. - table (str): Name of the table or view to load. - schema (Optional[str]): Optional name of the schema the table belongs to. - metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. If provided, the `schema` argument is ignored. - incremental (Optional[dlt.sources.incremental[Any]]): Option to enable incremental loading for the table. - E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))` - chunk_size (int): Number of rows yielded in one batch. SQL Alchemy will create additional internal rows buffer twice the chunk size. - backend (TableBackend): Type of backend to generate table data. One of: "sqlalchemy", "pyarrow", "pandas" and "connectorx". - "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames. - "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types, - "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself. - reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema. - "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data. - "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option. - "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types. - detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables. - This is disabled by default. - defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Enable this option when running on Airflow. Available - on dlt 0.4.4 and later - table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected. - backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx. - type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns. - Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data) - - Returns: - DltResource: The dlt resource for loading data from the SQL database table. - """ - _detect_precision_hints_deprecated(detect_precision_hints) - - if detect_precision_hints: - reflection_level = "full_with_precision" - else: - reflection_level = reflection_level or "minimal" - - engine = engine_from_credentials(credentials, may_dispose_after_use=True) - engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size) - metadata = metadata or MetaData(schema=schema) - - table_obj = Table( - table, metadata, autoload_with=None if defer_table_reflect else engine - ) - if table_adapter_callback and not defer_table_reflect: - table_adapter_callback(table_obj) - - return dlt.resource( - table_rows, - name=table_obj.name, - primary_key=get_primary_key(table_obj), - columns=table_to_columns(table_obj, reflection_level, type_adapter_callback), - )( - engine, - table_obj, - chunk_size, - backend, - incremental=incremental, - reflection_level=reflection_level, - defer_table_reflect=defer_table_reflect, - table_adapter_callback=table_adapter_callback, - backend_kwargs=backend_kwargs, - type_adapter_callback=type_adapter_callback, - ) diff --git a/src/teamster/libraries/dlt_sources/sql_database/arrow_helpers.py b/src/teamster/libraries/dlt_sources/sql_database/arrow_helpers.py deleted file mode 100644 index 62c06bc838..0000000000 --- a/src/teamster/libraries/dlt_sources/sql_database/arrow_helpers.py +++ /dev/null @@ -1,139 +0,0 @@ -from typing import Any, Optional, Sequence - -from dlt.common import logger -from dlt.common.configuration import with_config -from dlt.common.destination import DestinationCapabilitiesContext -from dlt.common.json import custom_encode, map_nested_in_place -from dlt.common.schema.typing import TTableSchemaColumns - -from .schema_types import RowAny - - -@with_config -def columns_to_arrow( - columns_schema: TTableSchemaColumns, - caps: DestinationCapabilitiesContext = None, - tz: str = "UTC", -) -> Any: - """Converts `column_schema` to arrow schema using `caps` and `tz`. `caps` are injected from the container - which - is always the case if run within the pipeline. This will generate arrow schema compatible with the destination. - Otherwise generic capabilities are used - """ - from dlt.common.destination.capabilities import DestinationCapabilitiesContext - from dlt.common.libs.pyarrow import get_py_arrow_datatype - from dlt.common.libs.pyarrow import pyarrow as pa - - return pa.schema( - [ - pa.field( - name, - get_py_arrow_datatype( - schema_item, - caps or DestinationCapabilitiesContext.generic_capabilities(), - tz, - ), - nullable=schema_item.get("nullable", True), - ) - for name, schema_item in columns_schema.items() - if schema_item.get("data_type") is not None - ] - ) - - -def row_tuples_to_arrow( - rows: Sequence[RowAny], columns: TTableSchemaColumns, tz: str -) -> Any: - """Converts the rows to an arrow table using the columns schema. - Columns missing `data_type` will be inferred from the row data. - Columns with object types not supported by arrow are excluded from the resulting table. - """ - import numpy as np - from dlt.common.libs.pyarrow import pyarrow as pa - - try: - from pandas._libs import lib - - pivoted_rows = lib.to_object_array_tuples(rows).T # type: ignore[attr-defined] - except ImportError: - logger.info( - "Pandas not installed, reverting to numpy.asarray to create a table which is slower" - ) - pivoted_rows = np.asarray(rows, dtype="object", order="k").T # type: ignore[call-overload] - - columnar = { - col: dat.ravel() - for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns))) - } - columnar_known_types = { - col["name"]: columnar[col["name"]] - for col in columns.values() - if col.get("data_type") is not None - } - columnar_unknown_types = { - col["name"]: columnar[col["name"]] - for col in columns.values() - if col.get("data_type") is None - } - - arrow_schema = columns_to_arrow(columns, tz=tz) - - for idx in range(0, len(arrow_schema.names)): - field = arrow_schema.field(idx) - py_type = type(rows[0][idx]) - # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects - if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)): - logger.warning( - f"Field {field.name} was reflected as decimal type, but rows contains {py_type.__name__}. Additional cast is required which may slow down arrow table generation." - ) - float_array = pa.array(columnar_known_types[field.name], type=pa.float64()) - columnar_known_types[field.name] = float_array.cast(field.type, safe=False) - - # If there are unknown type columns, first create a table to infer their types - if columnar_unknown_types: - new_schema_fields = [] - for key in list(columnar_unknown_types): - arrow_col: Optional[pa.Array] = None - try: - arrow_col = pa.array(columnar_unknown_types[key]) - if pa.types.is_null(arrow_col.type): - logger.warning( - f"Column {key} contains only NULL values and data type could not be inferred. This column is removed from a arrow table" - ) - continue - - except pa.ArrowInvalid as e: - # Try coercing types not supported by arrow to a json friendly format - # E.g. dataclasses -> dict, UUID -> str - try: - arrow_col = pa.array( - map_nested_in_place( - custom_encode, list(columnar_unknown_types[key]) - ) - ) - logger.warning( - f"Column {key} contains a data type which is not supported by pyarrow and got converted into {arrow_col.type}. This slows down arrow table generation." - ) - except (pa.ArrowInvalid, TypeError): - logger.warning( - f"Column {key} contains a data type which is not supported by pyarrow. This column will be ignored. Error: {e}" - ) - if arrow_col is not None: - columnar_known_types[key] = arrow_col - new_schema_fields.append( - pa.field( - key, - arrow_col.type, - nullable=columns[key]["nullable"], - ) - ) - - # New schema - column_order = {name: idx for idx, name in enumerate(columns)} - arrow_schema = pa.schema( - sorted( - list(arrow_schema) + new_schema_fields, - key=lambda x: column_order[x.name], - ) - ) - - return pa.Table.from_pydict(columnar_known_types, schema=arrow_schema) diff --git a/src/teamster/libraries/dlt_sources/sql_database/helpers.py b/src/teamster/libraries/dlt_sources/sql_database/helpers.py deleted file mode 100644 index f2c9a1a4ab..0000000000 --- a/src/teamster/libraries/dlt_sources/sql_database/helpers.py +++ /dev/null @@ -1,284 +0,0 @@ -"""SQL database source helpers""" - -import operator -import warnings -from typing import Any, Callable, Dict, Iterator, Literal, Optional, Union - -import dlt -from dlt.common.configuration.specs import BaseConfiguration, configspec -from dlt.common.exceptions import MissingDependencyException -from dlt.common.schema import TTableSchemaColumns -from dlt.common.typing import TDataItem, TSortOrder -from dlt.sources.credentials import ConnectionStringCredentials -from sqlalchemy import Table, create_engine -from sqlalchemy.engine import Engine -from sqlalchemy.exc import CompileError - -from .arrow_helpers import row_tuples_to_arrow -from .schema_types import ( - ReflectionLevel, - SelectAny, - TTypeAdapter, - get_primary_key, - table_to_columns, -) - -TableBackend = Literal["sqlalchemy", "pyarrow", "pandas", "connectorx"] - - -class TableLoader: - def __init__( - self, - engine: Engine, - backend: TableBackend, - table: Table, - columns: TTableSchemaColumns, - chunk_size: int = 1000, - incremental: Optional[dlt.sources.incremental[Any]] = None, - ) -> None: - self.engine = engine - self.backend = backend - self.table = table - self.columns = columns - self.chunk_size = chunk_size - self.incremental = incremental - if incremental: - try: - self.cursor_column = table.c[incremental.cursor_path] - except KeyError as e: - raise KeyError( - f"Cursor column '{incremental.cursor_path}' does not exist in table '{table.name}'" - ) from e - self.last_value = incremental.last_value - self.end_value = incremental.end_value - self.row_order: TSortOrder = self.incremental.row_order - else: - self.cursor_column = None - self.last_value = None - self.end_value = None - self.row_order = None - - def make_query(self) -> SelectAny: - table = self.table - query = table.select() - if not self.incremental: - return query - last_value_func = self.incremental.last_value_func - - # generate where - if ( - last_value_func is max - ): # Query ordered and filtered according to last_value function - filter_op = operator.ge - filter_op_end = operator.lt - elif last_value_func is min: - filter_op = operator.le - filter_op_end = operator.gt - else: # Custom last_value, load everything and let incremental handle filtering - return query - - if self.last_value is not None: - query = query.where(filter_op(self.cursor_column, self.last_value)) - if self.end_value is not None: - query = query.where(filter_op_end(self.cursor_column, self.end_value)) - - # generate order by from declared row order - order_by = None - if (self.row_order == "asc" and last_value_func is max) or ( - self.row_order == "desc" and last_value_func is min - ): - order_by = self.cursor_column.asc() - elif (self.row_order == "asc" and last_value_func is min) or ( - self.row_order == "desc" and last_value_func is max - ): - order_by = self.cursor_column.desc() - if order_by is not None: - query = query.order_by(order_by) - - return query - - def load_rows(self, backend_kwargs: Dict[str, Any] = None) -> Iterator[TDataItem]: - # make copy of kwargs - backend_kwargs = dict(backend_kwargs or {}) - query = self.make_query() - if self.backend == "connectorx": - yield from self._load_rows_connectorx(query, backend_kwargs) - else: - yield from self._load_rows(query, backend_kwargs) - - def _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem: - with self.engine.connect() as conn: - result = conn.execution_options(yield_per=self.chunk_size).execute(query) - # NOTE: cursor returns not normalized column names! may be quite useful in case of Oracle dialect - # that normalizes columns - # columns = [c[0] for c in result.cursor.description] - columns = list(result.keys()) - for partition in result.partitions(size=self.chunk_size): - if self.backend == "sqlalchemy": - yield [dict(row._mapping) for row in partition] - elif self.backend == "pandas": - from dlt.common.libs.pandas_sql import _wrap_result - - df = _wrap_result( - partition, - columns, - **{"dtype_backend": "pyarrow", **backend_kwargs}, - ) - yield df - elif self.backend == "pyarrow": - yield row_tuples_to_arrow( - partition, self.columns, tz=backend_kwargs.get("tz", "UTC") - ) - - def _load_rows_connectorx( - self, query: SelectAny, backend_kwargs: Dict[str, Any] - ) -> Iterator[TDataItem]: - try: - import connectorx as cx # type: ignore - except ImportError: - # trunk-ignore(ruff/B904) - raise MissingDependencyException( - "Connector X table backend", ["connectorx"] - ) - - # default settings - backend_kwargs = { - "return_type": "arrow2", - "protocol": "binary", - **backend_kwargs, - } - conn = backend_kwargs.pop( - "conn", - self.engine.url._replace( - drivername=self.engine.url.get_backend_name() - ).render_as_string(hide_password=False), - ) - try: - query_str = str( - query.compile(self.engine, compile_kwargs={"literal_binds": True}) - ) - except CompileError as ex: - raise NotImplementedError( - f"Query for table {self.table.name} could not be compiled to string to execute it on ConnectorX. If you are on SQLAlchemy 1.4.x the causing exception is due to literals that cannot be rendered, upgrade to 2.x: {str(ex)}" - ) from ex - df = cx.read_sql(conn, query_str, **backend_kwargs) - yield df - - -def table_rows( - engine: Engine, - table: Table, - chunk_size: int, - backend: TableBackend, - incremental: Optional[dlt.sources.incremental[Any]] = None, - defer_table_reflect: bool = False, - table_adapter_callback: Callable[[Table], None] = None, - reflection_level: ReflectionLevel = "minimal", - backend_kwargs: Dict[str, Any] = None, - type_adapter_callback: Optional[TTypeAdapter] = None, -) -> Iterator[TDataItem]: - columns: TTableSchemaColumns = None - if defer_table_reflect: - table = Table( - table.name, table.metadata, autoload_with=engine, extend_existing=True - ) - if table_adapter_callback: - table_adapter_callback(table) - columns = table_to_columns(table, reflection_level, type_adapter_callback) - - # set the primary_key in the incremental - if incremental and incremental.primary_key is None: - primary_key = get_primary_key(table) - if primary_key is not None: - incremental.primary_key = primary_key - # yield empty record to set hints - yield dlt.mark.with_hints( - [], - dlt.mark.make_hints( - primary_key=get_primary_key(table), - columns=columns, - ), - ) - else: - # table was already reflected - columns = table_to_columns(table, reflection_level, type_adapter_callback) - - loader = TableLoader( - engine, backend, table, columns, incremental=incremental, chunk_size=chunk_size - ) - try: - yield from loader.load_rows(backend_kwargs) - finally: - # dispose the engine if created for this particular table - # NOTE: database wide engines are not disposed, not externally provided - if getattr(engine, "may_dispose_after_use", False): - engine.dispose() - - -def engine_from_credentials( - credentials: Union[ConnectionStringCredentials, Engine, str], - may_dispose_after_use: bool = False, - **backend_kwargs: Any, -) -> Engine: - if isinstance(credentials, Engine): - return credentials - if isinstance(credentials, ConnectionStringCredentials): - credentials = credentials.to_native_representation() - engine = create_engine(credentials, **backend_kwargs) - setattr(engine, "may_dispose_after_use", may_dispose_after_use) # noqa - return engine - - -def unwrap_json_connector_x(field: str) -> TDataItem: - """Creates a transform function to be added with `add_map` that will unwrap JSON columns - ingested via connectorx. Such columns are additionally quoted and translate SQL NULL to json "null" - """ - import pyarrow as pa - import pyarrow.compute as pc - - def _unwrap(table: TDataItem) -> TDataItem: - col_index = table.column_names.index(field) - # remove quotes - column = pc.replace_substring_regex(table[field], '"(.*)"', "\\1") - # convert json null to null - column = pc.replace_with_mask( - column, - pc.equal(column, "null").combine_chunks(), - pa.scalar(None, pa.large_string()), - ) - return table.set_column(col_index, table.schema.field(col_index), column) - - return _unwrap - - -def _detect_precision_hints_deprecated(value: Optional[bool]) -> None: - if value is None: - return - - msg = "`detect_precision_hints` argument is deprecated and will be removed in a future release. " - if value: - msg += "Use `reflection_level='full_with_precision'` which has the same effect instead." - - # trunk-ignore(ruff/B028) - warnings.warn( - msg, - DeprecationWarning, - ) - - -@configspec -class SqlDatabaseTableConfiguration(BaseConfiguration): - incremental: Optional[dlt.sources.incremental] = None # type: ignore[type-arg] - - -@configspec -class SqlTableResourceConfiguration(BaseConfiguration): - credentials: Union[ConnectionStringCredentials, Engine, str] = None - table: str = None - schema: Optional[str] = None - incremental: Optional[dlt.sources.incremental] = None # type: ignore[type-arg] - chunk_size: int = 50000 - backend: TableBackend = "sqlalchemy" - detect_precision_hints: Optional[bool] = None - defer_table_reflect: Optional[bool] = False - reflection_level: Optional[ReflectionLevel] = "full" diff --git a/src/teamster/libraries/dlt_sources/sql_database/schema_types.py b/src/teamster/libraries/dlt_sources/sql_database/schema_types.py deleted file mode 100644 index ecf540b232..0000000000 --- a/src/teamster/libraries/dlt_sources/sql_database/schema_types.py +++ /dev/null @@ -1,130 +0,0 @@ -from typing import TYPE_CHECKING, Any, Callable, List, Literal, Optional, Type, Union - -from dlt.common import logger -from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns -from sqlalchemy import Column, Table -from sqlalchemy.engine import Row -from sqlalchemy.sql import Select, sqltypes -from sqlalchemy.sql.sqltypes import TypeEngine -from typing_extensions import TypeAlias - -ReflectionLevel = Literal["minimal", "full", "full_with_precision"] - - -# optionally create generics with any so they can be imported by dlt importer -if TYPE_CHECKING: - SelectAny: TypeAlias = Select[Any] - ColumnAny: TypeAlias = Column[Any] - RowAny: TypeAlias = Row[Any] - TypeEngineAny = TypeEngine[Any] -else: - SelectAny: TypeAlias = Type[Any] - ColumnAny: TypeAlias = Type[Any] - RowAny: TypeAlias = Type[Any] - TypeEngineAny = Type[Any] - - -TTypeAdapter = Callable[ - [TypeEngineAny], Optional[Union[TypeEngineAny, Type[TypeEngineAny]]] -] - - -def sqla_col_to_column_schema( - sql_col: ColumnAny, - reflection_level: ReflectionLevel, - type_adapter_callback: Optional[TTypeAdapter] = None, -) -> Optional[TColumnSchema]: - """Infer dlt schema column type from an sqlalchemy type. - - If `add_precision` is set, precision and scale is inferred from that types that support it, - such as numeric, varchar, int, bigint. Numeric (decimal) types have always precision added. - """ - col: TColumnSchema = { - "name": sql_col.name, - "nullable": sql_col.nullable, - } - if reflection_level == "minimal": - return col - - sql_t = sql_col.type - - if type_adapter_callback: - sql_t = type_adapter_callback(sql_t) # type: ignore[assignment] - # Check if sqla type class rather than instance is returned - if sql_t is not None and isinstance(sql_t, type): - sql_t = sql_t() - - if sql_t is None: - # Column ignored by callback - return col - - add_precision = reflection_level == "full_with_precision" - - if isinstance(sql_t, sqltypes.Numeric): - # check for Numeric type first and integer later, some numeric types (ie. Oracle) - # derive from both - # all Numeric types that are returned as floats will assume "double" type - # and returned as decimals will assume "decimal" type - if sql_t.asdecimal is False: - col["data_type"] = "double" - else: - col["data_type"] = "decimal" - if sql_t.precision is not None: - col["precision"] = sql_t.precision - # must have a precision for any meaningful scale - if sql_t.scale is not None: - col["scale"] = sql_t.scale - elif sql_t.decimal_return_scale is not None: - col["scale"] = sql_t.decimal_return_scale - elif isinstance(sql_t, sqltypes.SmallInteger): - col["data_type"] = "bigint" - if add_precision: - col["precision"] = 32 - elif isinstance(sql_t, sqltypes.Integer): - col["data_type"] = "bigint" - elif isinstance(sql_t, sqltypes.String): - col["data_type"] = "text" - if add_precision and sql_t.length: - col["precision"] = sql_t.length - elif isinstance(sql_t, sqltypes._Binary): - col["data_type"] = "binary" - if add_precision and sql_t.length: - col["precision"] = sql_t.length - elif isinstance(sql_t, sqltypes.DateTime): - col["data_type"] = "timestamp" - elif isinstance(sql_t, sqltypes.Date): - col["data_type"] = "date" - elif isinstance(sql_t, sqltypes.Time): - col["data_type"] = "time" - elif isinstance(sql_t, sqltypes.JSON): - col["data_type"] = "complex" - elif isinstance(sql_t, sqltypes.Boolean): - col["data_type"] = "bool" - else: - logger.warning( - f"A column with name {sql_col.name} contains unknown data type {sql_t} which cannot be mapped to `dlt` data type. When using sqlalchemy backend such data will be passed to the normalizer. In case of `pyarrow` and `pandas` backend, data types are detected from numpy ndarrays. In case of other backends, the behavior is backend-specific." - ) - - return {key: value for key, value in col.items() if value is not None} # type: ignore[return-value] - - -def get_primary_key(table: Table) -> Optional[List[str]]: - """Create primary key or return None if no key defined""" - primary_key = [c.name for c in table.primary_key] - return primary_key if len(primary_key) > 0 else None - - -def table_to_columns( - table: Table, - reflection_level: ReflectionLevel = "full", - type_conversion_fallback: Optional[TTypeAdapter] = None, -) -> TTableSchemaColumns: - """Convert an sqlalchemy table to a dlt table schema.""" - return { - col["name"]: col - for col in ( - sqla_col_to_column_schema(c, reflection_level, type_conversion_fallback) - for c in table.columns - ) - if col is not None - } diff --git a/src/teamster/libraries/dlt_sources/sql_database_pipeline.py b/src/teamster/libraries/dlt_sources/sql_database_pipeline.py new file mode 100644 index 0000000000..9ca98b2eff --- /dev/null +++ b/src/teamster/libraries/dlt_sources/sql_database_pipeline.py @@ -0,0 +1,367 @@ +# flake8: noqa +import os +from typing import Any + +import dlt +import humanize +import sqlalchemy as sa +from dlt.common import pendulum +from dlt.sources.credentials import ConnectionStringCredentials +from dlt.sources.sql_database import Table, sql_database, sql_table +from sqlalchemy.sql.sqltypes import TypeEngine + + +def load_select_tables_from_database() -> None: + """Use the sql_database source to reflect an entire database schema and load select tables from it. + + This example sources data from the public Rfam MySQL database. + """ + # Create a pipeline + pipeline = dlt.pipeline( + pipeline_name="rfam", destination="bigquery", dataset_name="rfam_data" + ) + + # Credentials for the sample database. + # Note: It is recommended to configure credentials in `.dlt/secrets.toml` under `sources.sql_database.credentials` + credentials = ConnectionStringCredentials( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam" + ) + # To pass the credentials from `secrets.toml`, comment out the above credentials. + # And the credentials will be automatically read from `secrets.toml`. + + # Configure the source to load a few select tables incrementally + source_1 = sql_database(credentials).with_resources("family", "clan") + + # Add incremental config to the resources. "updated" is a timestamp column in these tables that gets used as a cursor + source_1.family.apply_hints(incremental=dlt.sources.incremental("updated")) + source_1.clan.apply_hints(incremental=dlt.sources.incremental("updated")) + + # Run the pipeline. The merge write disposition merges existing rows in the destination by primary key + info = pipeline.run(source_1, write_disposition="merge") + print(info) + + # Load some other tables with replace write disposition. This overwrites the existing tables in destination + source_2 = sql_database(credentials).with_resources("features", "author") + info = pipeline.run(source_2, write_disposition="replace") + print(info) + + # Load a table incrementally with append write disposition + # this is good when a table only has new rows inserted, but not updated + source_3 = sql_database(credentials).with_resources("genome") + source_3.genome.apply_hints(incremental=dlt.sources.incremental("created")) + + info = pipeline.run(source_3, write_disposition="append") + print(info) + + +def load_entire_database() -> None: + """Use the sql_database source to completely load all tables in a database""" + pipeline = dlt.pipeline( + pipeline_name="rfam", destination="bigquery", dataset_name="rfam_data" + ) + + # By default the sql_database source reflects all tables in the schema + # The database credentials are sourced from the `.dlt/secrets.toml` configuration + source = sql_database() + + # Run the pipeline. For a large db this may take a while + info = pipeline.run(source, write_disposition="replace") + print( + humanize.precisedelta( + pipeline.last_trace.finished_at - pipeline.last_trace.started_at + ) + ) + print(info) + + +def load_standalone_table_resource() -> None: + """Load a few known tables with the standalone sql_table resource, request full schema and deferred + table reflection""" + pipeline = dlt.pipeline( + pipeline_name="rfam_database", + destination="bigquery", + dataset_name="rfam_data", + full_refresh=True, + ) + + # Load a table incrementally starting at a given date + # Adding incremental via argument like this makes extraction more efficient + # as only rows newer than the start date are fetched from the table + # we also use `detect_precision_hints` to get detailed column schema + # and defer_table_reflect to reflect schema only during execution + family = sql_table( + credentials=ConnectionStringCredentials( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam" + ), + table="family", + incremental=dlt.sources.incremental( + "updated", + ), + reflection_level="full_with_precision", + defer_table_reflect=True, + ) + # columns will be empty here due to defer_table_reflect set to True + print(family.compute_table_schema()) + + # Load all data from another table + genome = sql_table( + credentials="mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam", + table="genome", + reflection_level="full_with_precision", + defer_table_reflect=True, + ) + + # Run the resources together + info = pipeline.extract([family, genome], write_disposition="merge") + print(info) + # Show inferred columns + print(pipeline.default_schema.to_pretty_yaml()) + + +def select_columns() -> None: + """Uses table adapter callback to modify list of columns to be selected""" + pipeline = dlt.pipeline( + pipeline_name="rfam_database", + destination="bigquery", + dataset_name="rfam_data_cols", + full_refresh=True, + ) + + def table_adapter(table: Table) -> None: + print(table.name) + if table.name == "family": + # this is SqlAlchemy table. _columns are writable + # let's drop updated column + table._columns.remove(table.columns["updated"]) # type: ignore + + family = sql_table( + credentials="mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam", + table="family", + chunk_size=10, + reflection_level="full_with_precision", + table_adapter_callback=table_adapter, + ) + + # also we do not want the whole table, so we add limit to get just one chunk (10 records) + pipeline.run(family.add_limit(1)) + # only 10 rows + print(pipeline.last_trace.last_normalize_info) + # no "updated" column in "family" table + print(pipeline.default_schema.to_pretty_yaml()) + + +def select_with_end_value_and_row_order() -> None: + """Gets data from a table withing a specified range and sorts rows descending""" + pipeline = dlt.pipeline( + pipeline_name="rfam_database", + destination="bigquery", + dataset_name="rfam_data", + full_refresh=True, + ) + + # gets data from this range + start_date = pendulum.now().subtract(years=1) + end_date = pendulum.now() + + family = sql_table( + credentials="mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam", + table="family", + incremental=dlt.sources.incremental( # declares desc row order + "updated", initial_value=start_date, end_value=end_date, row_order="desc" + ), + chunk_size=10, + ) + # also we do not want the whole table, so we add limit to get just one chunk (10 records) + pipeline.run(family.add_limit(1)) + # only 10 rows + print(pipeline.last_trace.last_normalize_info) + + +def my_sql_via_pyarrow() -> None: + """Uses pyarrow backend to load tables from mysql""" + + # uncomment line below to get load_id into your data (slows pyarrow loading down) + # dlt.config["normalize.parquet_normalizer.add_dlt_load_id"] = True + + # Create a pipeline + pipeline = dlt.pipeline( + pipeline_name="rfam_cx", + destination="bigquery", + dataset_name="rfam_data_arrow_4", + ) + + def _double_as_decimal_adapter(table: sa.Table) -> None: + """Return double as double, not decimals, only works if you are using sqlalchemy 2.0""" + for column in table.columns.values(): + if hasattr(sa, "Double") and isinstance(column.type, sa.Double): + column.type.asdecimal = False + + sql_alchemy_source = sql_database( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam?&binary_prefix=true", + backend="pyarrow", + table_adapter_callback=_double_as_decimal_adapter, + ).with_resources("family", "genome") + + info = pipeline.run(sql_alchemy_source) + print(info) + + +def create_unsw_flow() -> None: + """Uploads UNSW_Flow dataset to postgres via csv stream skipping dlt normalizer. + You need to download the dataset from https://github.com/rdpahalavan/nids-datasets + """ + from pyarrow.parquet import ParquetFile + + # from dlt.destinations import postgres + # use those config to get 3x speedup on parallelism + # [sources.data_writer] + # file_max_bytes=3000000 + # buffer_max_items=200000 + # [normalize] + # workers=3 + + data_iter = ParquetFile("UNSW-NB15/Network-Flows/UNSW_Flow.parquet").iter_batches( + batch_size=128 * 1024 + ) + + pipeline = dlt.pipeline( + pipeline_name="unsw_upload", + # destination=postgres("postgres://loader:loader@localhost:5432/dlt_data"), + destination="bigquery", + progress="log", + ) + pipeline.run( + data_iter, + dataset_name="speed_test", + table_name="unsw_flow_7", + loader_file_format="csv", + ) + + +def test_connectorx_speed() -> None: + """Uses unsw_flow dataset (~2mln rows, 25+ columns) to test connectorx speed""" + import os + + # from dlt.destinations import filesystem + + unsw_table = sql_table( + "postgresql://loader:loader@localhost:5432/dlt_data", + "unsw_flow_7", + "speed_test", + # this is ignored by connectorx + chunk_size=100000, + backend="connectorx", + # keep source data types + reflection_level="full_with_precision", + # just to demonstrate how to setup a separate connection string for connectorx + backend_kwargs={"conn": "postgresql://loader:loader@localhost:5432/dlt_data"}, + ) + + pipeline = dlt.pipeline( + pipeline_name="unsw_download", + destination="bigquery", + # destination=filesystem(os.path.abspath("../_storage/unsw")), + progress="log", + full_refresh=True, + ) + + info = pipeline.run( + unsw_table, + dataset_name="speed_test", + table_name="unsw_flow", + loader_file_format="parquet", + ) + print(info) + + +def test_pandas_backend_verbatim_decimals() -> None: + pipeline = dlt.pipeline( + pipeline_name="rfam_cx", + destination="bigquery", + dataset_name="rfam_data_pandas_2", + ) + + def _double_as_decimal_adapter(table: sa.Table) -> None: + """Emits decimals instead of floats.""" + for column in table.columns.values(): + if isinstance(column.type, sa.Float): + column.type.asdecimal = True + + sql_alchemy_source = sql_database( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam?&binary_prefix=true", + backend="pandas", + table_adapter_callback=_double_as_decimal_adapter, + chunk_size=100000, + # set coerce_float to False to represent them as string + backend_kwargs={"coerce_float": False, "dtype_backend": "numpy_nullable"}, + # preserve full typing info. this will parse + reflection_level="full_with_precision", + ).with_resources("family", "genome") + + info = pipeline.run(sql_alchemy_source) + print(info) + + +def use_type_adapter() -> None: + """Example use of type adapter to coerce unknown data types""" + pipeline = dlt.pipeline( + pipeline_name="dummy", + destination="bigquery", + dataset_name="dummy", + ) + + def type_adapter(sql_type: Any) -> Any: + if isinstance(sql_type, sa.ARRAY): + return sa.JSON() # Load arrays as JSON + return sql_type + + sql_alchemy_source = sql_database( + "postgresql://loader:loader@localhost:5432/dlt_data", + backend="pyarrow", + type_adapter_callback=type_adapter, + reflection_level="full_with_precision", + ).with_resources("table_with_array_column") + + info = pipeline.run(sql_alchemy_source) + print(info) + + +def specify_columns_to_load() -> None: + """Run the SQL database source with a subset of table columns loaded""" + pipeline = dlt.pipeline( + pipeline_name="dummy", + destination="bigquery", + dataset_name="dummy", + ) + + # Columns can be specified per table in env var (json array) or in `.dlt/config.toml` + os.environ["SOURCES__SQL_DATABASE__FAMILY__INCLUDED_COLUMNS"] = ( + '["rfam_acc", "description"]' + ) + + sql_alchemy_source = sql_database( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam?&binary_prefix=true", + backend="pyarrow", + reflection_level="full_with_precision", + ).with_resources("family", "genome") + + info = pipeline.run(sql_alchemy_source) + print(info) + + +if __name__ == "__main__": + # Load selected tables with different settings + # load_select_tables_from_database() + + # load a table and select columns + # select_columns() + + # load_entire_database() + # select_with_end_value_and_row_order() + + # Load tables with the standalone table resource + load_standalone_table_resource() + + # Load all tables from the database. + # Warning: The sample database is very large + # load_entire_database() From 033918f41e876758f39fe586a7aa8c546489f3fc Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:27:22 +0000 Subject: [PATCH 09/20] feat: dlt --- pdm.lock | 70 ++++++++++++++++++- pyproject.toml | 3 +- requirements.txt | 3 + .../code_locations/kipptaf/definitions.py | 2 + .../kipptaf/illuminate/assets.py | 3 +- .../code_locations/kipptaf/resources.py | 22 ------ 6 files changed, 77 insertions(+), 26 deletions(-) diff --git a/pdm.lock b/pdm.lock index 7f94ac56e0..d7f940f80e 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:7550121135ad9170f6eba2cc38681d7e0a4b54cf75791735336b40c6bd48bf77" +content_hash = "sha256:b1fdabd8bc2618efb683c86529aff77ccf453021e98492ec4624dba18dc3d782" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -2399,6 +2399,74 @@ files = [ {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, ] +[[package]] +name = "psycopg" +version = "3.2.3" +requires_python = ">=3.8" +summary = "PostgreSQL database adapter for Python" +groups = ["default"] +dependencies = [ + "backports-zoneinfo>=0.2.0; python_version < \"3.9\"", + "typing-extensions>=4.6; python_version < \"3.13\"", + "tzdata; sys_platform == \"win32\"", +] +files = [ + {file = "psycopg-3.2.3-py3-none-any.whl", hash = "sha256:644d3973fe26908c73d4be746074f6e5224b03c1101d302d9a53bf565ad64907"}, + {file = "psycopg-3.2.3.tar.gz", hash = "sha256:a5764f67c27bec8bfac85764d23c534af2c27b893550377e37ce59c12aac47a2"}, +] + +[[package]] +name = "psycopg-binary" +version = "3.2.3" +requires_python = ">=3.8" +summary = "PostgreSQL database adapter for Python -- C optimisation distribution" +groups = ["default"] +marker = "implementation_name != \"pypy\"" +files = [ + {file = "psycopg_binary-3.2.3-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:48f8ca6ee8939bab760225b2ab82934d54330eec10afe4394a92d3f2a0c37dd6"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5361ea13c241d4f0ec3f95e0bf976c15e2e451e9cc7ef2e5ccfc9d170b197a40"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb987f14af7da7c24f803111dbc7392f5070fd350146af3345103f76ea82e339"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0463a11b1cace5a6aeffaf167920707b912b8986a9c7920341c75e3686277920"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b7be9a6c06518967b641fb15032b1ed682fd3b0443f64078899c61034a0bca6"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64a607e630d9f4b2797f641884e52b9f8e239d35943f51bef817a384ec1678fe"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fa33ead69ed133210d96af0c63448b1385df48b9c0247eda735c5896b9e6dbbf"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:1f8b0d0e99d8e19923e6e07379fa00570be5182c201a8c0b5aaa9a4d4a4ea20b"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:709447bd7203b0b2debab1acec23123eb80b386f6c29e7604a5d4326a11e5bd6"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5e37d5027e297a627da3551a1e962316d0f88ee4ada74c768f6c9234e26346d9"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:261f0031ee6074765096a19b27ed0f75498a8338c3dcd7f4f0d831e38adf12d1"}, +] + +[[package]] +name = "psycopg-pool" +version = "3.2.3" +requires_python = ">=3.8" +summary = "Connection Pool for Psycopg" +groups = ["default"] +dependencies = [ + "typing-extensions>=4.6", +] +files = [ + {file = "psycopg_pool-3.2.3-py3-none-any.whl", hash = "sha256:53bd8e640625e01b2927b2ad96df8ed8e8f91caea4597d45e7673fc7bbb85eb1"}, + {file = "psycopg_pool-3.2.3.tar.gz", hash = "sha256:bb942f123bef4b7fbe4d55421bd3fb01829903c95c0f33fd42b7e94e5ac9b52a"}, +] + +[[package]] +name = "psycopg" +version = "3.2.3" +extras = ["binary", "pool"] +requires_python = ">=3.8" +summary = "PostgreSQL database adapter for Python" +groups = ["default"] +dependencies = [ + "psycopg-binary==3.2.3; implementation_name != \"pypy\"", + "psycopg-pool", + "psycopg==3.2.3", +] +files = [ + {file = "psycopg-3.2.3-py3-none-any.whl", hash = "sha256:644d3973fe26908c73d4be746074f6e5224b03c1101d302d9a53bf565ad64907"}, + {file = "psycopg-3.2.3.tar.gz", hash = "sha256:a5764f67c27bec8bfac85764d23c534af2c27b893550377e37ce59c12aac47a2"}, +] + [[package]] name = "py-avro-schema" version = "3.8.2" diff --git a/pyproject.toml b/pyproject.toml index 786af88248..d12d5b92c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,12 +26,13 @@ dependencies = [ "ldap3>=2.9.1", "oracledb>=1.4.2", "pendulum>=3.0.0", + "psycopg[binary,pool]>=3.2.3", "py-avro-schema>=3.4.1", "pycryptodome>=3.19.0", + "pypdf>=5.0.0", "scikit-learn>=1.4.0", "tableauserverclient>=0.25", "tenacity>=8.2.3", - "pypdf>=5.0.0", ] requires-python = ">=3.12,<3.13" license = { text = "GPL-3.0-or-later" } diff --git a/requirements.txt b/requirements.txt index 79886416b3..b773e7093b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -119,6 +119,9 @@ prompt-toolkit==3.0.36 proto-plus==1.25.0 protobuf==4.25.5 psutil==6.1.0; platform_system == "Windows" +psycopg-binary==3.2.3; implementation_name != "pypy" +psycopg-pool==3.2.3 +psycopg[binary,pool]==3.2.3 py-avro-schema==3.8.2 pyarrow==18.0.0 pyasn1==0.6.1 diff --git a/src/teamster/code_locations/kipptaf/definitions.py b/src/teamster/code_locations/kipptaf/definitions.py index b7bce4aa1c..529d8a4d3a 100644 --- a/src/teamster/code_locations/kipptaf/definitions.py +++ b/src/teamster/code_locations/kipptaf/definitions.py @@ -5,6 +5,7 @@ build_sensor_for_freshness_checks, load_assets_from_modules, ) +from dagster_embedded_elt.dlt import DagsterDltResource from dagster_k8s import k8s_job_executor from teamster.code_locations.kipptaf import ( @@ -99,6 +100,7 @@ "db_bigquery": BIGQUERY_RESOURCE, "dbt_cli": get_dbt_cli_resource(DBT_PROJECT), "dds": resources.DIBELS_DATA_SYSTEM_RESOURCE, + "dlt": DagsterDltResource(), "fivetran": resources.FIVETRAN_RESOURCE, "gcs": GCS_RESOURCE, "google_directory": resources.GOOGLE_DIRECTORY_RESOURCE, diff --git a/src/teamster/code_locations/kipptaf/illuminate/assets.py b/src/teamster/code_locations/kipptaf/illuminate/assets.py index 10a8644e02..97ed600476 100644 --- a/src/teamster/code_locations/kipptaf/illuminate/assets.py +++ b/src/teamster/code_locations/kipptaf/illuminate/assets.py @@ -1,8 +1,7 @@ from dagster import AssetExecutionContext from dagster_embedded_elt.dlt import DagsterDltResource, dlt_assets from dlt import pipeline - -from teamster.libraries.dlt_sources.sql_database import sql_database +from dlt.sources.sql_database import sql_database @dlt_assets( diff --git a/src/teamster/code_locations/kipptaf/resources.py b/src/teamster/code_locations/kipptaf/resources.py index 2d5de133d5..dd9315f4f3 100644 --- a/src/teamster/code_locations/kipptaf/resources.py +++ b/src/teamster/code_locations/kipptaf/resources.py @@ -1,7 +1,6 @@ from dagster import EnvVar from dagster_airbyte import AirbyteCloudResource from dagster_embedded_elt.dlt import DagsterDltResource -from dagster_embedded_elt.sling.resources import SlingConnectionResource, SlingResource from dagster_fivetran import FivetranResource from teamster.libraries.adp.workforce_manager.resources import ( @@ -163,24 +162,3 @@ username=EnvVar("LITTLESIS_SFTP_USERNAME"), password=EnvVar("LITTLESIS_SFTP_PASSWORD"), ) - -SLING_RESOURCE = SlingResource( - connections=[ - SlingConnectionResource( - name="ILLUMINATE", - type="postgres", - # trunk-ignore-begin(pyright/reportCallIssue) - host=EnvVar("SNOWFLAKE_HOST"), - user=EnvVar("SNOWFLAKE_USER"), - # trunk-ignore-end(pyright/reportCallIssue) - ), - SlingConnectionResource( - name="BIGQUERY", - type="bigquery", - # trunk-ignore-begin(pyright/reportCallIssue) - host=EnvVar("SNOWFLAKE_HOST"), - user=EnvVar("SNOWFLAKE_USER"), - # trunk-ignore-end(pyright/reportCallIssue) - ), - ] -) From 54e18988b16007a9b528be5359a680f7097085ee Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 13 Nov 2024 16:52:21 +0000 Subject: [PATCH 10/20] fix: missing vars --- .../code_locations/kipptaf/dagster-cloud.yaml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index bed6626e35..5808c6cf79 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -381,6 +381,36 @@ locations: secretKeyRef: name: op-slack-api key: credential + - name: SOURCES__SQL_DATABASE__CREDENTIALS__DATABASE + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: database + - name: SOURCES__SQL_DATABASE__CREDENTIALS__DRIVERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: driver + - name: SOURCES__SQL_DATABASE__CREDENTIALS__HOST + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: ip + - name: SOURCES__SQL_DATABASE__CREDENTIALS__PASSWORD + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: password + - name: SOURCES__SQL_DATABASE__CREDENTIALS__PORT + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: port + - name: SOURCES__SQL_DATABASE__CREDENTIALS__USERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: username run_k8s_config: container_config: resources: From 3a355269ae86b8fc641af368119b45970f27f583 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 13 Nov 2024 21:09:58 +0000 Subject: [PATCH 11/20] fix: add dlt creds --- .../code_locations/kipptaf/dagster-cloud.yaml | 45 ++++++++++++------- .../kipptaf/illuminate/assets.py | 5 ++- .../dlt_sources/sql_database_pipeline.py | 5 +-- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index 5808c6cf79..4e8299ce03 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -411,6 +411,21 @@ locations: secretKeyRef: name: op-illuminate-db key: username + - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: username + - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: password + - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: project-id run_k8s_config: container_config: resources: @@ -779,18 +794,18 @@ locations: secretKeyRef: name: op-illuminate-db key: username - # - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: username - # - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: password - # - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: project-id + - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: username + - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: password + - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + valueFrom: + secretKeyRef: + name: op-gcp-service-account-dlt + key: project-id diff --git a/src/teamster/code_locations/kipptaf/illuminate/assets.py b/src/teamster/code_locations/kipptaf/illuminate/assets.py index 97ed600476..b1955844b4 100644 --- a/src/teamster/code_locations/kipptaf/illuminate/assets.py +++ b/src/teamster/code_locations/kipptaf/illuminate/assets.py @@ -6,12 +6,13 @@ @dlt_assets( dlt_source=sql_database( - schema="dna_assessments", table_names=["agg_student_responses_standard"] + schema="dna_assessments", + table_names=["assessments", "agg_student_responses_standard"], ), dlt_pipeline=pipeline( pipeline_name="illuminate", destination="bigquery", - dataset_name="dev_illuminate", + dataset_name="dlt_illuminate", progress="log", ), ) diff --git a/src/teamster/libraries/dlt_sources/sql_database_pipeline.py b/src/teamster/libraries/dlt_sources/sql_database_pipeline.py index 9ca98b2eff..a26f8cedf6 100644 --- a/src/teamster/libraries/dlt_sources/sql_database_pipeline.py +++ b/src/teamster/libraries/dlt_sources/sql_database_pipeline.py @@ -1,4 +1,3 @@ -# flake8: noqa import os from typing import Any @@ -8,7 +7,6 @@ from dlt.common import pendulum from dlt.sources.credentials import ConnectionStringCredentials from dlt.sources.sql_database import Table, sql_database, sql_table -from sqlalchemy.sql.sqltypes import TypeEngine def load_select_tables_from_database() -> None: @@ -132,7 +130,7 @@ def table_adapter(table: Table) -> None: if table.name == "family": # this is SqlAlchemy table. _columns are writable # let's drop updated column - table._columns.remove(table.columns["updated"]) # type: ignore + table._columns.remove(table.columns["updated"]) family = sql_table( credentials="mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam", @@ -240,7 +238,6 @@ def create_unsw_flow() -> None: def test_connectorx_speed() -> None: """Uses unsw_flow dataset (~2mln rows, 25+ columns) to test connectorx speed""" - import os # from dlt.destinations import filesystem From 6de5e6936dd6fdc0143857193385c9fa04d93a14 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 13 Nov 2024 23:27:14 +0000 Subject: [PATCH 12/20] defer_table_reflect --- src/teamster/code_locations/kipptaf/illuminate/assets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/teamster/code_locations/kipptaf/illuminate/assets.py b/src/teamster/code_locations/kipptaf/illuminate/assets.py index b1955844b4..0cd286b5a5 100644 --- a/src/teamster/code_locations/kipptaf/illuminate/assets.py +++ b/src/teamster/code_locations/kipptaf/illuminate/assets.py @@ -8,6 +8,7 @@ dlt_source=sql_database( schema="dna_assessments", table_names=["assessments", "agg_student_responses_standard"], + defer_table_reflect=True, ), dlt_pipeline=pipeline( pipeline_name="illuminate", From 924e99420eaeba0a2ed9e914bb0c3783dfa9db6f Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 14 Nov 2024 01:38:58 +0000 Subject: [PATCH 13/20] build: try credentials file --- .../code_locations/kipptaf/dagster-cloud.yaml | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index 4e8299ce03..8a8ef50192 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -411,21 +411,26 @@ locations: secretKeyRef: name: op-illuminate-db key: username - - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + - name: DESTINATION__BIGQUERY__CREDENTIALS valueFrom: secretKeyRef: name: op-gcp-service-account-dlt - key: username - - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: password - - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: project-id + key: keyfile.json + # - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: username + # - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: password + # - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: project-id run_k8s_config: container_config: resources: @@ -794,18 +799,23 @@ locations: secretKeyRef: name: op-illuminate-db key: username - - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: username - - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: password - - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + - name: DESTINATION__BIGQUERY__CREDENTIALS valueFrom: secretKeyRef: name: op-gcp-service-account-dlt - key: project-id + key: keyfile.json + # - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: username + # - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: password + # - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID + # valueFrom: + # secretKeyRef: + # name: op-gcp-service-account-dlt + # key: project-id From b1440e5d8a9340e9fe39b8cac1381085b21df2f4 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 14 Nov 2024 04:01:14 +0000 Subject: [PATCH 14/20] refactor: dlt creds as dagster objects --- .../code_locations/kipptaf/dagster-cloud.yaml | 82 ++++++------------- .../kipptaf/illuminate/assets.py | 20 ++++- 2 files changed, 42 insertions(+), 60 deletions(-) diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index 8a8ef50192..6da0b4b6ee 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -381,56 +381,41 @@ locations: secretKeyRef: name: op-slack-api key: credential - - name: SOURCES__SQL_DATABASE__CREDENTIALS__DATABASE - valueFrom: - secretKeyRef: - name: op-illuminate-db - key: database - - name: SOURCES__SQL_DATABASE__CREDENTIALS__DRIVERNAME + - name: ILLUMINATE_DB_DRIVERNAME valueFrom: secretKeyRef: name: op-illuminate-db key: driver - - name: SOURCES__SQL_DATABASE__CREDENTIALS__HOST + - name: ILLUMINATE_DB_HOST valueFrom: secretKeyRef: name: op-illuminate-db key: ip - - name: SOURCES__SQL_DATABASE__CREDENTIALS__PASSWORD + - name: ILLUMINATE_DB_DATABASE valueFrom: secretKeyRef: name: op-illuminate-db - key: password - - name: SOURCES__SQL_DATABASE__CREDENTIALS__PORT + key: database + - name: ILLUMINATE_DB_USERNAME valueFrom: secretKeyRef: name: op-illuminate-db - key: port - - name: SOURCES__SQL_DATABASE__CREDENTIALS__USERNAME + key: username + - name: ILLUMINATE_DB_PASSWORD valueFrom: secretKeyRef: name: op-illuminate-db - key: username + key: password + # - name: ILLUMINATE_DB_PORT + # valueFrom: + # secretKeyRef: + # name: op-illuminate-db + # key: port - name: DESTINATION__BIGQUERY__CREDENTIALS valueFrom: secretKeyRef: name: op-gcp-service-account-dlt key: keyfile.json - # - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: username - # - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: password - # - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: project-id run_k8s_config: container_config: resources: @@ -769,53 +754,38 @@ locations: secretKeyRef: name: op-slack-api key: credential - - name: SOURCES__SQL_DATABASE__CREDENTIALS__DATABASE - valueFrom: - secretKeyRef: - name: op-illuminate-db - key: database - - name: SOURCES__SQL_DATABASE__CREDENTIALS__DRIVERNAME + - name: ILLUMINATE_DB_DRIVERNAME valueFrom: secretKeyRef: name: op-illuminate-db key: driver - - name: SOURCES__SQL_DATABASE__CREDENTIALS__HOST + - name: ILLUMINATE_DB_HOST valueFrom: secretKeyRef: name: op-illuminate-db key: ip - - name: SOURCES__SQL_DATABASE__CREDENTIALS__PASSWORD + - name: ILLUMINATE_DB_DATABASE valueFrom: secretKeyRef: name: op-illuminate-db - key: password - - name: SOURCES__SQL_DATABASE__CREDENTIALS__PORT + key: database + - name: ILLUMINATE_DB_USERNAME valueFrom: secretKeyRef: name: op-illuminate-db - key: port - - name: SOURCES__SQL_DATABASE__CREDENTIALS__USERNAME + key: username + - name: ILLUMINATE_DB_PASSWORD valueFrom: secretKeyRef: name: op-illuminate-db - key: username + key: password + # - name: ILLUMINATE_DB_PORT + # valueFrom: + # secretKeyRef: + # name: op-illuminate-db + # key: port - name: DESTINATION__BIGQUERY__CREDENTIALS valueFrom: secretKeyRef: name: op-gcp-service-account-dlt key: keyfile.json - # - name: DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: username - # - name: DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: password - # - name: DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: project-id diff --git a/src/teamster/code_locations/kipptaf/illuminate/assets.py b/src/teamster/code_locations/kipptaf/illuminate/assets.py index 0cd286b5a5..0f7379762c 100644 --- a/src/teamster/code_locations/kipptaf/illuminate/assets.py +++ b/src/teamster/code_locations/kipptaf/illuminate/assets.py @@ -1,11 +1,23 @@ -from dagster import AssetExecutionContext +from dagster import AssetExecutionContext, EnvVar, _check from dagster_embedded_elt.dlt import DagsterDltResource, dlt_assets from dlt import pipeline from dlt.sources.sql_database import sql_database +from sqlalchemy import URL, create_engine @dlt_assets( dlt_source=sql_database( + credentials=create_engine( + url=URL.create( + drivername=_check.not_none( + value=EnvVar("ILLUMINATE_DB_DRIVERNAME").get_value() + ), + host=EnvVar("ILLUMINATE_DB_HOST").get_value(), + database=EnvVar("ILLUMINATE_DB_DATABASE").get_value(), + username=EnvVar("ILLUMINATE_DB_USERNAME").get_value(), + password=EnvVar("ILLUMINATE_DB_PASSWORD").get_value(), + ) + ), schema="dna_assessments", table_names=["assessments", "agg_student_responses_standard"], defer_table_reflect=True, @@ -13,14 +25,14 @@ dlt_pipeline=pipeline( pipeline_name="illuminate", destination="bigquery", - dataset_name="dlt_illuminate", + dataset_name="dlt_illuminate_dna_assessments", progress="log", ), ) -def illuminate_assets(context: AssetExecutionContext, dlt: DagsterDltResource): +def illuminate_dna_assessments(context: AssetExecutionContext, dlt: DagsterDltResource): yield from dlt.run(context=context) assets = [ - illuminate_assets, + illuminate_dna_assessments, ] From ae23039342de18b78eb142d684cef79fbc1873c9 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 14 Nov 2024 04:35:48 +0000 Subject: [PATCH 15/20] refactor: dlt bq creds --- .devcontainer/scripts/postCreate.sh | 4 ++++ .../tpl/gcloud_teamster_dlt_keyfile.json.tpl | 1 + .../code_locations/kipptaf/dagster-cloud.yaml | 22 +++++-------------- .../kipptaf/illuminate/assets.py | 9 +++++++- 4 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 .devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl diff --git a/.devcontainer/scripts/postCreate.sh b/.devcontainer/scripts/postCreate.sh index 4acff5e48b..f32f82a273 100644 --- a/.devcontainer/scripts/postCreate.sh +++ b/.devcontainer/scripts/postCreate.sh @@ -53,6 +53,10 @@ op inject -f --in-file=.devcontainer/tpl/dbt_cloud.yml.tpl \ --out-file=env/dbt_cloud.yml && sudo mv -f env/dbt_cloud.yml /home/vscode/.dbt/dbt_cloud.yml +op inject -f --in-file=.devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl \ + --out-file=env/gcloud_teamster_dlt_keyfile.json && + sudo mv -f env/gcloud_teamster_dlt_keyfile.json /etc/secret-volume/gcloud_teamster_dlt_keyfile.json + # install pdm dependencies pdm install --frozen-lockfile diff --git a/.devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl b/.devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl new file mode 100644 index 0000000000..aa7c6d6046 --- /dev/null +++ b/.devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl @@ -0,0 +1 @@ +op://Data Team/Teamster Service Account - dlt/keyfile.json \ No newline at end of file diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index 6da0b4b6ee..b96ec938f2 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -406,16 +406,11 @@ locations: secretKeyRef: name: op-illuminate-db key: password - # - name: ILLUMINATE_DB_PORT + # - name: DESTINATION__BIGQUERY__CREDENTIALS # valueFrom: # secretKeyRef: - # name: op-illuminate-db - # key: port - - name: DESTINATION__BIGQUERY__CREDENTIALS - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: keyfile.json + # name: op-gcp-service-account-dlt + # key: keyfile.json run_k8s_config: container_config: resources: @@ -779,13 +774,8 @@ locations: secretKeyRef: name: op-illuminate-db key: password - # - name: ILLUMINATE_DB_PORT + # - name: DESTINATION__BIGQUERY__CREDENTIALS # valueFrom: # secretKeyRef: - # name: op-illuminate-db - # key: port - - name: DESTINATION__BIGQUERY__CREDENTIALS - valueFrom: - secretKeyRef: - name: op-gcp-service-account-dlt - key: keyfile.json + # name: op-gcp-service-account-dlt + # key: keyfile.json diff --git a/src/teamster/code_locations/kipptaf/illuminate/assets.py b/src/teamster/code_locations/kipptaf/illuminate/assets.py index 0f7379762c..393ad62c07 100644 --- a/src/teamster/code_locations/kipptaf/illuminate/assets.py +++ b/src/teamster/code_locations/kipptaf/illuminate/assets.py @@ -1,3 +1,5 @@ +import json + from dagster import AssetExecutionContext, EnvVar, _check from dagster_embedded_elt.dlt import DagsterDltResource, dlt_assets from dlt import pipeline @@ -30,7 +32,12 @@ ), ) def illuminate_dna_assessments(context: AssetExecutionContext, dlt: DagsterDltResource): - yield from dlt.run(context=context) + yield from dlt.run( + context=context, + credentials=json.load( + fp=open(file="/etc/secret-volume/gcloud_teamster_dlt_keyfile.json") + ), + ) assets = [ From 16458e8c11bd077822c382a1fe67ce12bad15b87 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 14 Nov 2024 04:49:17 +0000 Subject: [PATCH 16/20] build: fix secret --- src/teamster/code_locations/kipptaf/dagster-cloud.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index b96ec938f2..a0f77ce133 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -41,6 +41,11 @@ locations: items: - key: service_account_gserviceaccount.json path: gcloud_service_account_json + - secret: + name: op-gcp-service-account-dlt + items: + - key: keyfile.json + path: gcloud_teamster_dlt_keyfile.json server_k8s_config: container_config: resources: From b9ad989d8a5ec4358b5d0cf167ae7ad24a2cfdab Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 14 Nov 2024 05:06:16 +0000 Subject: [PATCH 17/20] build: cleanup --- src/teamster/code_locations/kipptaf/dagster-cloud.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index a0f77ce133..2d6bf25a4f 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -411,11 +411,6 @@ locations: secretKeyRef: name: op-illuminate-db key: password - # - name: DESTINATION__BIGQUERY__CREDENTIALS - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: keyfile.json run_k8s_config: container_config: resources: @@ -779,8 +774,3 @@ locations: secretKeyRef: name: op-illuminate-db key: password - # - name: DESTINATION__BIGQUERY__CREDENTIALS - # valueFrom: - # secretKeyRef: - # name: op-gcp-service-account-dlt - # key: keyfile.json From f00a41371466095ed2502247c0544121b82f2db3 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Fri, 15 Nov 2024 01:03:32 +0000 Subject: [PATCH 18/20] build: deps --- pdm.lock | 210 ++++++++++++++++++++++++++++++++++++++++------- pyproject.toml | 1 - requirements.txt | 34 +++----- 3 files changed, 191 insertions(+), 54 deletions(-) diff --git a/pdm.lock b/pdm.lock index bdf9223d2f..5b5146eafd 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:d2937b7f663664fcd3b7c03db3d4690b8f7b8a78793f1df5248cc957da6f5f4e" +content_hash = "sha256:7558dfb62e381caef94fd84b6145b06dfe1ed7822bd9763d323b9091b0e2f72f" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -29,6 +29,66 @@ files = [ {file = "agate-1.9.1.tar.gz", hash = "sha256:bc60880c2ee59636a2a80cd8603d63f995be64526abf3cbba12f00767bcd5b3d"}, ] +[[package]] +name = "aiohappyeyeballs" +version = "2.4.3" +requires_python = ">=3.8" +summary = "Happy Eyeballs for asyncio" +groups = ["default"] +files = [ + {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, + {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, +] + +[[package]] +name = "aiohttp" +version = "3.11.2" +requires_python = ">=3.9" +summary = "Async http client/server framework (asyncio)" +groups = ["default"] +dependencies = [ + "aiohappyeyeballs>=2.3.0", + "aiosignal>=1.1.2", + "async-timeout<6.0,>=4.0; python_version < \"3.11\"", + "attrs>=17.3.0", + "frozenlist>=1.1.1", + "multidict<7.0,>=4.5", + "propcache>=0.2.0", + "yarl<2.0,>=1.17.0", +] +files = [ + {file = "aiohttp-3.11.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f833a80d9de9307d736b6af58c235b17ef7f90ebea7b9c49cd274dec7a66a2f1"}, + {file = "aiohttp-3.11.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:382f853516664d2ebfc75dc01da4a10fdef5edcb335fe7b45cf471ce758ecb18"}, + {file = "aiohttp-3.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3a2bcf6c81639a165da93469e1e0aff67c956721f3fa9c0560f07dd1e505116"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de3b4d5fb5d69749104b880a157f38baeea7765c93d9cd3837cedd5b84729e10"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0a90a0dc4b054b5af299a900bf950fe8f9e3e54322bc405005f30aa5cacc5c98"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32334f35824811dd20a12cc90825d000e6b50faaeaa71408d42269151a66140d"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cba0b8d25aa2d450762f3dd6df85498f5e7c3ad0ddeb516ef2b03510f0eea32"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bbb2dbc2701ab7e9307ca3a8fa4999c5b28246968e0a0202a5afabf48a42e22"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97fba98fc5d9ccd3d33909e898d00f2494d6a9eec7cbda3d030632e2c8bb4d00"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0ebdf5087e2ce903d8220cc45dcece90c2199ae4395fd83ca616fcc81010db2c"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:122768e3ae9ce74f981b46edefea9c6e5a40aea38aba3ac50168e6370459bf20"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5587da333b7d280a312715b843d43e734652aa382cba824a84a67c81f75b338b"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85de9904bc360fd29a98885d2bfcbd4e02ab33c53353cb70607f2bea2cb92468"}, + {file = "aiohttp-3.11.2-cp312-cp312-win32.whl", hash = "sha256:b470de64d17156c37e91effc109d3b032b39867000e2c126732fe01d034441f9"}, + {file = "aiohttp-3.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:3f617a48b70f4843d54f52440ea1e58da6bdab07b391a3a6aed8d3b311a4cc04"}, + {file = "aiohttp-3.11.2.tar.gz", hash = "sha256:68d1f46f9387db3785508f5225d3acbc5825ca13d9c29f2b5cce203d5863eb79"}, +] + +[[package]] +name = "aiosignal" +version = "1.3.1" +requires_python = ">=3.7" +summary = "aiosignal: a list of registered asynchronous callbacks" +groups = ["default"] +dependencies = [ + "frozenlist>=1.1.0", +] +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + [[package]] name = "alembic" version = "1.14.0" @@ -572,18 +632,18 @@ files = [ [[package]] name = "dagster-embedded-elt" -version = "0.25.1" +version = "0.25.2" requires_python = "<3.13,>=3.9" summary = "Package for performing ETL/ELT tasks with Dagster." groups = ["default"] dependencies = [ - "dagster==1.9.1", + "dagster==1.9.2", "dlt>=0.4", "sling>=1.1.5", ] files = [ - {file = "dagster-embedded-elt-0.25.1.tar.gz", hash = "sha256:41eba4c82814d8b7c1fb0dd02fb0113b7a32d9b3d1abfb94a2ce8bed45223893"}, - {file = "dagster_embedded_elt-0.25.1-py3-none-any.whl", hash = "sha256:b7eb014bf2fe93cab717ef5564b2596b0b0bf83c6f9397e4a36fe580e2a2980c"}, + {file = "dagster-embedded-elt-0.25.2.tar.gz", hash = "sha256:431b7f4ed94130b16ac75e66d04dc06d29ae8ae2fdc01b9ae9f7e4ebbe82941e"}, + {file = "dagster_embedded_elt-0.25.2-py3-none-any.whl", hash = "sha256:0dc8eb5acc3eb82d91537a8c4f54ce967386307f2a1406cdea7793e90af2d7c3"}, ] [[package]] @@ -943,12 +1003,13 @@ files = [ [[package]] name = "dlt" -version = "1.3.0" +version = "1.4.0" requires_python = "<3.13,>=3.8.1" summary = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." groups = ["default"] dependencies = [ "PyYAML>=5.4.1", + "aiohttp>=3.9", "astunparse>=1.6.3", "click>=7.1", "fsspec>=2022.4.0", @@ -977,8 +1038,8 @@ dependencies = [ "win-precise-time>=1.4.2; os_name == \"nt\"", ] files = [ - {file = "dlt-1.3.0-py3-none-any.whl", hash = "sha256:e2583ed0ad4a0d9941b8f9cb0e078f4443bcbeb0e1cf1cce586cf35107ccf266"}, - {file = "dlt-1.3.0.tar.gz", hash = "sha256:57eecee99ace25b6d37027a78f59f8c735d1913cc81f1101e1b47bf96fc544b8"}, + {file = "dlt-1.4.0-py3-none-any.whl", hash = "sha256:c3a69e4067581bf0335796bec62d58058ff1f11249f16b699d6657544b126247"}, + {file = "dlt-1.4.0.tar.gz", hash = "sha256:75208448dc11dd501cf15d76742368816fef8e1b22fb07417f69d5ceb720b324"}, ] [[package]] @@ -1057,6 +1118,32 @@ files = [ {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] +[[package]] +name = "frozenlist" +version = "1.5.0" +requires_python = ">=3.8" +summary = "A list-like structure which implements collections.abc.MutableSequence" +groups = ["default"] +files = [ + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f"}, + {file = "frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8"}, + {file = "frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f"}, + {file = "frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3"}, + {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, +] + [[package]] name = "fsspec" version = "2024.10.0" @@ -2013,7 +2100,7 @@ name = "multidict" version = "6.1.0" requires_python = ">=3.8" summary = "multidict implementation" -groups = ["dev"] +groups = ["default", "dev"] dependencies = [ "typing-extensions>=4.1.0; python_version < \"3.11\"", ] @@ -2229,6 +2316,44 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] +[[package]] +name = "pathvalidate" +version = "3.2.1" +requires_python = ">=3.7" +summary = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." +groups = ["default"] +files = [ + {file = "pathvalidate-3.2.1-py3-none-any.whl", hash = "sha256:9a6255eb8f63c9e2135b9be97a5ce08f10230128c4ae7b3e935378b82b22c4c9"}, + {file = "pathvalidate-3.2.1.tar.gz", hash = "sha256:f5d07b1e2374187040612a1fcd2bcb2919f8db180df254c9581bb90bf903377d"}, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +requires_python = ">=3.8" +summary = "Python datetimes made easy" +groups = ["default"] +dependencies = [ + "backports-zoneinfo>=0.2.1; python_version < \"3.9\"", + "importlib-resources>=5.9.0; python_version < \"3.9\"", + "python-dateutil>=2.6", + "time-machine>=2.6.0; implementation_name != \"pypy\"", + "tzdata>=2020.1", +] +files = [ + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + [[package]] name = "pex" version = "2.24.1" @@ -2291,7 +2416,7 @@ name = "propcache" version = "0.2.0" requires_python = ">=3.8" summary = "Accelerated property cache" -groups = ["dev"] +groups = ["default", "dev"] files = [ {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2"}, {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7"}, @@ -3286,6 +3411,31 @@ files = [ {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, ] +[[package]] +name = "time-machine" +version = "2.16.0" +requires_python = ">=3.9" +summary = "Travel through time in your tests." +groups = ["default"] +marker = "implementation_name != \"pypy\"" +dependencies = [ + "python-dateutil", +] +files = [ + {file = "time_machine-2.16.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:84788f4d62a8b1bf5e499bb9b0e23ceceea21c415ad6030be6267ce3d639842f"}, + {file = "time_machine-2.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:15ec236b6571730236a193d9d6c11d472432fc6ab54e85eac1c16d98ddcd71bf"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cedc989717c8b44a3881ac3d68ab5a95820448796c550de6a2149ed1525157f0"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d26d79de1c63a8c6586c75967e09b0ff306aa7e944a1eaddb74595c9b1839ca"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:317b68b56a9c3731e0cf8886e0f94230727159e375988b36c60edce0ddbcb44a"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:43e1e18279759897be3293a255d53e6b1cb0364b69d9591d0b80c51e461c94b0"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e43adb22def972a29d2b147999b56897116085777a0fea182fd93ee45730611e"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0c766bea27a0600e36806d628ebc4b47178b12fcdfb6c24dc0a566a9c06bfe7f"}, + {file = "time_machine-2.16.0-cp312-cp312-win32.whl", hash = "sha256:6dae82ab647d107817e013db82223e20a9853fa88543fec853ae326382d03c2e"}, + {file = "time_machine-2.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:265462c77dc9576267c3c7f20707780a171a9fdbac93ac22e608c309efd68c33"}, + {file = "time_machine-2.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:ef768e14768eebe3bb1196c0dece8e14c1c6991605721214a0c3c68cf77eb216"}, + {file = "time_machine-2.16.0.tar.gz", hash = "sha256:4a99acc273d2f98add23a89b94d4dd9e14969c01214c8514bfa78e4e9364c7e2"}, +] + [[package]] name = "tomli" version = "2.1.0" @@ -3366,13 +3516,13 @@ files = [ [[package]] name = "types-setuptools" -version = "75.3.0.20241107" +version = "75.3.0.20241112" requires_python = ">=3.8" summary = "Typing stubs for setuptools" groups = ["default"] files = [ - {file = "types-setuptools-75.3.0.20241107.tar.gz", hash = "sha256:f66710e1cd4a936e5fcc12d4e49be1a67c34372cf753e87ebe704426451b4012"}, - {file = "types_setuptools-75.3.0.20241107-py3-none-any.whl", hash = "sha256:bc6de6e2bcb6d610556304d0a69fe4ca208ac4896162647314ecfd9fd73d8550"}, + {file = "types-setuptools-75.3.0.20241112.tar.gz", hash = "sha256:f9e1ebd17a56f606e16395c4ee4efa1cdc394b9a2a0ee898a624058b4b62ef8f"}, + {file = "types_setuptools-75.3.0.20241112-py3-none-any.whl", hash = "sha256:78cb5fef4a6056d2f37114d27da90f4655a306e4e38042d7034a8a880bc3f5dd"}, ] [[package]] @@ -3564,24 +3714,24 @@ files = [ [[package]] name = "websockets" -version = "13.1" -requires_python = ">=3.8" +version = "14.1" +requires_python = ">=3.9" summary = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" groups = ["dev"] files = [ - {file = "websockets-13.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d75baf00138f80b48f1eac72ad1535aac0b6461265a0bcad391fc5aba875cfc"}, - {file = "websockets-13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9b6f347deb3dcfbfde1c20baa21c2ac0751afaa73e64e5b693bb2b848efeaa49"}, - {file = "websockets-13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de58647e3f9c42f13f90ac7e5f58900c80a39019848c5547bc691693098ae1bd"}, - {file = "websockets-13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1b54689e38d1279a51d11e3467dd2f3a50f5f2e879012ce8f2d6943f00e83f0"}, - {file = "websockets-13.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf1781ef73c073e6b0f90af841aaf98501f975d306bbf6221683dd594ccc52b6"}, - {file = "websockets-13.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d23b88b9388ed85c6faf0e74d8dec4f4d3baf3ecf20a65a47b836d56260d4b9"}, - {file = "websockets-13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3c78383585f47ccb0fcf186dcb8a43f5438bd7d8f47d69e0b56f71bf431a0a68"}, - {file = "websockets-13.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d6d300f8ec35c24025ceb9b9019ae9040c1ab2f01cddc2bcc0b518af31c75c14"}, - {file = "websockets-13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9dcaf8b0cc72a392760bb8755922c03e17a5a54e08cca58e8b74f6902b433cf"}, - {file = "websockets-13.1-cp312-cp312-win32.whl", hash = "sha256:2f85cf4f2a1ba8f602298a853cec8526c2ca42a9a4b947ec236eaedb8f2dc80c"}, - {file = "websockets-13.1-cp312-cp312-win_amd64.whl", hash = "sha256:38377f8b0cdeee97c552d20cf1865695fcd56aba155ad1b4ca8779a5b6ef4ac3"}, - {file = "websockets-13.1-py3-none-any.whl", hash = "sha256:a9a396a6ad26130cdae92ae10c36af09d9bfe6cafe69670fd3b6da9b07b4044f"}, - {file = "websockets-13.1.tar.gz", hash = "sha256:a3b3366087c1bc0a2795111edcadddb8b3b59509d5db5d7ea3fdd69f954a8878"}, + {file = "websockets-14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed907449fe5e021933e46a3e65d651f641975a768d0649fee59f10c2985529ed"}, + {file = "websockets-14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:87e31011b5c14a33b29f17eb48932e63e1dcd3fa31d72209848652310d3d1f0d"}, + {file = "websockets-14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bc6ccf7d54c02ae47a48ddf9414c54d48af9c01076a2e1023e3b486b6e72c707"}, + {file = "websockets-14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9777564c0a72a1d457f0848977a1cbe15cfa75fa2f67ce267441e465717dcf1a"}, + {file = "websockets-14.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a655bde548ca98f55b43711b0ceefd2a88a71af6350b0c168aa77562104f3f45"}, + {file = "websockets-14.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3dfff83ca578cada2d19e665e9c8368e1598d4e787422a460ec70e531dbdd58"}, + {file = "websockets-14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6a6c9bcf7cdc0fd41cc7b7944447982e8acfd9f0d560ea6d6845428ed0562058"}, + {file = "websockets-14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4b6caec8576e760f2c7dd878ba817653144d5f369200b6ddf9771d64385b84d4"}, + {file = "websockets-14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eb6d38971c800ff02e4a6afd791bbe3b923a9a57ca9aeab7314c21c84bf9ff05"}, + {file = "websockets-14.1-cp312-cp312-win32.whl", hash = "sha256:1d045cbe1358d76b24d5e20e7b1878efe578d9897a25c24e6006eef788c0fdf0"}, + {file = "websockets-14.1-cp312-cp312-win_amd64.whl", hash = "sha256:90f4c7a069c733d95c308380aae314f2cb45bd8a904fb03eb36d1a4983a4993f"}, + {file = "websockets-14.1-py3-none-any.whl", hash = "sha256:4d4fc827a20abe6d544a119896f6b78ee13fe81cbfef416f3f2ddf09a03f0e2e"}, + {file = "websockets-14.1.tar.gz", hash = "sha256:398b10c77d471c0aab20a845e7a60076b6390bfdaac7a6d2edb0d2c59d75e8d8"}, ] [[package]] @@ -3613,7 +3763,7 @@ name = "yarl" version = "1.17.1" requires_python = ">=3.9" summary = "Yet another URL library" -groups = ["dev"] +groups = ["default", "dev"] dependencies = [ "idna>=2.0", "multidict>=4.0", @@ -3649,4 +3799,4 @@ groups = ["default"] files = [ {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, -] \ No newline at end of file +] diff --git a/pyproject.toml b/pyproject.toml index e1b1624de7..44be485af4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ dependencies = [ "py-avro-schema>=3.4.1", "pycryptodome>=3.19.0", "pypdf>=5.0.0", - "pypdf>=5.0.0", "python-dateutil>=2.9.0.post0", "scikit-learn>=1.4.0", "tableauserverclient>=0.25", diff --git a/requirements.txt b/requirements.txt index c1e3fd3f37..2290bee02f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,9 @@ # Please do not edit it manually. agate==1.9.1 +aiohappyeyeballs==2.4.3 +aiohttp==3.11.2 +aiosignal==1.3.1 alembic==1.14.0 annotated-types==0.7.0 astunparse==1.6.3 @@ -27,6 +30,7 @@ dagster-airbyte==0.25.2 dagster-cloud==1.9.2 dagster-cloud-cli==1.9.2 dagster-dbt==0.25.2 +dagster-embedded-elt==0.25.2 dagster-fivetran==0.25.2 dagster-gcp==0.25.2 dagster-k8s==0.25.2 @@ -43,23 +47,19 @@ dbt-extractor==0.5.1 dbt-semantic-interfaces==0.5.1 deepdiff==7.0.1 defusedxml==0.7.1 -dlt==1.3.0 +dlt==1.4.0 docstring-parser==0.16 durationpy==0.9 fastavro==1.9.7 filelock==3.16.1 +frozenlist==1.5.0 fsspec==2024.10.0 gitdb==4.0.11 github3-py==4.0.1 -<<<<<<< HEAD gitpython==3.1.43 giturlparse==0.12.0 -google-api-core[grpc]==2.22.0 -google-api-python-client==2.151.0 -======= google-api-core[grpc]==2.23.0 google-api-python-client==2.153.0 ->>>>>>> main google-auth==2.36.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.1 @@ -102,6 +102,7 @@ memoization==0.4.0 minimal-snowplow-tracker==0.0.2 more-itertools==10.5.0 msgpack==1.1.0 +multidict==6.1.0 networkx==3.4.2 numpy==2.1.3 oauth2client==4.1.3 @@ -114,15 +115,13 @@ pandas==2.2.3 paramiko==3.5.0 parsedatetime==2.6 pathspec==0.12.1 -<<<<<<< HEAD pathvalidate==3.2.1 pendulum==3.0.0 -======= ->>>>>>> main pex==2.24.1 pluggy==1.5.0 ply==3.11 prompt-toolkit==3.0.36 +propcache==0.2.0 proto-plus==1.25.0 protobuf==4.25.5 psutil==6.1.0; platform_system == "Windows" @@ -160,12 +159,8 @@ rpds-py==0.21.0 rsa==4.9 scikit-learn==1.5.2 scipy==1.14.1 -<<<<<<< HEAD semver==3.0.2 -setuptools==75.3.0 -======= setuptools==75.5.0 ->>>>>>> main shellingham==1.5.4 simplejson==3.19.3 six==1.16.0 @@ -185,18 +180,14 @@ tabulate==0.9.0 tenacity==9.0.0 text-unidecode==1.3 threadpoolctl==3.5.0 -<<<<<<< HEAD time-machine==2.16.0; implementation_name != "pypy" -tomli==2.0.2 -tomlkit==0.13.2 -======= tomli==2.1.0 ->>>>>>> main +tomlkit==0.13.2 toposort==1.10 tqdm==4.67.0 typeguard==4.4.1 typer==0.13.0 -types-setuptools==75.3.0.20241107 +types-setuptools==75.3.0.20241112 typing-extensions==4.12.2 tzdata==2024.2 universal-pathlib==0.2.5; python_version >= "3.12" @@ -205,10 +196,7 @@ urllib3==2.2.3 watchdog==5.0.3 wcwidth==0.2.13 websocket-client==1.8.0 -<<<<<<< HEAD wheel==0.45.0 win-precise-time==1.4.2; os_name == "nt" -zipp==3.20.2 -======= +yarl==1.17.1 zipp==3.21.0 ->>>>>>> main \ No newline at end of file From d68baac26f6d1c3d30ec552328838ac900469c34 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:45:55 +0000 Subject: [PATCH 19/20] build: deps --- pdm.lock | 176 +++++++++++++++++++++++------------------------ requirements.txt | 18 ++--- 2 files changed, 97 insertions(+), 97 deletions(-) diff --git a/pdm.lock b/pdm.lock index 5b5146eafd..70fbd53e2b 100644 --- a/pdm.lock +++ b/pdm.lock @@ -42,7 +42,7 @@ files = [ [[package]] name = "aiohttp" -version = "3.11.2" +version = "3.11.5" requires_python = ">=3.9" summary = "Async http client/server framework (asyncio)" groups = ["default"] @@ -57,22 +57,22 @@ dependencies = [ "yarl<2.0,>=1.17.0", ] files = [ - {file = "aiohttp-3.11.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f833a80d9de9307d736b6af58c235b17ef7f90ebea7b9c49cd274dec7a66a2f1"}, - {file = "aiohttp-3.11.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:382f853516664d2ebfc75dc01da4a10fdef5edcb335fe7b45cf471ce758ecb18"}, - {file = "aiohttp-3.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3a2bcf6c81639a165da93469e1e0aff67c956721f3fa9c0560f07dd1e505116"}, - {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de3b4d5fb5d69749104b880a157f38baeea7765c93d9cd3837cedd5b84729e10"}, - {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0a90a0dc4b054b5af299a900bf950fe8f9e3e54322bc405005f30aa5cacc5c98"}, - {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32334f35824811dd20a12cc90825d000e6b50faaeaa71408d42269151a66140d"}, - {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cba0b8d25aa2d450762f3dd6df85498f5e7c3ad0ddeb516ef2b03510f0eea32"}, - {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bbb2dbc2701ab7e9307ca3a8fa4999c5b28246968e0a0202a5afabf48a42e22"}, - {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97fba98fc5d9ccd3d33909e898d00f2494d6a9eec7cbda3d030632e2c8bb4d00"}, - {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0ebdf5087e2ce903d8220cc45dcece90c2199ae4395fd83ca616fcc81010db2c"}, - {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:122768e3ae9ce74f981b46edefea9c6e5a40aea38aba3ac50168e6370459bf20"}, - {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5587da333b7d280a312715b843d43e734652aa382cba824a84a67c81f75b338b"}, - {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85de9904bc360fd29a98885d2bfcbd4e02ab33c53353cb70607f2bea2cb92468"}, - {file = "aiohttp-3.11.2-cp312-cp312-win32.whl", hash = "sha256:b470de64d17156c37e91effc109d3b032b39867000e2c126732fe01d034441f9"}, - {file = "aiohttp-3.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:3f617a48b70f4843d54f52440ea1e58da6bdab07b391a3a6aed8d3b311a4cc04"}, - {file = "aiohttp-3.11.2.tar.gz", hash = "sha256:68d1f46f9387db3785508f5225d3acbc5825ca13d9c29f2b5cce203d5863eb79"}, + {file = "aiohttp-3.11.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3f21c6d1fae17f4466af3796975ab34010db3ac1f0d688272a6ce2f9fa2a4ea5"}, + {file = "aiohttp-3.11.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2041691ac9a4ac5f3ccda419efdbd97f3b25bcc64c5badf57a85a69b8579268"}, + {file = "aiohttp-3.11.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7ad77209639aa7f8d1bd87bd0aa961cac791658c9dd1d32225cbabee95b70bd4"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca5c02fec19113abb7d9df9350471fa1ed25f76ad24be81690c96b3b759da795"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:35c429f0f761619ea659cfe5bed5c26bc62c5e09c2da28b5ee86d006b1a1eb4d"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68f0f8213e891b81800812ec70c58bac3899f4828e7ad14ba5997c26dd88aa6f"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:381c1d8334fb010968dfc8eb1140ed349c5ade9ba20feb0aee2a047d9af0b7a5"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ea7b22c2569007df2c39dbe72b7c7cf4e6f6424b505545c68fde8495a35bcc9"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:50d8784cdc111ed0709debe595be831ebb1f0c536b0840684d02fd12d100a092"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0a7a8915129e6e9b43b5e2f13e0533314462f34e8f8589fb388b8f35becb997e"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c7e0cdfdc6ea4b974c3d546e683bf5a408a8777886c7ec389a780da58a8aa284"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9a23bd19042768281c06858a55ee3d85e572111681e5f5dd68ebd27a6ae1e2af"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:304316862900286574e38dbd58c9c5c25dfd52bcfea16514a00dd741f992871e"}, + {file = "aiohttp-3.11.5-cp312-cp312-win32.whl", hash = "sha256:3e0f4119290d432fa7babfc76cbde4f3e21b826240ba51a6d4fdb82935cf82bd"}, + {file = "aiohttp-3.11.5-cp312-cp312-win_amd64.whl", hash = "sha256:1fe98b92f943b00e1831aece85638af6ca6c699f82625f7a6c64a2543b7a9769"}, + {file = "aiohttp-3.11.5.tar.gz", hash = "sha256:7b857fdad5f95d05bbd27c68cdd549889287dea7fe3376265a8a85d554deec1e"}, ] [[package]] @@ -1593,21 +1593,21 @@ files = [ [[package]] name = "grpcio" -version = "1.67.1" +version = "1.68.0" requires_python = ">=3.8" summary = "HTTP/2-based RPC framework" groups = ["default", "dev"] files = [ - {file = "grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953"}, - {file = "grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e"}, - {file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75"}, - {file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38"}, - {file = "grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78"}, - {file = "grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc"}, - {file = "grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732"}, + {file = "grpcio-1.68.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8af6137cc4ae8e421690d276e7627cfc726d4293f6607acf9ea7260bd8fc3d7d"}, + {file = "grpcio-1.68.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4028b8e9a3bff6f377698587d642e24bd221810c06579a18420a17688e421af7"}, + {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f60fa2adf281fd73ae3a50677572521edca34ba373a45b457b5ebe87c2d01e1d"}, + {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e18589e747c1e70b60fab6767ff99b2d0c359ea1db8a2cb524477f93cdbedf5b"}, + {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0d30f3fee9372796f54d3100b31ee70972eaadcc87314be369360248a3dcffe"}, + {file = "grpcio-1.68.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7e0a3e72c0e9a1acab77bef14a73a416630b7fd2cbd893c0a873edc47c42c8cd"}, + {file = "grpcio-1.68.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a831dcc343440969aaa812004685ed322cdb526cd197112d0db303b0da1e8659"}, + {file = "grpcio-1.68.0-cp312-cp312-win32.whl", hash = "sha256:5a180328e92b9a0050958ced34dddcb86fec5a8b332f5a229e353dafc16cd332"}, + {file = "grpcio-1.68.0-cp312-cp312-win_amd64.whl", hash = "sha256:2bddd04a790b69f7a7385f6a112f46ea0b34c4746f361ebafe9ca0be567c78e9"}, + {file = "grpcio-1.68.0.tar.gz", hash = "sha256:7e7483d39b4a4fddb9906671e9ea21aaad4f031cdfc349fec76bdfa1e404543a"}, ] [[package]] @@ -2717,29 +2717,29 @@ files = [ [[package]] name = "pyjwt" -version = "2.9.0" -requires_python = ">=3.8" +version = "2.10.0" +requires_python = ">=3.9" summary = "JSON Web Token implementation in Python" groups = ["default"] files = [ - {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, - {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, + {file = "PyJWT-2.10.0-py3-none-any.whl", hash = "sha256:543b77207db656de204372350926bed5a86201c4cbff159f623f79c7bb487a15"}, + {file = "pyjwt-2.10.0.tar.gz", hash = "sha256:7628a7eb7938959ac1b26e819a1df0fd3259505627b575e4bad6d08f76db695c"}, ] [[package]] name = "pyjwt" -version = "2.9.0" +version = "2.10.0" extras = ["crypto"] -requires_python = ">=3.8" +requires_python = ">=3.9" summary = "JSON Web Token implementation in Python" groups = ["default"] dependencies = [ "cryptography>=3.4.0", - "pyjwt==2.9.0", + "pyjwt==2.10.0", ] files = [ - {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, - {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, + {file = "PyJWT-2.10.0-py3-none-any.whl", hash = "sha256:543b77207db656de204372350926bed5a86201c4cbff159f623f79c7bb487a15"}, + {file = "pyjwt-2.10.0.tar.gz", hash = "sha256:7628a7eb7938959ac1b26e819a1df0fd3259505627b575e4bad6d08f76db695c"}, ] [[package]] @@ -3166,13 +3166,13 @@ files = [ [[package]] name = "slack-sdk" -version = "3.33.3" +version = "3.33.4" requires_python = ">=3.6" summary = "The Slack API Platform SDK for Python" groups = ["default"] files = [ - {file = "slack_sdk-3.33.3-py2.py3-none-any.whl", hash = "sha256:0515fb93cd03b18de61f876a8304c4c3cef4dd3c2a3bad62d7394d2eb5a3c8e6"}, - {file = "slack_sdk-3.33.3.tar.gz", hash = "sha256:4cc44c9ffe4bb28a01fbe3264c2f466c783b893a4eca62026ab845ec7c176ff1"}, + {file = "slack_sdk-3.33.4-py2.py3-none-any.whl", hash = "sha256:9f30cb3c9c07b441c49d53fc27f9f1837ad1592a7e9d4ca431f53cdad8826cc6"}, + {file = "slack_sdk-3.33.4.tar.gz", hash = "sha256:5e109847f6b6a22d227609226ba4ed936109dc00675bddeb7e0bee502d3ee7e0"}, ] [[package]] @@ -3255,49 +3255,49 @@ files = [ [[package]] name = "sqlglot" -version = "25.30.0" +version = "25.31.4" requires_python = ">=3.7" summary = "An easily customizable SQL parser and transpiler" groups = ["default"] files = [ - {file = "sqlglot-25.30.0-py3-none-any.whl", hash = "sha256:2008e51f0d352d6bb967be71767810abdd396e84f2d07dd7816b75775323246b"}, - {file = "sqlglot-25.30.0.tar.gz", hash = "sha256:f39ad054750f3be164e69ea74c784b5bcbb237c49416695ba95df696cd81c9d5"}, + {file = "sqlglot-25.31.4-py3-none-any.whl", hash = "sha256:fa4f8042e843d7fa57dadf1329e4735601cae4d84392554acc6a2af0a66cf8e8"}, + {file = "sqlglot-25.31.4.tar.gz", hash = "sha256:1cabcbb835fc49c485d4ea84ecdb571f244b6ba78acdfabca9492dbd39b8f091"}, ] [[package]] name = "sqlglot" -version = "25.30.0" +version = "25.31.4" extras = ["rs"] requires_python = ">=3.7" summary = "An easily customizable SQL parser and transpiler" groups = ["default"] dependencies = [ - "sqlglot==25.30.0", - "sqlglotrs==0.2.13", + "sqlglot==25.31.4", + "sqlglotrs==0.2.14", ] files = [ - {file = "sqlglot-25.30.0-py3-none-any.whl", hash = "sha256:2008e51f0d352d6bb967be71767810abdd396e84f2d07dd7816b75775323246b"}, - {file = "sqlglot-25.30.0.tar.gz", hash = "sha256:f39ad054750f3be164e69ea74c784b5bcbb237c49416695ba95df696cd81c9d5"}, + {file = "sqlglot-25.31.4-py3-none-any.whl", hash = "sha256:fa4f8042e843d7fa57dadf1329e4735601cae4d84392554acc6a2af0a66cf8e8"}, + {file = "sqlglot-25.31.4.tar.gz", hash = "sha256:1cabcbb835fc49c485d4ea84ecdb571f244b6ba78acdfabca9492dbd39b8f091"}, ] [[package]] name = "sqlglotrs" -version = "0.2.13" +version = "0.2.14" requires_python = ">=3.7" summary = "An easily customizable SQL parser and transpiler" groups = ["default"] files = [ - {file = "sqlglotrs-0.2.13-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1bef4dae451f25d7dd7ae5fb76e7a23d9d6951c8a1e6f83558d07d947ea43747"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8aec6c9d5d3db9009c19cdc513c3bb26ffd74010d2446c297afdc9c39abc065f"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cf22beeed2f45a0f8ccd5a6f1e05ccef2ed363628505087809df284bb67fc43"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26aafec54ce1619c0b192747665c301a5f3e064bd1df17a213ad90cfff9d69d4"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:387dfb3e10ff67c6dc7dd86137e6c182ebf478a4b137b361486d1bb8af7505f9"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e11b5a00ecdbb35f6eccb02e7d901441d5db6bb5e4a655751dc5236e4dd20090"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e848c49dc1cf1530c5e4b5c44a96b272943dbb3fb5d0e59baecc11bfc4d1c6b"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f321ff2cdcb0de32ac1c9ebe4b9b8c0e121541c224459563a67135a9f1410b79"}, - {file = "sqlglotrs-0.2.13-cp312-none-win32.whl", hash = "sha256:a632366a9ba92a5a2d9194febbb5324ecc34346437277247698b8842517091e2"}, - {file = "sqlglotrs-0.2.13-cp312-none-win_amd64.whl", hash = "sha256:9670480c6c5e4260ef5a016edd23db10a376e5db5fe9cdf360bbd61ba920d3c8"}, - {file = "sqlglotrs-0.2.13.tar.gz", hash = "sha256:9bcc16c2264471d049f465dea8efbb946d174251ca41e52950e2a03e85b87cdc"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e286c2549064fd8e01e66f90c8eccf353ff9355033d9a53a486a8153fe046bd4"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83aad4d7a659b4f32d30983758f9d095fff04a888834f916ec19d84b80831161"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a28b632abb510a67755fa9a6bda2e901a99dc365955ac1c60d2348f31d1c9636"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:65c4e06aac5fbcf08fa77cbf026c5980e73a045a3910992f8a3117b33372f760"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3433d4ab25caba64068da6fd2c8b4dbcdb4b60583e52fc10e01ac5df60bf70c"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a279028c1af93f6fdf7ad281ea82a418049395dfa5ca7c177458205b6018b258"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef4f7ccfef4d3b1bf07cfd3d7e3b1c07ed98e921ab20cda5d009da5194e48b7"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dc14c4580627da6f6823cef187fd1a4003ed624c1ff1fc9e8efef58098ca5f0"}, + {file = "sqlglotrs-0.2.14-cp312-none-win32.whl", hash = "sha256:35b82c52e178e9ff83f51661c8656a5378583e83688839865d87a17d5e77fc0d"}, + {file = "sqlglotrs-0.2.14-cp312-none-win_amd64.whl", hash = "sha256:266c692265032fd23062b26e5501fb3cd07e818b8b37526788d3719272d002a2"}, + {file = "sqlglotrs-0.2.14.tar.gz", hash = "sha256:16dfc6c40af10d79c507afdbec9cf98e383a8296da677b614239781816c5848b"}, ] [[package]] @@ -3326,7 +3326,7 @@ files = [ [[package]] name = "starlette" -version = "0.41.2" +version = "0.41.3" requires_python = ">=3.8" summary = "The little ASGI library that shines." groups = ["dev"] @@ -3335,8 +3335,8 @@ dependencies = [ "typing-extensions>=3.10.0; python_version < \"3.10\"", ] files = [ - {file = "starlette-0.41.2-py3-none-any.whl", hash = "sha256:fbc189474b4731cf30fcef52f18a8d070e3f3b46c6a04c97579e85e6ffca942d"}, - {file = "starlette-0.41.2.tar.gz", hash = "sha256:9834fd799d1a87fd346deb76158668cfa0b0d56f85caefe8268e2d97c3468b62"}, + {file = "starlette-0.41.3-py3-none-any.whl", hash = "sha256:44cedb2b7c77a9de33a8b74b2b90e9f50d11fcf25d8270ea525ad71a25374ff7"}, + {file = "starlette-0.41.3.tar.gz", hash = "sha256:0e4ab3d16522a255be6b28260b938eae2482f98ce5cc934cb08dce8dc3ba5835"}, ] [[package]] @@ -3499,7 +3499,7 @@ files = [ [[package]] name = "typer" -version = "0.13.0" +version = "0.13.1" requires_python = ">=3.7" summary = "Typer, build great CLIs. Easy to code. Based on Python type hints." groups = ["default"] @@ -3510,19 +3510,19 @@ dependencies = [ "typing-extensions>=3.7.4.3", ] files = [ - {file = "typer-0.13.0-py3-none-any.whl", hash = "sha256:d85fe0b777b2517cc99c8055ed735452f2659cd45e451507c76f48ce5c1d00e2"}, - {file = "typer-0.13.0.tar.gz", hash = "sha256:f1c7198347939361eec90139ffa0fd8b3df3a2259d5852a0f7400e476d95985c"}, + {file = "typer-0.13.1-py3-none-any.whl", hash = "sha256:5b59580fd925e89463a29d363e0a43245ec02765bde9fb77d39e5d0f29dd7157"}, + {file = "typer-0.13.1.tar.gz", hash = "sha256:9d444cb96cc268ce6f8b94e13b4335084cef4c079998a9f4851a90229a3bd25c"}, ] [[package]] name = "types-setuptools" -version = "75.3.0.20241112" +version = "75.5.0.20241121" requires_python = ">=3.8" summary = "Typing stubs for setuptools" groups = ["default"] files = [ - {file = "types-setuptools-75.3.0.20241112.tar.gz", hash = "sha256:f9e1ebd17a56f606e16395c4ee4efa1cdc394b9a2a0ee898a624058b4b62ef8f"}, - {file = "types_setuptools-75.3.0.20241112-py3-none-any.whl", hash = "sha256:78cb5fef4a6056d2f37114d27da90f4655a306e4e38042d7034a8a880bc3f5dd"}, + {file = "types_setuptools-75.5.0.20241121-py3-none-any.whl", hash = "sha256:8ea9489a8a9b98d4eb182d302117cbe1d5ae7c01a864e091b21a9107c2ba186c"}, + {file = "types_setuptools-75.5.0.20241121.tar.gz", hash = "sha256:47b261411cfca811681183f5371656ebc3d8e722b8ee857276fdb5402e5f6fb3"}, ] [[package]] @@ -3760,7 +3760,7 @@ files = [ [[package]] name = "yarl" -version = "1.17.1" +version = "1.17.2" requires_python = ">=3.9" summary = "Yet another URL library" groups = ["default", "dev"] @@ -3770,24 +3770,24 @@ dependencies = [ "propcache>=0.2.0", ] files = [ - {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:327828786da2006085a4d1feb2594de6f6d26f8af48b81eb1ae950c788d97f61"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc353841428d56b683a123a813e6a686e07026d6b1c5757970a877195f880c2d"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c73df5b6e8fabe2ddb74876fb82d9dd44cbace0ca12e8861ce9155ad3c886139"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdff5e0995522706c53078f531fb586f56de9c4c81c243865dd5c66c132c3b5"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06157fb3c58f2736a5e47c8fcbe1afc8b5de6fb28b14d25574af9e62150fcaac"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1654ec814b18be1af2c857aa9000de7a601400bd4c9ca24629b18486c2e35463"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6595c852ca544aaeeb32d357e62c9c780eac69dcd34e40cae7b55bc4fb1147"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459e81c2fb920b5f5df744262d1498ec2c8081acdcfe18181da44c50f51312f7"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e48cdb8226644e2fbd0bdb0a0f87906a3db07087f4de77a1b1b1ccfd9e93685"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d9b6b28a57feb51605d6ae5e61a9044a31742db557a3b851a74c13bc61de5172"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e594b22688d5747b06e957f1ef822060cb5cb35b493066e33ceac0cf882188b7"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5f236cb5999ccd23a0ab1bd219cfe0ee3e1c1b65aaf6dd3320e972f7ec3a39da"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a2a64e62c7a0edd07c1c917b0586655f3362d2c2d37d474db1a509efb96fea1c"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d0eea830b591dbc68e030c86a9569826145df485b2b4554874b07fea1275a199"}, - {file = "yarl-1.17.1-cp312-cp312-win32.whl", hash = "sha256:46ddf6e0b975cd680eb83318aa1d321cb2bf8d288d50f1754526230fcf59ba96"}, - {file = "yarl-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:117ed8b3732528a1e41af3aa6d4e08483c2f0f2e3d3d7dca7cf538b3516d93df"}, - {file = "yarl-1.17.1-py3-none-any.whl", hash = "sha256:f1790a4b1e8e8e028c391175433b9c8122c39b46e1663228158e61e6f915bf06"}, - {file = "yarl-1.17.1.tar.gz", hash = "sha256:067a63fcfda82da6b198fa73079b1ca40b7c9b7994995b6ee38acda728b64d47"}, + {file = "yarl-1.17.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:dd90238d3a77a0e07d4d6ffdebc0c21a9787c5953a508a2231b5f191455f31e9"}, + {file = "yarl-1.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c74f0b0472ac40b04e6d28532f55cac8090e34c3e81f118d12843e6df14d0909"}, + {file = "yarl-1.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d486ddcaca8c68455aa01cf53d28d413fb41a35afc9f6594a730c9779545876"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25b7e93f5414b9a983e1a6c1820142c13e1782cc9ed354c25e933aebe97fcf2"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a0baff7827a632204060f48dca9e63fbd6a5a0b8790c1a2adfb25dc2c9c0d50"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:460024cacfc3246cc4d9f47a7fc860e4fcea7d1dc651e1256510d8c3c9c7cde0"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5870d620b23b956f72bafed6a0ba9a62edb5f2ef78a8849b7615bd9433384171"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2941756754a10e799e5b87e2319bbec481ed0957421fba0e7b9fb1c11e40509f"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9611b83810a74a46be88847e0ea616794c406dbcb4e25405e52bff8f4bee2d0a"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cd7e35818d2328b679a13268d9ea505c85cd773572ebb7a0da7ccbca77b6a52e"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6b981316fcd940f085f646b822c2ff2b8b813cbd61281acad229ea3cbaabeb6b"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:688058e89f512fb7541cb85c2f149c292d3fa22f981d5a5453b40c5da49eb9e8"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:56afb44a12b0864d17b597210d63a5b88915d680f6484d8d202ed68ade38673d"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:17931dfbb84ae18b287279c1f92b76a3abcd9a49cd69b92e946035cff06bcd20"}, + {file = "yarl-1.17.2-cp312-cp312-win32.whl", hash = "sha256:ff8d95e06546c3a8c188f68040e9d0360feb67ba8498baf018918f669f7bc39b"}, + {file = "yarl-1.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:4c840cc11163d3c01a9d8aad227683c48cd3e5be5a785921bcc2a8b4b758c4f3"}, + {file = "yarl-1.17.2-py3-none-any.whl", hash = "sha256:dd7abf4f717e33b7487121faf23560b3a50924f80e4bef62b22dab441ded8f3b"}, + {file = "yarl-1.17.2.tar.gz", hash = "sha256:753eaaa0c7195244c84b5cc159dc8204b7fd99f716f11198f999f2332a86b178"}, ] [[package]] diff --git a/requirements.txt b/requirements.txt index 2290bee02f..73f92223b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ agate==1.9.1 aiohappyeyeballs==2.4.3 -aiohttp==3.11.2 +aiohttp==3.11.5 aiosignal==1.3.1 alembic==1.14.0 annotated-types==0.7.0 @@ -72,7 +72,7 @@ google-resumable-media==2.7.2 googleapis-common-protos[grpc]==1.66.0 greenlet==3.1.1; (platform_machine == "win32" or platform_machine == "WIN32" or platform_machine == "AMD64" or platform_machine == "amd64" or platform_machine == "x86_64" or platform_machine == "ppc64le" or platform_machine == "aarch64") and python_version < "3.13" grpc-google-iam-v1==0.13.1 -grpcio==1.67.1 +grpcio==1.68.0 grpcio-health-checking==1.62.3 grpcio-status==1.62.3 gspread==6.1.4 @@ -137,7 +137,7 @@ pycryptodome==3.21.0 pydantic==2.9.2 pydantic-core==2.23.4 pygments==2.18.0 -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.0 pynacl==1.5.0 pyparsing==3.2.0; python_version > "3.0" pypdf==5.1.0 @@ -164,14 +164,14 @@ setuptools==75.5.0 shellingham==1.5.4 simplejson==3.19.3 six==1.16.0 -slack-sdk==3.33.3 +slack-sdk==3.33.4 sling==1.2.22 sling-linux-amd64==1.2.22 smmap==5.0.1 soupsieve==2.6 sqlalchemy==2.0.36 -sqlglot[rs]==25.30.0 -sqlglotrs==0.2.13 +sqlglot[rs]==25.31.4 +sqlglotrs==0.2.14 sqlparse==0.5.2 sshtunnel==0.4.0 structlog==24.4.0 @@ -186,8 +186,8 @@ tomlkit==0.13.2 toposort==1.10 tqdm==4.67.0 typeguard==4.4.1 -typer==0.13.0 -types-setuptools==75.3.0.20241112 +typer==0.13.1 +types-setuptools==75.5.0.20241121 typing-extensions==4.12.2 tzdata==2024.2 universal-pathlib==0.2.5; python_version >= "3.12" @@ -198,5 +198,5 @@ wcwidth==0.2.13 websocket-client==1.8.0 wheel==0.45.0 win-precise-time==1.4.2; os_name == "nt" -yarl==1.17.1 +yarl==1.17.2 zipp==3.21.0 From 3360396736d72c83652c02889ce8d89222582ee8 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:37:00 +0000 Subject: [PATCH 20/20] build: update uv lock --- uv.lock | 455 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 446 insertions(+), 9 deletions(-) diff --git a/uv.lock b/uv.lock index f5726404ab..dc12d1130b 100644 --- a/uv.lock +++ b/uv.lock @@ -19,6 +19,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/53/89b197cb472a3175d73384761a3413fd58e6b65a794c1102d148b8de87bd/agate-1.9.1-py2.py3-none-any.whl", hash = "sha256:1cf329510b3dde07c4ad1740b7587c9c679abc3dcd92bb1107eabc10c2e03c50", size = 95085 }, ] +[[package]] +name = "aiohappyeyeballs" +version = "2.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/69/2f6d5a019bd02e920a3417689a89887b39ad1e350b562f9955693d900c40/aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586", size = 21809 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/d8/120cd0fe3e8530df0539e71ba9683eade12cae103dd7543e50d15f737917/aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572", size = 14742 }, +] + +[[package]] +name = "aiohttp" +version = "3.11.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/cb/f9bb10e0cf6f01730b27d370b10cc15822bea4395acd687abc8cc5fed3ed/aiohttp-3.11.7.tar.gz", hash = "sha256:01a8aca4af3da85cea5c90141d23f4b0eee3cbecfd33b029a45a80f28c66c668", size = 7666482 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/1e/2e96b2526c590dcb99db0b94ac4f9b927ecc07f94735a8a941dee143d48b/aiohttp-3.11.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4bb7493c3e3a36d3012b8564bd0e2783259ddd7ef3a81a74f0dbfa000fce48b7", size = 702326 }, + { url = "https://files.pythonhosted.org/packages/b5/ce/b5d7f3e68849f1f5e0b85af4ac9080b9d3c0a600857140024603653c2209/aiohttp-3.11.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e143b0ef9cb1a2b4f74f56d4fbe50caa7c2bb93390aff52f9398d21d89bc73ea", size = 461944 }, + { url = "https://files.pythonhosted.org/packages/28/fa/f4d98db1b7f8f0c3f74bdbd6d0d98cfc89984205cd33f1b8ee3f588ee5ad/aiohttp-3.11.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f7c58a240260822dc07f6ae32a0293dd5bccd618bb2d0f36d51c5dbd526f89c0", size = 454348 }, + { url = "https://files.pythonhosted.org/packages/04/f0/c238dda5dc9a3d12b76636e2cf0ea475890ac3a1c7e4ff0fd6c3cea2fc2d/aiohttp-3.11.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d20cfe63a1c135d26bde8c1d0ea46fd1200884afbc523466d2f1cf517d1fe33", size = 1678795 }, + { url = "https://files.pythonhosted.org/packages/79/ee/3a18f792247e6d95dba13aaedc9dc317c3c6e75f4b88c2dd4b960d20ad2f/aiohttp-3.11.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12e4d45847a174f77b2b9919719203769f220058f642b08504cf8b1cf185dacf", size = 1734411 }, + { url = "https://files.pythonhosted.org/packages/f5/79/3eb84243087a9a32cae821622c935107b4b55a5b21b76772e8e6c41092e9/aiohttp-3.11.7-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf4efa2d01f697a7dbd0509891a286a4af0d86902fc594e20e3b1712c28c0106", size = 1788959 }, + { url = "https://files.pythonhosted.org/packages/91/93/ad77782c5edfa17aafc070bef978fbfb8459b2f150595ffb01b559c136f9/aiohttp-3.11.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ee6a4cdcbf54b8083dc9723cdf5f41f722c00db40ccf9ec2616e27869151129", size = 1687463 }, + { url = "https://files.pythonhosted.org/packages/ba/48/db35bd21b7877efa0be5f28385d8978c55323c5ce7685712e53f3f6c0bd9/aiohttp-3.11.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6095aaf852c34f42e1bd0cf0dc32d1e4b48a90bfb5054abdbb9d64b36acadcb", size = 1618374 }, + { url = "https://files.pythonhosted.org/packages/ba/77/30f87db55c79fd145ed5fd15b92f2e820ce81065d41ae437797aaa550e3b/aiohttp-3.11.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1cf03d27885f8c5ebf3993a220cc84fc66375e1e6e812731f51aab2b2748f4a6", size = 1637021 }, + { url = "https://files.pythonhosted.org/packages/af/76/10b188b78ee18d0595af156d6a238bc60f9d8571f0f546027eb7eaf65b25/aiohttp-3.11.7-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:1a17f6a230f81eb53282503823f59d61dff14fb2a93847bf0399dc8e87817307", size = 1650792 }, + { url = "https://files.pythonhosted.org/packages/fa/33/4411bbb8ad04c47d0f4c7bd53332aaf350e49469cf6b65b132d4becafe27/aiohttp-3.11.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:481f10a1a45c5f4c4a578bbd74cff22eb64460a6549819242a87a80788461fba", size = 1696248 }, + { url = "https://files.pythonhosted.org/packages/fe/2d/6135d0dc1851a33d3faa937b20fef81340bc95e8310536d4c7f1f8ecc026/aiohttp-3.11.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:db37248535d1ae40735d15bdf26ad43be19e3d93ab3f3dad8507eb0f85bb8124", size = 1729188 }, + { url = "https://files.pythonhosted.org/packages/f5/76/a57ceff577ae26fe9a6f31ac799bc638ecf26e4acdf04295290b9929b349/aiohttp-3.11.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9d18a8b44ec8502a7fde91446cd9c9b95ce7c49f1eacc1fb2358b8907d4369fd", size = 1690038 }, + { url = "https://files.pythonhosted.org/packages/4b/81/b20e09003b6989a7f23a721692137a6143420a151063c750ab2a04878e3c/aiohttp-3.11.7-cp312-cp312-win32.whl", hash = "sha256:3d1c9c15d3999107cbb9b2d76ca6172e6710a12fda22434ee8bd3f432b7b17e8", size = 409887 }, + { url = "https://files.pythonhosted.org/packages/b7/0b/607c98bff1d07bb21e0c39e7711108ef9ff4f2a361a3ec1ce8dce93623a5/aiohttp-3.11.7-cp312-cp312-win_amd64.whl", hash = "sha256:018f1b04883a12e77e7fc161934c0f298865d3a484aea536a6a2ca8d909f0ba0", size = 436462 }, +] + +[[package]] +name = "aiosignal" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/67/0952ed97a9793b4958e5736f6d2b346b414a2cd63e82d05940032f45b32f/aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc", size = 19422 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/ac/a7305707cb852b7e16ff80eaf5692309bde30e2b1100a1fcacdc8f731d97/aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17", size = 7617 }, +] + [[package]] name = "alembic" version = "1.14.0" @@ -73,6 +126,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/be/a606a6701d491cfae75583c80a6583f8abe9c36c0b9666e867e7cdd62fe8/argcomplete-3.5.1-py3-none-any.whl", hash = "sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363", size = 43498 }, ] +[[package]] +name = "astunparse" +version = "1.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, + { name = "wheel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8", size = 12732 }, +] + [[package]] name = "attrs" version = "24.2.0" @@ -476,6 +542,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/7b/54714279fc09f66bfc2feec4156d3034b485320ba7585ba5eacc22717659/dagster_dbt-0.25.3-py3-none-any.whl", hash = "sha256:c7db9478acf1632c314f4591f9572e179501d5a91469ad19bca4e2cc7a60309e", size = 85672 }, ] +[[package]] +name = "dagster-embedded-elt" +version = "0.25.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dagster" }, + { name = "dlt" }, + { name = "sling" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/83/b0751b63ffdeb07df351580c5256c44fad8c8e7c65e4f8deba99e1a92753/dagster-embedded-elt-0.25.3.tar.gz", hash = "sha256:3377c99764089ee3b8a6b9181f6132489dab1052ea17a3dfd8a7d38062cbed8f", size = 22913 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/9b/aca1bc72d1dbfab736d1b1297063216c90085d5f06587693cf5e591d6db0/dagster_embedded_elt-0.25.3-py3-none-any.whl", hash = "sha256:1091cd0e9b2071cc4d971397de0c6ed0d4be9d2c2ffa82a1b7a58793dda90bd8", size = 27809 }, +] + [[package]] name = "dagster-fivetran" version = "0.25.3" @@ -789,6 +869,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 }, ] +[[package]] +name = "dlt" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "astunparse" }, + { name = "click" }, + { name = "fsspec" }, + { name = "gitpython" }, + { name = "giturlparse" }, + { name = "hexbytes" }, + { name = "humanize" }, + { name = "jsonpath-ng" }, + { name = "makefun" }, + { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, + { name = "packaging" }, + { name = "pathvalidate" }, + { name = "pendulum" }, + { name = "pluggy" }, + { name = "pytz" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "requirements-parser" }, + { name = "semver" }, + { name = "setuptools" }, + { name = "simplejson" }, + { name = "tenacity" }, + { name = "tomlkit" }, + { name = "typing-extensions" }, + { name = "tzdata" }, + { name = "win-precise-time", marker = "os_name == 'nt'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/1f/55ad7f79dd3fda37c3f2b7e86a7d1e84b9a4102f93cde077460123500bf8/dlt-1.4.0.tar.gz", hash = "sha256:75208448dc11dd501cf15d76742368816fef8e1b22fb07417f69d5ceb720b324", size = 668769 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/3d/446645d009501bc719f18d27b3c441c12c992ac58e8e2157c39d4c7e7132/dlt-1.4.0-py3-none-any.whl", hash = "sha256:c3a69e4067581bf0335796bec62d58058ff1f11249f16b699d6657544b126247", size = 846914 }, +] + [[package]] name = "dnspython" version = "2.7.0" @@ -852,6 +970,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/f8/feced7779d755758a52d1f6635d990b8d98dc0a29fa568bbe0625f18fdf3/filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0", size = 16163 }, ] +[[package]] +name = "frozenlist" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/ed/0f4cec13a93c02c47ec32d81d11c0c1efbadf4a471e3f3ce7cad366cbbd3/frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817", size = 39930 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/73/fa6d1a96ab7fd6e6d1c3500700963eab46813847f01ef0ccbaa726181dd5/frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21", size = 94026 }, + { url = "https://files.pythonhosted.org/packages/ab/04/ea8bf62c8868b8eada363f20ff1b647cf2e93377a7b284d36062d21d81d1/frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d", size = 54150 }, + { url = "https://files.pythonhosted.org/packages/d0/9a/8e479b482a6f2070b26bda572c5e6889bb3ba48977e81beea35b5ae13ece/frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e", size = 51927 }, + { url = "https://files.pythonhosted.org/packages/e3/12/2aad87deb08a4e7ccfb33600871bbe8f0e08cb6d8224371387f3303654d7/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a", size = 282647 }, + { url = "https://files.pythonhosted.org/packages/77/f2/07f06b05d8a427ea0060a9cef6e63405ea9e0d761846b95ef3fb3be57111/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a", size = 289052 }, + { url = "https://files.pythonhosted.org/packages/bd/9f/8bf45a2f1cd4aa401acd271b077989c9267ae8463e7c8b1eb0d3f561b65e/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee", size = 291719 }, + { url = "https://files.pythonhosted.org/packages/41/d1/1f20fd05a6c42d3868709b7604c9f15538a29e4f734c694c6bcfc3d3b935/frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6", size = 267433 }, + { url = "https://files.pythonhosted.org/packages/af/f2/64b73a9bb86f5a89fb55450e97cd5c1f84a862d4ff90d9fd1a73ab0f64a5/frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e", size = 283591 }, + { url = "https://files.pythonhosted.org/packages/29/e2/ffbb1fae55a791fd6c2938dd9ea779509c977435ba3940b9f2e8dc9d5316/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9", size = 273249 }, + { url = "https://files.pythonhosted.org/packages/2e/6e/008136a30798bb63618a114b9321b5971172a5abddff44a100c7edc5ad4f/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039", size = 271075 }, + { url = "https://files.pythonhosted.org/packages/ae/f0/4e71e54a026b06724cec9b6c54f0b13a4e9e298cc8db0f82ec70e151f5ce/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784", size = 285398 }, + { url = "https://files.pythonhosted.org/packages/4d/36/70ec246851478b1c0b59f11ef8ade9c482ff447c1363c2bd5fad45098b12/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631", size = 294445 }, + { url = "https://files.pythonhosted.org/packages/37/e0/47f87544055b3349b633a03c4d94b405956cf2437f4ab46d0928b74b7526/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f", size = 280569 }, + { url = "https://files.pythonhosted.org/packages/f9/7c/490133c160fb6b84ed374c266f42800e33b50c3bbab1652764e6e1fc498a/frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8", size = 44721 }, + { url = "https://files.pythonhosted.org/packages/b1/56/4e45136ffc6bdbfa68c29ca56ef53783ef4c2fd395f7cbf99a2624aa9aaa/frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f", size = 51329 }, + { url = "https://files.pythonhosted.org/packages/c6/c8/a5be5b7550c10858fcf9b0ea054baccab474da77d37f1e828ce043a3a5d4/frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3", size = 11901 }, +] + [[package]] name = "fsspec" version = "2024.10.0" @@ -870,6 +1012,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/5c/e226de133afd8bb267ec27eead9ae3d784b95b39a287ed404caab39a5f50/genson-1.3.0-py3-none-any.whl", hash = "sha256:468feccd00274cc7e4c09e84b08704270ba8d95232aa280f65b986139cec67f7", size = 21470 }, ] +[[package]] +name = "gitdb" +version = "4.0.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/0d/bbb5b5ee188dec84647a4664f3e11b06ade2bde568dbd489d9d64adef8ed/gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b", size = 394469 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/5b/8f0c4a5bb9fd491c277c21eff7ccae71b47d43c4446c9d0c6cff2fe8c2c4/gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4", size = 62721 }, +] + [[package]] name = "github3-py" version = "4.0.1" @@ -885,6 +1039,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/ad/2394d4fb542574678b0ba342daf734d4d811768da3c2ee0c84d509dcb26c/github3.py-4.0.1-py3-none-any.whl", hash = "sha256:a89af7de25650612d1da2f0609622bcdeb07ee8a45a1c06b2d16a05e4234e753", size = 151800 }, ] +[[package]] +name = "gitpython" +version = "3.1.43" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/a1/106fd9fa2dd989b6fb36e5893961f82992cf676381707253e0bf93eb1662/GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c", size = 214149 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/bd/cc3a402a6439c15c3d4294333e13042b915bbeab54edc457c723931fed3f/GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff", size = 207337 }, +] + +[[package]] +name = "giturlparse" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/5f/543dc54c82842376139748226e5aa61eb95093992f63dd495af9c6b4f076/giturlparse-0.12.0.tar.gz", hash = "sha256:c0fff7c21acc435491b1779566e038757a205c1ffdcb47e4f81ea52ad8c3859a", size = 14907 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/94/c6ff3388b8e3225a014e55aed957188639aa0966443e0408d38f0c9614a7/giturlparse-0.12.0-py2.py3-none-any.whl", hash = "sha256:412b74f2855f1da2fefa89fd8dde62df48476077a72fc19b62039554d27360eb", size = 15752 }, +] + [[package]] name = "google-api-core" version = "2.23.0" @@ -1230,6 +1405,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259 }, ] +[[package]] +name = "hexbytes" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/51/06836a542b773bfc60ab871fa08d4a7963e7df6754ca57169e2654287cc1/hexbytes-1.2.1.tar.gz", hash = "sha256:515f00dddf31053db4d0d7636dd16061c1d896c3109b8e751005db4ca46bcca7", size = 7722 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/c6/20f25ea73e4ceffb3eb4e38347f2992cb25e5ff6eb644d52e753a7a72f57/hexbytes-1.2.1-py3-none-any.whl", hash = "sha256:e64890b203a31f4a23ef11470ecfcca565beaee9198df623047df322b757471a", size = 5160 }, +] + [[package]] name = "httplib2" version = "0.22.0" @@ -1269,6 +1453,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794 }, ] +[[package]] +name = "humanize" +version = "4.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/40/64a912b9330786df25e58127194d4a5a7441f818b400b155e748a270f924/humanize-4.11.0.tar.gz", hash = "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be", size = 80374 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/75/4bc3e242ad13f2e6c12e0b0401ab2c5e5c6f0d7da37ec69bc808e24e0ccb/humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0", size = 128055 }, +] + [[package]] name = "idna" version = "3.10" @@ -1350,6 +1543,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/29/df4b9b42f2be0b623cbd5e2140cafcaa2bef0759a00b7b70104dcfe2fb51/joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6", size = 301817 }, ] +[[package]] +name = "jsonpath-ng" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ply" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/86/08646239a313f895186ff0a4573452038eed8c86f54380b3ebac34d32fb2/jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c", size = 37838 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/5a/73ecb3d82f8615f32ccdadeb9356726d6cae3a4bbc840b437ceb95708063/jsonpath_ng-1.7.0-py3-none-any.whl", hash = "sha256:f3d7f9e848cba1b6da28c55b1c26ff915dc9e0b1ba7e752a53d6da8d5cbd00b6", size = 30105 }, +] + [[package]] name = "jsonschema" version = "4.23.0" @@ -1426,6 +1631,15 @@ version = "1.5.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/2f/d9/16ac346f7c0102835814cc9e5b684aaadea101560bb932a2403bd26b2320/Logbook-1.5.3.tar.gz", hash = "sha256:66f454ada0f56eae43066f604a222b09893f98c1adc18df169710761b8f32fe8", size = 85783 } +[[package]] +name = "makefun" +version = "1.15.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/00/62966769824620717a3c2d76b1d442489648398b599bdcd490af13bff101/makefun-1.15.6.tar.gz", hash = "sha256:26bc63442a6182fb75efed8b51741dd2d1db2f176bec8c64e20a586256b8f149", size = 72583 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/a1/3e145759e776c8866488a71270c399bf7c4e554551ac2e247aa0a18a0596/makefun-1.15.6-py2.py3-none-any.whl", hash = "sha256:e69b870f0bb60304765b1e3db576aaecf2f9b3e5105afe8cfeff8f2afe6ad067", size = 22946 }, +] + [[package]] name = "mako" version = "1.3.6" @@ -1731,6 +1945,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, ] +[[package]] +name = "pathvalidate" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/8c/8713d8dcd8e357b9358695b441ee974580a8addfaea4f01437df07577052/pathvalidate-3.2.1.tar.gz", hash = "sha256:f5d07b1e2374187040612a1fcd2bcb2919f8db180df254c9581bb90bf903377d", size = 59263 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/5e/76a9d08b4b4e4583f269cb9f64de267f9aeae0dacef23307f53a14211716/pathvalidate-3.2.1-py3-none-any.whl", hash = "sha256:9a6255eb8f63c9e2135b9be97a5ce08f10230128c4ae7b3e935378b82b22c4c9", size = 23833 }, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "time-machine", marker = "implementation_name != 'pypy'" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/fe/27c7438c6ac8b8f8bef3c6e571855602ee784b85d072efddfff0ceb1cd77/pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e", size = 84524 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/37/17c8f0e7481a32f21b9002dd68912a8813f2c1d77b984e00af56eb9ae31b/pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7", size = 362284 }, + { url = "https://files.pythonhosted.org/packages/12/e6/08f462f6ea87e2159f19b43ff88231d26e02bda31c10bcb29290a617ace4/pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc", size = 352964 }, + { url = "https://files.pythonhosted.org/packages/47/29/b6877f6b53b91356c2c56d19ddab17b165ca994ad1e57b32c089e79f3fb5/pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37", size = 335848 }, + { url = "https://files.pythonhosted.org/packages/2b/77/62ca666f30b2558342deadda26290a575459a7b59248ea1e978b84175227/pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319", size = 362215 }, + { url = "https://files.pythonhosted.org/packages/e0/29/ce37593f5ea51862c60dadf4e863d604f954478b3abbcc60a14dc05e242c/pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5", size = 448673 }, + { url = "https://files.pythonhosted.org/packages/72/6a/68a8c7b8f1977d89aabfd0e2becb0921e5515dfb365097e98a522334a151/pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f", size = 384891 }, + { url = "https://files.pythonhosted.org/packages/30/e6/edd699300f47a3c53c0d8ed26e905b9a31057c3646211e58cc540716a440/pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518", size = 559558 }, + { url = "https://files.pythonhosted.org/packages/d4/97/95a44aa5e1763d3a966551ed0e12f56508d8dfcc60e1f0395909b6a08626/pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9", size = 558240 }, + { url = "https://files.pythonhosted.org/packages/9a/91/fcd992eb36b77ab43f2cf44307b72c01a6fbb27f55c1bb2d4af30e9a6cb7/pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5", size = 293456 }, + { url = "https://files.pythonhosted.org/packages/3b/60/ba8aa296ca6d76603d58146b4a222cd99e7da33831158b8c00240a896a56/pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f", size = 288054 }, +] + [[package]] name = "pex" version = "2.24.1" @@ -1758,6 +2004,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, ] +[[package]] +name = "ply" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", size = 159130 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567 }, +] + [[package]] name = "prompt-toolkit" version = "3.0.36" @@ -1831,6 +2086,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be", size = 254228 }, ] +[[package]] +name = "psycopg" +version = "3.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/ad/7ce016ae63e231575df0498d2395d15f005f05e32d3a2d439038e1bd0851/psycopg-3.2.3.tar.gz", hash = "sha256:a5764f67c27bec8bfac85764d23c534af2c27b893550377e37ce59c12aac47a2", size = 155550 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/21/534b8f5bd9734b7a2fcd3a16b1ee82ef6cad81a4796e95ebf4e0c6a24119/psycopg-3.2.3-py3-none-any.whl", hash = "sha256:644d3973fe26908c73d4be746074f6e5224b03c1101d302d9a53bf565ad64907", size = 197934 }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] +pool = [ + { name = "psycopg-pool" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.2.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/6b/9805a5c743c1d54dcd035bd5c069202fde21b4cf69857ca40c2a55e69f8c/psycopg_binary-3.2.3-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:48f8ca6ee8939bab760225b2ab82934d54330eec10afe4394a92d3f2a0c37dd6", size = 3363376 }, + { url = "https://files.pythonhosted.org/packages/a8/82/45ac156b20e08e8f556a323c9568a011c71cf6e734e49667a398719ce0e4/psycopg_binary-3.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5361ea13c241d4f0ec3f95e0bf976c15e2e451e9cc7ef2e5ccfc9d170b197a40", size = 3506449 }, + { url = "https://files.pythonhosted.org/packages/e4/be/760cef50e1adfbc87dab2b05b30f544d7297040cce495835df9016556517/psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb987f14af7da7c24f803111dbc7392f5070fd350146af3345103f76ea82e339", size = 4445757 }, + { url = "https://files.pythonhosted.org/packages/b4/9c/bae6a9c6949aac577cc93f58705f649b50c62827038903bd75ff8956e63e/psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0463a11b1cace5a6aeffaf167920707b912b8986a9c7920341c75e3686277920", size = 4248376 }, + { url = "https://files.pythonhosted.org/packages/e5/0e/9db06ef94e4a156f3ed06043ee4f370e21866b0e3b7959691c8c4abfb698/psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b7be9a6c06518967b641fb15032b1ed682fd3b0443f64078899c61034a0bca6", size = 4487765 }, + { url = "https://files.pythonhosted.org/packages/9f/5f/8afc32b60ee8bc5c4af51e7cf6c42d93a989a09609524d0a393106e300cd/psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64a607e630d9f4b2797f641884e52b9f8e239d35943f51bef817a384ec1678fe", size = 4188374 }, + { url = "https://files.pythonhosted.org/packages/ed/5d/210cb75aff0296dc5c09bcf67babf8679905412d7a11357b983f0d877360/psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fa33ead69ed133210d96af0c63448b1385df48b9c0247eda735c5896b9e6dbbf", size = 3113180 }, + { url = "https://files.pythonhosted.org/packages/40/ec/46b1a5cdb2fe995b8ec0376f0695003e97fed9ac077e090a3165ea15f735/psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:1f8b0d0e99d8e19923e6e07379fa00570be5182c201a8c0b5aaa9a4d4a4ea20b", size = 3099455 }, + { url = "https://files.pythonhosted.org/packages/11/68/eaf85b3421b3f01b638dd6b16f4e9bc8de42eb1d000da62964fb29f8c823/psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:709447bd7203b0b2debab1acec23123eb80b386f6c29e7604a5d4326a11e5bd6", size = 3189977 }, + { url = "https://files.pythonhosted.org/packages/83/5a/cf94c3ba87ea6c8331aa0aba36a18a837a3231764457780661968804673e/psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5e37d5027e297a627da3551a1e962316d0f88ee4ada74c768f6c9234e26346d9", size = 3232263 }, + { url = "https://files.pythonhosted.org/packages/0e/3a/9d912b16059e87b04e3eb4fca457f079d78d6468f627d5622fbda80e9378/psycopg_binary-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:261f0031ee6074765096a19b27ed0f75498a8338c3dcd7f4f0d831e38adf12d1", size = 2912530 }, +] + +[[package]] +name = "psycopg-pool" +version = "3.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/71/01d4e589dc5fd1f21368b7d2df183ed0e5bbc160ce291d745142b229797b/psycopg_pool-3.2.4.tar.gz", hash = "sha256:61774b5bbf23e8d22bedc7504707135aaf744679f8ef9b3fe29942920746a6ed", size = 29749 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/28/2b56ac94c236ee033c7b291bcaa6a83089d0cc0fe7830c35f6521177c199/psycopg_pool-3.2.4-py3-none-any.whl", hash = "sha256:f6a22cff0f21f06d72fb2f5cb48c618946777c49385358e0c88d062c59cbd224", size = 38240 }, +] + [[package]] name = "py-avro-schema" version = "3.8.2" @@ -1849,17 +2155,17 @@ wheels = [ [[package]] name = "pyarrow" -version = "18.0.0" +version = "18.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/41/6bfd027410ba2cc35da4682394fdc4285dc345b1d99f7bd55e96255d0c7d/pyarrow-18.0.0.tar.gz", hash = "sha256:a6aa027b1a9d2970cf328ccd6dbe4a996bc13c39fd427f502782f5bdb9ca20f5", size = 1118457 } +sdist = { url = "https://files.pythonhosted.org/packages/7f/7b/640785a9062bb00314caa8a387abce547d2a420cf09bd6c715fe659ccffb/pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73", size = 1118671 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/ea/a7f77688e6c529723b37589af4db3e7179414e223878301907c5bd49d6bc/pyarrow-18.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:e7ab04f272f98ebffd2a0661e4e126036f6936391ba2889ed2d44c5006237802", size = 29493113 }, - { url = "https://files.pythonhosted.org/packages/79/8a/a3af902af623a1cf4f9d4d27d81e634caf1585a819b7530728a8147e391c/pyarrow-18.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:03f40b65a43be159d2f97fd64dc998f769d0995a50c00f07aab58b0b3da87e1f", size = 30833386 }, - { url = "https://files.pythonhosted.org/packages/46/1e/f38b22e12e2ce9ee7c9d805ce234f68b23a0568b9a6bea223e3a99ca0068/pyarrow-18.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be08af84808dff63a76860847c48ec0416928a7b3a17c2f49a072cac7c45efbd", size = 39170798 }, - { url = "https://files.pythonhosted.org/packages/f8/fb/fd0ef3e0f03227ab183f8dc941f4ef59636d8c382e246954601dd29cf1b0/pyarrow-18.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c70c1965cde991b711a98448ccda3486f2a336457cf4ec4dca257a926e149c9", size = 40103326 }, - { url = "https://files.pythonhosted.org/packages/7c/bd/5de139adba486db5ccc1b7ecab51e328a9dce354c82c6d26c2f642b178d3/pyarrow-18.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:00178509f379415a3fcf855af020e3340254f990a8534294ec3cf674d6e255fd", size = 38583592 }, - { url = "https://files.pythonhosted.org/packages/8d/1f/9bb3b3a644892d631dbbe99053cdb5295092d2696b4bcd3d21f29624c689/pyarrow-18.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a71ab0589a63a3e987beb2bc172e05f000a5c5be2636b4b263c44034e215b5d7", size = 40043128 }, - { url = "https://files.pythonhosted.org/packages/74/39/323621402c2b1ce7ba600d03c81cf9645b862350d7c495f3fcef37850d1d/pyarrow-18.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe92efcdbfa0bcf2fa602e466d7f2905500f33f09eb90bf0bcf2e6ca41b574c8", size = 25075300 }, + { url = "https://files.pythonhosted.org/packages/6a/50/12829e7111b932581e51dda51d5cb39207a056c30fe31ef43f14c63c4d7e/pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d", size = 29514620 }, + { url = "https://files.pythonhosted.org/packages/d1/41/468c944eab157702e96abab3d07b48b8424927d4933541ab43788bb6964d/pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee", size = 30856494 }, + { url = "https://files.pythonhosted.org/packages/68/f9/29fb659b390312a7345aeb858a9d9c157552a8852522f2c8bad437c29c0a/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992", size = 39203624 }, + { url = "https://files.pythonhosted.org/packages/6e/f6/19360dae44200e35753c5c2889dc478154cd78e61b1f738514c9f131734d/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54", size = 40139341 }, + { url = "https://files.pythonhosted.org/packages/bb/e6/9b3afbbcf10cc724312e824af94a2e993d8ace22994d823f5c35324cebf5/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33", size = 38618629 }, + { url = "https://files.pythonhosted.org/packages/3a/2e/3b99f8a3d9e0ccae0e961978a0d0089b25fb46ebbcfb5ebae3cca179a5b3/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30", size = 40078661 }, + { url = "https://files.pythonhosted.org/packages/76/52/f8da04195000099d394012b8d42c503d7041b79f778d854f410e5f05049a/pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99", size = 25092330 }, ] [[package]] @@ -2182,6 +2488,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481 }, ] +[[package]] +name = "requirements-parser" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "types-setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/70/80ed53ebd21853855aad552d4ed6c4934df62cd32fe9a3669fcdef59429c/requirements_parser-0.11.0.tar.gz", hash = "sha256:35f36dc969d14830bf459803da84f314dc3d17c802592e9e970f63d0359e5920", size = 23663 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/33/190393a7d36872e237cbc99e6c44d9a078a1ba7b406462fe6eafd5a28e04/requirements_parser-0.11.0-py3-none-any.whl", hash = "sha256:50379eb50311834386c2568263ae5225d7b9d0867fb55cf4ecc93959de2c2684", size = 14800 }, +] + [[package]] name = "rich" version = "13.9.4" @@ -2266,6 +2585,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/7d/43ab67228ef98c6b5dd42ab386eae2d7877036970a0d7e3dd3eb47a0d530/scipy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f", size = 44521212 }, ] +[[package]] +name = "semver" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/6c/a536cc008f38fd83b3c1b98ce19ead13b746b5588c9a0cb9dd9f6ea434bc/semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc", size = 214988 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/77/0cc7a8a3bc7e53d07e8f47f147b92b0960e902b8254859f4aee5c4d7866b/semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4", size = 17099 }, +] + [[package]] name = "setuptools" version = "75.6.0" @@ -2284,6 +2612,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 }, ] +[[package]] +name = "simplejson" +version = "3.19.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/29/085111f19717f865eceaf0d4397bf3e76b08d60428b076b64e2a1903706d/simplejson-3.19.3.tar.gz", hash = "sha256:8e086896c36210ab6050f2f9f095a5f1e03c83fa0e7f296d6cba425411364680", size = 85237 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/15/513fea93fafbdd4993eacfcb762965b2ff3d29e618c029e2956174d68c4b/simplejson-3.19.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:66a0399e21c2112acacfebf3d832ebe2884f823b1c7e6d1363f2944f1db31a99", size = 92921 }, + { url = "https://files.pythonhosted.org/packages/a4/4f/998a907ae1a6c104dc0ee48aa248c2478490152808d34d8e07af57f396c3/simplejson-3.19.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6ef9383c5e05f445be60f1735c1816163c874c0b1ede8bb4390aff2ced34f333", size = 75311 }, + { url = "https://files.pythonhosted.org/packages/db/44/acd6122201e927451869d45952b9ab1d3025cdb5e61548d286d08fbccc08/simplejson-3.19.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42e5acf80d4d971238d4df97811286a044d720693092b20a56d5e56b7dcc5d09", size = 74964 }, + { url = "https://files.pythonhosted.org/packages/27/ca/d0a1e8f16e1bbdc0b8c6d88166f45f565ed7285f53928cfef3b6ce78f14d/simplejson-3.19.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0b0efc7279d768db7c74d3d07f0b5c81280d16ae3fb14e9081dc903e8360771", size = 150106 }, + { url = "https://files.pythonhosted.org/packages/63/59/0554b78cf26c98e2b9cae3f44723bd72c2394e2afec1a14eedc6211f7187/simplejson-3.19.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0552eb06e7234da892e1d02365cd2b7b2b1f8233aa5aabdb2981587b7cc92ea0", size = 158347 }, + { url = "https://files.pythonhosted.org/packages/b2/fe/9f30890352e431e8508cc569912d3322147d3e7e4f321e48c0adfcb4c97d/simplejson-3.19.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5bf6a3b9a7d7191471b464fe38f684df10eb491ec9ea454003edb45a011ab187", size = 148456 }, + { url = "https://files.pythonhosted.org/packages/37/e3/663a09542ee021d4131162f7a164cb2e7f04ef48433a67591738afbf12ea/simplejson-3.19.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7017329ca8d4dca94ad5e59f496e5fc77630aecfc39df381ffc1d37fb6b25832", size = 152190 }, + { url = "https://files.pythonhosted.org/packages/31/20/4e0c4d35e10ff6465003bec304316d822a559a1c38c66ef6892ca199c207/simplejson-3.19.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:67a20641afebf4cfbcff50061f07daad1eace6e7b31d7622b6fa2c40d43900ba", size = 149846 }, + { url = "https://files.pythonhosted.org/packages/08/7a/46e2e072cac3987cbb05946f25167f0ad2fe536748e7405953fd6661a486/simplejson-3.19.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:dd6a7dabcc4c32daf601bc45e01b79175dde4b52548becea4f9545b0a4428169", size = 151714 }, + { url = "https://files.pythonhosted.org/packages/7f/7d/dbeeac10eb61d5d8858d0bb51121a21050d281dc83af4c557f86da28746c/simplejson-3.19.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:08f9b443a94e72dd02c87098c96886d35790e79e46b24e67accafbf13b73d43b", size = 158777 }, + { url = "https://files.pythonhosted.org/packages/fc/8f/a98bdbb799c6a4a884b5823db31785a96ba895b4b0f4d8ac345d6fe98bbf/simplejson-3.19.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa97278ae6614346b5ca41a45a911f37a3261b57dbe4a00602048652c862c28b", size = 154230 }, + { url = "https://files.pythonhosted.org/packages/b1/db/852eebceb85f969ae40e06babed1a93d3bacb536f187d7a80ff5823a5979/simplejson-3.19.3-cp312-cp312-win32.whl", hash = "sha256:ef28c3b328d29b5e2756903aed888960bc5df39b4c2eab157ae212f70ed5bf74", size = 74002 }, + { url = "https://files.pythonhosted.org/packages/fe/68/9f0e5df0651cb79ef83cba1378765a00ee8038e6201cc82b8e7178a7778e/simplejson-3.19.3-cp312-cp312-win_amd64.whl", hash = "sha256:1e662336db50ad665777e6548b5076329a94a0c3d4a0472971c588b3ef27de3a", size = 75596 }, + { url = "https://files.pythonhosted.org/packages/0d/e7/f9fafbd4f39793a20cc52e77bbd766f7384312526d402c382928dc7667f6/simplejson-3.19.3-py3-none-any.whl", hash = "sha256:49cc4c7b940d43bd12bf87ec63f28cbc4964fc4e12c031cc8cd01650f43eb94e", size = 57004 }, +] + [[package]] name = "six" version = "1.16.0" @@ -2302,6 +2652,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/7d/23074983ffd0d2d01163b74c621d7ffa3ea21e938b6d6443659adb63bb73/slack_sdk-3.33.4-py2.py3-none-any.whl", hash = "sha256:9f30cb3c9c07b441c49d53fc27f9f1837ad1592a7e9d4ca431f53cdad8826cc6", size = 292095 }, ] +[[package]] +name = "sling" +version = "1.2.23" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sling-linux-amd64" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/dc/deacac9b90fad0df3f61a17b1ee808b1c9edea8820a8db11f6cbd9e7e564/sling-1.2.23.tar.gz", hash = "sha256:a656962bfcbe076305013585e7e59d21bdd83019b80ff08e1bcaf8f9e104e9c2", size = 6939 } + +[[package]] +name = "sling-linux-amd64" +version = "1.2.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/c3/507d3285493aa72dec02f6b30d18872839f06ec2d92adf759cb5606b8c81/sling-linux-amd64-1.2.23.tar.gz", hash = "sha256:1bd108b8bf448d44f8b9b0f7be0c0a7643479503935f14b3a970cccdf61cc13f", size = 52749129 } + +[[package]] +name = "smmap" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/04/b5bf6d21dc4041000ccba7eb17dd3055feb237e7ffc2c20d3fae3af62baa/smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62", size = 22291 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/a5/10f97f73544edcdef54409f1d839f6049a0d79df68adbc1ceb24d1aaca42/smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da", size = 24282 }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -2452,6 +2826,7 @@ dependencies = [ { name = "dagster-airbyte" }, { name = "dagster-cloud" }, { name = "dagster-dbt" }, + { name = "dagster-embedded-elt" }, { name = "dagster-fivetran" }, { name = "dagster-gcp" }, { name = "dagster-k8s" }, @@ -2463,6 +2838,7 @@ dependencies = [ { name = "gspread" }, { name = "ldap3" }, { name = "oracledb" }, + { name = "psycopg", extra = ["binary", "pool"] }, { name = "py-avro-schema" }, { name = "pycryptodome" }, { name = "pypdf" }, @@ -2488,6 +2864,7 @@ requires-dist = [ { name = "dagster-airbyte" }, { name = "dagster-cloud" }, { name = "dagster-dbt" }, + { name = "dagster-embedded-elt" }, { name = "dagster-fivetran" }, { name = "dagster-gcp" }, { name = "dagster-k8s" }, @@ -2499,6 +2876,7 @@ requires-dist = [ { name = "gspread", specifier = ">=5.12.0" }, { name = "ldap3", specifier = ">=2.9.1" }, { name = "oracledb", specifier = ">=1.4.2" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.2.3" }, { name = "py-avro-schema", specifier = ">=3.4.1" }, { name = "pycryptodome", specifier = ">=3.19.0" }, { name = "pypdf", specifier = ">=5.0.0" }, @@ -2542,6 +2920,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2c/ffbf7a134b9ab11a67b0cf0726453cedd9c5043a4fe7a35d1cefa9a1bcfb/threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467", size = 18414 }, ] +[[package]] +name = "time-machine" +version = "2.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/dd/5022939b9cadefe3af04f4012186c29b8afbe858b1ec2cfa38baeec94dab/time_machine-2.16.0.tar.gz", hash = "sha256:4a99acc273d2f98add23a89b94d4dd9e14969c01214c8514bfa78e4e9364c7e2", size = 24626 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/f4/603a84e7ae6427a53953db9f61b689dc6adf233e03c5f5ca907a901452fd/time_machine-2.16.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:84788f4d62a8b1bf5e499bb9b0e23ceceea21c415ad6030be6267ce3d639842f", size = 20155 }, + { url = "https://files.pythonhosted.org/packages/d8/94/dbe69aecb4b84be52d34814e63176c5ca61f38ee9e6ecda11104653405b5/time_machine-2.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:15ec236b6571730236a193d9d6c11d472432fc6ab54e85eac1c16d98ddcd71bf", size = 16640 }, + { url = "https://files.pythonhosted.org/packages/da/13/27f11be25d7bd298e033b9da93217e5b68309bf724b6e494cdadb471d00d/time_machine-2.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cedc989717c8b44a3881ac3d68ab5a95820448796c550de6a2149ed1525157f0", size = 33721 }, + { url = "https://files.pythonhosted.org/packages/e6/9d/70e4640fed1fd8122204ae825c688d0ef8c04f515ec6bf3c5f3086d6510e/time_machine-2.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d26d79de1c63a8c6586c75967e09b0ff306aa7e944a1eaddb74595c9b1839ca", size = 31646 }, + { url = "https://files.pythonhosted.org/packages/a1/cb/93bc0e51bea4e171a85151dbba3c3b3f612b50b953cd3076f5b4f0db9e14/time_machine-2.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:317b68b56a9c3731e0cf8886e0f94230727159e375988b36c60edce0ddbcb44a", size = 33403 }, + { url = "https://files.pythonhosted.org/packages/89/71/2c6a63ad4fbce3d62d46bbd9ac4433f30bade7f25978ce00815b905bcfcf/time_machine-2.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:43e1e18279759897be3293a255d53e6b1cb0364b69d9591d0b80c51e461c94b0", size = 33327 }, + { url = "https://files.pythonhosted.org/packages/68/4e/205c2b26763b8817cd6b8868242843800a1fbf275f2af35f5ba35ff2b01a/time_machine-2.16.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e43adb22def972a29d2b147999b56897116085777a0fea182fd93ee45730611e", size = 31454 }, + { url = "https://files.pythonhosted.org/packages/d7/95/44c1aa3994919f93534244c40cfd2fb9416d7686dc0c8b9b262c751b5118/time_machine-2.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0c766bea27a0600e36806d628ebc4b47178b12fcdfb6c24dc0a566a9c06bfe7f", size = 32972 }, + { url = "https://files.pythonhosted.org/packages/d4/ee/75243df9c7cf30f108758e887141a58e6544baaa46e2e647b9ccc56db819/time_machine-2.16.0-cp312-cp312-win32.whl", hash = "sha256:6dae82ab647d107817e013db82223e20a9853fa88543fec853ae326382d03c2e", size = 19078 }, + { url = "https://files.pythonhosted.org/packages/d4/7c/d4e67cc031f9653c92167ccf87d241e3208653d191c96ac79281c273ab92/time_machine-2.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:265462c77dc9576267c3c7f20707780a171a9fdbac93ac22e608c309efd68c33", size = 19923 }, + { url = "https://files.pythonhosted.org/packages/aa/b6/7047226fcb9afefe47fc80f605530535bf71ad99b6797f057abbfa4cd9a5/time_machine-2.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:ef768e14768eebe3bb1196c0dece8e14c1c6991605721214a0c3c68cf77eb216", size = 18003 }, +] + [[package]] name = "tomli" version = "2.1.0" @@ -2551,6 +2951,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/f7/4da0ffe1892122c9ea096c57f64c2753ae5dd3ce85488802d11b0992cc6d/tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391", size = 13750 }, ] +[[package]] +name = "tomlkit" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/09/a439bec5888f00a54b8b9f05fa94d7f901d6735ef4e55dcec9bc37b5d8fa/tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79", size = 192885 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/b6/a447b5e4ec71e13871be01ba81f5dfc9d0af7e473da256ff46bc0e24026f/tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde", size = 37955 }, +] + [[package]] name = "toposort" version = "1.10" @@ -2599,6 +3008,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/69/e90a0b4d0c16e095901679216c8ecdc728110c7c54e7b5f43a623bc4c789/typer-0.13.1-py3-none-any.whl", hash = "sha256:5b59580fd925e89463a29d363e0a43245ec02765bde9fb77d39e5d0f29dd7157", size = 44723 }, ] +[[package]] +name = "types-setuptools" +version = "75.6.0.20241126" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/d2/15ede73bc3faf647af2c7bfefa90dde563a4b6bb580b1199f6255463c272/types_setuptools-75.6.0.20241126.tar.gz", hash = "sha256:7bf25ad4be39740e469f9268b6beddda6e088891fa5a27e985c6ce68bf62ace0", size = 48569 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/a0/898a1363592d372d4103b76b7c723d84fcbde5fa4ed0c3a29102805ed7db/types_setuptools-75.6.0.20241126-py3-none-any.whl", hash = "sha256:aaae310a0e27033c1da8457d4d26ac673b0c8a0de7272d6d4708e263f2ea3b9b", size = 72732 }, +] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -2768,6 +3186,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/0b/c7e5d11020242984d9d37990310520ed663b942333b83a033c2f20191113/websockets-14.1-py3-none-any.whl", hash = "sha256:4d4fc827a20abe6d544a119896f6b78ee13fe81cbfef416f3f2ddf09a03f0e2e", size = 156277 }, ] +[[package]] +name = "wheel" +version = "0.45.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494 }, +] + +[[package]] +name = "win-precise-time" +version = "1.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/b0/21547e16a47206ccdd15769bf65e143ade1ffae67f0881c855f76e44e9fa/win-precise-time-1.4.2.tar.gz", hash = "sha256:89274785cbc5f2997e01675206da3203835a442c60fd97798415c6b3c179c0b9", size = 7982 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/96/55a14b5c0e90439951f4a72672223bba81a5f882033c5850f8a6c7f4308b/win_precise_time-1.4.2-cp312-cp312-win32.whl", hash = "sha256:0210dcea88a520c91de1708ae4c881e3c0ddc956daa08b9eabf2b7c35f3109f5", size = 14694 }, + { url = "https://files.pythonhosted.org/packages/17/19/7ea9a22a69fc23d5ca02e8edf65e4a335a210497794af1af0ef8fda91fa0/win_precise_time-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:85670f77cc8accd8f1e6d05073999f77561c23012a9ee988cbd44bb7ce655062", size = 14913 }, +] + [[package]] name = "yarl" version = "1.18.0"