From 44456e09b60679c16cdab2ba5041bf8fcf4eb2f4 Mon Sep 17 00:00:00 2001 From: Satwant Rana <4613501+satrana42@users.noreply.github.com> Date: Tue, 15 Oct 2024 22:52:20 +0530 Subject: [PATCH] Add changelog operator (#578) In this change we add changelog operator which converts a deltaframe into an append only changelog, with keys and kinds stored in value columns. --- fennel/CHANGELOG.md | 3 + fennel/client_tests/test_dataset.py | 121 +++++++++++++++++---- fennel/datasets/datasets.py | 32 ++++-- fennel/gen/dataset_pb2.py | 36 +++--- fennel/gen/dataset_pb2.pyi | 16 ++- fennel/gen/expression_pb2.py | 82 +++++++------- fennel/gen/expression_pb2.pyi | 58 +++++++++- fennel/internal_lib/to_proto/serializer.py | 3 +- fennel/testing/executor.py | 4 +- pyproject.toml | 2 +- 10 files changed, 263 insertions(+), 94 deletions(-) diff --git a/fennel/CHANGELOG.md b/fennel/CHANGELOG.md index 250058da1..77fc6c6fd 100644 --- a/fennel/CHANGELOG.md +++ b/fennel/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## [1.5.40] - 2024-10-05 +- Add changelog operator + ## [1.5.39] - 2024-10-14 - Add support for Snowflake sink diff --git a/fennel/client_tests/test_dataset.py b/fennel/client_tests/test_dataset.py index 600a1b9ce..2a15bff40 100644 --- a/fennel/client_tests/test_dataset.py +++ b/fennel/client_tests/test_dataset.py @@ -4619,9 +4619,11 @@ def pipeline(cls, event: Dataset): assert results["value"].tolist() == [[1], [2], [3]] +@pytest.mark.integration @mock -def test_unkey_operator(client): +def test_changelog_operator(client): @dataset + @source(webhook.endpoint("JobOpening"), disorder="14d", cdc="append") class JobOpening: creater_id: int job_id: int @@ -4630,7 +4632,8 @@ class JobOpening: @dataset(index=True) class JobRank: creater_id: int = field(key=True) - rank: int = field(key=True) + job_id: int = field(key=True) + is_delete: bool = field(key=True) creation_ts: datetime @pipeline @@ -4638,17 +4641,14 @@ class JobRank: def rank_job(cls, job_opening: Dataset): return ( job_opening.groupby("creater_id") - .aggregate( - rank=Count(window=Continuous("forever")), emit="final" - ) - .changelog("del") - .filter(lambda df: ~df["del"] & (df["rank"] < 4)) - .drop("del") - .groupby("creater_id", "rank") + .latest() + .changelog(delete="is_delete") + .groupby("creater_id", "job_id", "is_delete") .latest() ) client.commit(datasets=[JobOpening, JobRank], message="test") + # Creation ts is 1 every 2 hours creation_ts = [ datetime(2022, 1, 1, 2, 0, 0), @@ -4658,6 +4658,7 @@ def rank_job(cls, job_opening: Dataset): datetime(2022, 1, 1, 10, 0, 0), datetime(2022, 1, 1, 12, 0, 0), ] + openings = pd.DataFrame( { "creater_id": [1, 2, 1, 1, 1, 1], @@ -4665,23 +4666,105 @@ def rank_job(cls, job_opening: Dataset): "creation_ts": creation_ts, } ) - log(JobOpening, openings) + client.log("fennel_webhook", "JobOpening", openings) + + if client.is_integration_client(): + client.sleep(60) + + is_deletes = [True, False, True, True, True, False] + results, found = client.lookup( - "JobRank", + JobRank, keys=pd.DataFrame( - {"creater_id": [1, 1, 1, 1, 2], "rank": [4, 3, 2, 1, 1]} + { + "creater_id": [1, 2, 1, 1, 1, 1], + "job_id": [1, 2, 3, 4, 5, 6], + "is_delete": is_deletes, + } ), ) - assert found.tolist() == [False, True, True, True, True] - assert results.shape == (5, 3) - assert results["rank"].tolist() == [4, 3, 2, 1, 1] - assert results["creation_ts"].tolist() == [ - pd.NaT, - pd.Timestamp("2022-01-01 08:00:00", tz="UTC"), + + creation_ts_expected = [ pd.Timestamp("2022-01-01 06:00:00", tz="UTC"), - pd.Timestamp("2022-01-01 02:00:00", tz="UTC"), pd.Timestamp("2022-01-01 04:00:00", tz="UTC"), + pd.Timestamp("2022-01-01 08:00:00", tz="UTC"), + pd.Timestamp("2022-01-01 10:00:00", tz="UTC"), + pd.Timestamp("2022-01-01 12:00:00", tz="UTC"), + pd.Timestamp("2022-01-01 12:00:00", tz="UTC"), ] + assert found.tolist() == [True, True, True, True, True, True] + assert results.shape == (6, 4) + assert results["creation_ts"].tolist() == creation_ts_expected + + +@pytest.mark.integration +@mock +def test_changelog_operator_insert_identity(client): + @dataset + @source(webhook.endpoint("JobOpening"), disorder="14d", cdc="append") + class JobOpening: + creater_id: int + job_id: int + creation_ts: datetime + + @dataset(index=True) + class JobRank: + creater_id: int = field(key=True) + job_id: int = field(key=True) + creation_ts: datetime + + @pipeline + @inputs(JobOpening) + def rank_job(cls, job_opening: Dataset): + return ( + job_opening.groupby("creater_id") + .latest() + .changelog(insert="is_insert") + .filter(lambda df: df["is_insert"]) + .drop("is_insert") + .groupby("creater_id", "job_id") + .latest() + ) + + client.commit(datasets=[JobOpening, JobRank], message="test") + + # Creation ts is 1 every 2 hours + creation_ts = [ + datetime(2022, 1, 1, 2, 0, 0), + datetime(2022, 1, 1, 4, 0, 0), + datetime(2022, 1, 1, 6, 0, 0), + datetime(2022, 1, 1, 8, 0, 0), + datetime(2022, 1, 1, 10, 0, 0), + datetime(2022, 1, 1, 12, 0, 0), + ] + + openings = pd.DataFrame( + { + "creater_id": [1, 2, 1, 1, 1, 1], + "job_id": [1, 2, 3, 4, 5, 6], + "creation_ts": creation_ts, + } + ) + client.log("fennel_webhook", "JobOpening", openings) + + if client.is_integration_client(): + client.sleep(60) + + results, found = client.lookup( + JobRank, + keys=pd.DataFrame( + { + "creater_id": [1, 2, 1, 1, 1, 1], + "job_id": [1, 2, 3, 4, 5, 6], + } + ), + ) + + creation_ts_expected = [pd.Timestamp(d, tz="UTC") for d in creation_ts] + + assert found.tolist() == [True, True, True, True, True, True] + assert results.shape == (6, 3) + assert results["creation_ts"].tolist() == creation_ts_expected webhook = Webhook(name="fennel_webhook") diff --git a/fennel/datasets/datasets.py b/fennel/datasets/datasets.py index 058ccb53e..41c160f59 100644 --- a/fennel/datasets/datasets.py +++ b/fennel/datasets/datasets.py @@ -371,8 +371,8 @@ def explode(self, *args, columns: List[str] = None) -> _Node: columns = _Node.__get_list_args(*args, columns=columns, name="explode") return Explode(self, columns) - def changelog(self, delete_column: str) -> _Node: - return Changelog(self, delete_column) + def changelog(self, **kwargs) -> _Node: + return Changelog(self, **kwargs) def isignature(self): raise NotImplementedError @@ -579,22 +579,38 @@ def dsschema(self): class Changelog(_Node): - def __init__(self, node: _Node, delete_column: str): + def __init__(self, node: _Node, **kwargs): super().__init__() self.node = node - self.delete_column = delete_column + + delete_column = kwargs.get("delete") + insert_column = kwargs.get("insert") + if delete_column is None and insert_column is None: + raise ValueError("Either delete or insert column must be specified") + elif delete_column is not None and insert_column is not None: + raise ValueError( + "Only one of delete or insert column can be specified" + ) + elif delete_column is not None: + self.kind_column = delete_column + self.deletes = True + else: + assert insert_column is not None + self.kind_column = insert_column + self.deletes = False + self.node.out_edges.append(self) def signature(self): - return fhash(self.node.signature(), self.delete_column) + return fhash(self.node.signature(), self.kind_column, self.deletes) def dsschema(self): input_schema = self.node.dsschema() - # Remove all keys from the schema and make then values + # Remove all keys from the schema and make them values val_fields = copy.deepcopy(input_schema.keys) values = input_schema.values val_fields.update(values) - val_fields.update({self.delete_column: pd.BooleanDtype}) + val_fields.update({self.kind_column: pd.BooleanDtype}) return DSSchema( keys={}, values=val_fields, @@ -3346,7 +3362,7 @@ def visitChangelog(self, obj) -> DSSchema: # Unkey operation is allowed on keyed datasets only. if len(input_schema.keys) == 0: raise TypeError( - f"UnKey operation is allowed only on keyed datasets. Found dataset without keys in pipeline `{self.pipeline_name}`" + f"Changelog operation is allowed only on keyed datasets. Found dataset without keys in pipeline `{self.pipeline_name}`" ) output_schema = copy.deepcopy(obj.dsschema()) output_schema.name = ( diff --git a/fennel/gen/dataset_pb2.py b/fennel/gen/dataset_pb2.py index 0507da7df..65fc5320a 100644 --- a/fennel/gen/dataset_pb2.py +++ b/fennel/gen/dataset_pb2.py @@ -20,7 +20,7 @@ import fennel.gen.expression_pb2 as expression__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rdataset.proto\x12\x14\x66\x65nnel.proto.dataset\x1a\x1egoogle/protobuf/duration.proto\x1a\x0emetadata.proto\x1a\x0cpycode.proto\x1a\x0cschema.proto\x1a\nspec.proto\x1a\x0cwindow.proto\x1a\x10\x65xpression.proto\"\xe5\x03\n\x0b\x43oreDataset\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x08metadata\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.metadata.Metadata\x12/\n\x08\x64sschema\x18\x03 \x01(\x0b\x32\x1d.fennel.proto.schema.DSSchema\x12*\n\x07history\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\tretention\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12L\n\x0e\x66ield_metadata\x18\x06 \x03(\x0b\x32\x34.fennel.proto.dataset.CoreDataset.FieldMetadataEntry\x12+\n\x06pycode\x18\x07 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\x12\x19\n\x11is_source_dataset\x18\x08 \x01(\x08\x12\x0f\n\x07version\x18\t \x01(\r\x12\x0c\n\x04tags\x18\n \x03(\t\x1aU\n\x12\x46ieldMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.metadata.Metadata:\x02\x38\x01\"Q\n\x08OnDemand\x12\x1c\n\x14\x66unction_source_code\x18\x01 \x01(\t\x12\x10\n\x08\x66unction\x18\x02 \x01(\x0c\x12\x15\n\rexpires_after\x18\x03 \x01(\x03\"\xd2\x01\n\x08Pipeline\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0c\x64\x61taset_name\x18\x02 \x01(\t\x12\x11\n\tsignature\x18\x03 \x01(\t\x12\x31\n\x08metadata\x18\x04 \x01(\x0b\x32\x1f.fennel.proto.metadata.Metadata\x12\x1b\n\x13input_dataset_names\x18\x05 \x03(\t\x12\x12\n\nds_version\x18\x06 \x01(\r\x12+\n\x06pycode\x18\x07 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\"\x8f\x08\n\x08Operator\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07is_root\x18\x02 \x01(\x08\x12\x15\n\rpipeline_name\x18\x03 \x01(\t\x12\x14\n\x0c\x64\x61taset_name\x18\x04 \x01(\t\x12\x12\n\nds_version\x18\x14 \x01(\r\x12\x34\n\taggregate\x18\x05 \x01(\x0b\x32\x1f.fennel.proto.dataset.AggregateH\x00\x12*\n\x04join\x18\x06 \x01(\x0b\x32\x1a.fennel.proto.dataset.JoinH\x00\x12\x34\n\ttransform\x18\x07 \x01(\x0b\x32\x1f.fennel.proto.dataset.TransformH\x00\x12,\n\x05union\x18\x08 \x01(\x0b\x32\x1b.fennel.proto.dataset.UnionH\x00\x12.\n\x06\x66ilter\x18\t \x01(\x0b\x32\x1c.fennel.proto.dataset.FilterH\x00\x12\x37\n\x0b\x64\x61taset_ref\x18\n \x01(\x0b\x32 .fennel.proto.dataset.DatasetRefH\x00\x12.\n\x06rename\x18\x0c \x01(\x0b\x32\x1c.fennel.proto.dataset.RenameH\x00\x12*\n\x04\x64rop\x18\r \x01(\x0b\x32\x1a.fennel.proto.dataset.DropH\x00\x12\x30\n\x07\x65xplode\x18\x0e \x01(\x0b\x32\x1d.fennel.proto.dataset.ExplodeH\x00\x12,\n\x05\x64\x65\x64up\x18\x0f \x01(\x0b\x32\x1b.fennel.proto.dataset.DedupH\x00\x12,\n\x05\x66irst\x18\x10 \x01(\x0b\x32\x1b.fennel.proto.dataset.FirstH\x00\x12.\n\x06\x61ssign\x18\x11 \x01(\x0b\x32\x1c.fennel.proto.dataset.AssignH\x00\x12\x32\n\x08\x64ropnull\x18\x12 \x01(\x0b\x32\x1e.fennel.proto.dataset.DropnullH\x00\x12:\n\x06window\x18\x13 \x01(\x0b\x32(.fennel.proto.dataset.WindowOperatorKindH\x00\x12.\n\x06latest\x18\x15 \x01(\x0b\x32\x1c.fennel.proto.dataset.LatestH\x00\x12\x34\n\tchangelog\x18\x16 \x01(\x0b\x32\x1f.fennel.proto.dataset.ChangelogH\x00\x12\x37\n\x0b\x61ssign_expr\x18\x17 \x01(\x0b\x32 .fennel.proto.dataset.AssignExprH\x00\x12\x37\n\x0b\x66ilter_expr\x18\x18 \x01(\x0b\x32 .fennel.proto.dataset.FilterExprH\x00\x12\x0c\n\x04name\x18\x0b \x01(\tB\x06\n\x04kind\"\xc7\x01\n\tAggregate\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x0c\n\x04keys\x18\x02 \x03(\t\x12)\n\x05specs\x18\x03 \x03(\x0b\x32\x1a.fennel.proto.spec.PreSpec\x12\x12\n\x05\x61long\x18\x05 \x01(\tH\x00\x88\x01\x01\x12\x39\n\remit_strategy\x18\x06 \x01(\x0e\x32\".fennel.proto.dataset.EmitStrategy\x12\x14\n\x0coperand_name\x18\x04 \x01(\tB\x08\n\x06_along\"\xcd\x03\n\x04Join\x12\x16\n\x0elhs_operand_id\x18\x01 \x01(\t\x12\x1c\n\x14rhs_dsref_operand_id\x18\x02 \x01(\t\x12.\n\x02on\x18\x03 \x03(\x0b\x32\".fennel.proto.dataset.Join.OnEntry\x12\x32\n\nwithin_low\x18\x06 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x88\x01\x01\x12\x33\n\x0bwithin_high\x18\x07 \x01(\x0b\x32\x19.google.protobuf.DurationH\x01\x88\x01\x01\x12\x18\n\x10lhs_operand_name\x18\x04 \x01(\t\x12\x1e\n\x16rhs_dsref_operand_name\x18\x05 \x01(\t\x12+\n\x03how\x18\x08 \x01(\x0e\x32\x1e.fennel.proto.dataset.Join.How\x12\x15\n\tbroadcast\x18\t \x01(\x08\x42\x02\x18\x01\x12\x12\n\nrhs_fields\x18\n \x03(\t\x1a)\n\x07OnEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1a\n\x03How\x12\x08\n\x04Left\x10\x00\x12\t\n\x05Inner\x10\x01\x42\r\n\x0b_within_lowB\x0e\n\x0c_within_high\"\xed\x01\n\tTransform\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12;\n\x06schema\x18\x02 \x03(\x0b\x32+.fennel.proto.dataset.Transform.SchemaEntry\x12+\n\x06pycode\x18\x03 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\x12\x14\n\x0coperand_name\x18\x04 \x01(\t\x1aL\n\x0bSchemaEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType:\x02\x38\x01\"c\n\nFilterExpr\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12+\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"_\n\x06\x46ilter\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12+\n\x06pycode\x18\x02 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"\xa8\x01\n\x06\x41ssign\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12+\n\x06pycode\x18\x02 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\x12\x13\n\x0b\x63olumn_name\x18\x03 \x01(\t\x12\x32\n\x0boutput_type\x18\x04 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType\x12\x14\n\x0coperand_name\x18\x05 \x01(\t\"\xdb\x02\n\nAssignExpr\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12:\n\x05\x65xprs\x18\x02 \x03(\x0b\x32+.fennel.proto.dataset.AssignExpr.ExprsEntry\x12G\n\x0coutput_types\x18\x03 \x03(\x0b\x32\x31.fennel.proto.dataset.AssignExpr.OutputTypesEntry\x12\x14\n\x0coperand_name\x18\x05 \x01(\t\x1aK\n\nExprsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr:\x02\x38\x01\x1aQ\n\x10OutputTypesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType:\x02\x38\x01\"E\n\x08\x44ropnull\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x0f\n\x07\x63olumns\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"B\n\x04\x44rop\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x10\n\x08\x64ropcols\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"\xa5\x01\n\x06Rename\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12?\n\ncolumn_map\x18\x02 \x03(\x0b\x32+.fennel.proto.dataset.Rename.ColumnMapEntry\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\x1a\x30\n\x0e\x43olumnMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"3\n\x05Union\x12\x13\n\x0boperand_ids\x18\x01 \x03(\t\x12\x15\n\roperand_names\x18\x02 \x03(\t\"B\n\x05\x44\x65\x64up\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x0f\n\x07\x63olumns\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"D\n\x07\x45xplode\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x0f\n\x07\x63olumns\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"=\n\x05\x46irst\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\n\n\x02\x62y\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\">\n\x06Latest\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\n\n\x02\x62y\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"L\n\tChangelog\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x15\n\rdelete_column\x18\x02 \x01(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"\xcb\x01\n\x12WindowOperatorKind\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x30\n\x0bwindow_type\x18\x02 \x01(\x0b\x32\x1b.fennel.proto.window.Window\x12\n\n\x02\x62y\x18\x03 \x03(\t\x12\r\n\x05\x66ield\x18\x04 \x01(\t\x12\x32\n\x07summary\x18\x06 \x01(\x0b\x32\x1c.fennel.proto.window.SummaryH\x00\x88\x01\x01\x12\x14\n\x0coperand_name\x18\x05 \x01(\tB\n\n\x08_summary\",\n\nDatasetRef\x12\x1e\n\x16referring_dataset_name\x18\x01 \x01(\t\"\x80\x02\n\x08\x44\x61taflow\x12\x16\n\x0c\x64\x61taset_name\x18\x01 \x01(\tH\x00\x12L\n\x11pipeline_dataflow\x18\x02 \x01(\x0b\x32/.fennel.proto.dataset.Dataflow.PipelineDataflowH\x00\x12\x0c\n\x04tags\x18\x03 \x03(\t\x1ax\n\x10PipelineDataflow\x12\x14\n\x0c\x64\x61taset_name\x18\x01 \x01(\t\x12\x15\n\rpipeline_name\x18\x02 \x01(\t\x12\x37\n\x0finput_dataflows\x18\x03 \x03(\x0b\x32\x1e.fennel.proto.dataset.DataflowB\x06\n\x04kind\"\x9c\x01\n\x10PipelineLineages\x12\x14\n\x0c\x64\x61taset_name\x18\x01 \x01(\t\x12\x15\n\rpipeline_name\x18\x02 \x01(\t\x12=\n\x0einput_datasets\x18\x03 \x03(\x0b\x32%.fennel.proto.dataset.DatasetLineages\x12\x0e\n\x06\x61\x63tive\x18\x04 \x01(\x08\x12\x0c\n\x04tags\x18\x05 \x03(\t\"\\\n\x17\x44\x61tasetPipelineLineages\x12\x41\n\x11pipeline_lineages\x18\x02 \x03(\x0b\x32&.fennel.proto.dataset.PipelineLineages\"\x8b\x01\n\x0f\x44\x61tasetLineages\x12\x18\n\x0esource_dataset\x18\x01 \x01(\tH\x00\x12H\n\x0f\x64\x65rived_dataset\x18\x02 \x01(\x0b\x32-.fennel.proto.dataset.DatasetPipelineLineagesH\x00\x12\x0c\n\x04tags\x18\x03 \x03(\tB\x06\n\x04kind*$\n\x0c\x45mitStrategy\x12\t\n\x05\x45\x61ger\x10\x00\x12\t\n\x05\x46inal\x10\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rdataset.proto\x12\x14\x66\x65nnel.proto.dataset\x1a\x1egoogle/protobuf/duration.proto\x1a\x0emetadata.proto\x1a\x0cpycode.proto\x1a\x0cschema.proto\x1a\nspec.proto\x1a\x0cwindow.proto\x1a\x10\x65xpression.proto\"\xe5\x03\n\x0b\x43oreDataset\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x08metadata\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.metadata.Metadata\x12/\n\x08\x64sschema\x18\x03 \x01(\x0b\x32\x1d.fennel.proto.schema.DSSchema\x12*\n\x07history\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\tretention\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12L\n\x0e\x66ield_metadata\x18\x06 \x03(\x0b\x32\x34.fennel.proto.dataset.CoreDataset.FieldMetadataEntry\x12+\n\x06pycode\x18\x07 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\x12\x19\n\x11is_source_dataset\x18\x08 \x01(\x08\x12\x0f\n\x07version\x18\t \x01(\r\x12\x0c\n\x04tags\x18\n \x03(\t\x1aU\n\x12\x46ieldMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.metadata.Metadata:\x02\x38\x01\"Q\n\x08OnDemand\x12\x1c\n\x14\x66unction_source_code\x18\x01 \x01(\t\x12\x10\n\x08\x66unction\x18\x02 \x01(\x0c\x12\x15\n\rexpires_after\x18\x03 \x01(\x03\"\xd2\x01\n\x08Pipeline\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0c\x64\x61taset_name\x18\x02 \x01(\t\x12\x11\n\tsignature\x18\x03 \x01(\t\x12\x31\n\x08metadata\x18\x04 \x01(\x0b\x32\x1f.fennel.proto.metadata.Metadata\x12\x1b\n\x13input_dataset_names\x18\x05 \x03(\t\x12\x12\n\nds_version\x18\x06 \x01(\r\x12+\n\x06pycode\x18\x07 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\"\x8f\x08\n\x08Operator\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07is_root\x18\x02 \x01(\x08\x12\x15\n\rpipeline_name\x18\x03 \x01(\t\x12\x14\n\x0c\x64\x61taset_name\x18\x04 \x01(\t\x12\x12\n\nds_version\x18\x14 \x01(\r\x12\x34\n\taggregate\x18\x05 \x01(\x0b\x32\x1f.fennel.proto.dataset.AggregateH\x00\x12*\n\x04join\x18\x06 \x01(\x0b\x32\x1a.fennel.proto.dataset.JoinH\x00\x12\x34\n\ttransform\x18\x07 \x01(\x0b\x32\x1f.fennel.proto.dataset.TransformH\x00\x12,\n\x05union\x18\x08 \x01(\x0b\x32\x1b.fennel.proto.dataset.UnionH\x00\x12.\n\x06\x66ilter\x18\t \x01(\x0b\x32\x1c.fennel.proto.dataset.FilterH\x00\x12\x37\n\x0b\x64\x61taset_ref\x18\n \x01(\x0b\x32 .fennel.proto.dataset.DatasetRefH\x00\x12.\n\x06rename\x18\x0c \x01(\x0b\x32\x1c.fennel.proto.dataset.RenameH\x00\x12*\n\x04\x64rop\x18\r \x01(\x0b\x32\x1a.fennel.proto.dataset.DropH\x00\x12\x30\n\x07\x65xplode\x18\x0e \x01(\x0b\x32\x1d.fennel.proto.dataset.ExplodeH\x00\x12,\n\x05\x64\x65\x64up\x18\x0f \x01(\x0b\x32\x1b.fennel.proto.dataset.DedupH\x00\x12,\n\x05\x66irst\x18\x10 \x01(\x0b\x32\x1b.fennel.proto.dataset.FirstH\x00\x12.\n\x06\x61ssign\x18\x11 \x01(\x0b\x32\x1c.fennel.proto.dataset.AssignH\x00\x12\x32\n\x08\x64ropnull\x18\x12 \x01(\x0b\x32\x1e.fennel.proto.dataset.DropnullH\x00\x12:\n\x06window\x18\x13 \x01(\x0b\x32(.fennel.proto.dataset.WindowOperatorKindH\x00\x12.\n\x06latest\x18\x15 \x01(\x0b\x32\x1c.fennel.proto.dataset.LatestH\x00\x12\x34\n\tchangelog\x18\x16 \x01(\x0b\x32\x1f.fennel.proto.dataset.ChangelogH\x00\x12\x37\n\x0b\x61ssign_expr\x18\x17 \x01(\x0b\x32 .fennel.proto.dataset.AssignExprH\x00\x12\x37\n\x0b\x66ilter_expr\x18\x18 \x01(\x0b\x32 .fennel.proto.dataset.FilterExprH\x00\x12\x0c\n\x04name\x18\x0b \x01(\tB\x06\n\x04kind\"\xc7\x01\n\tAggregate\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x0c\n\x04keys\x18\x02 \x03(\t\x12)\n\x05specs\x18\x03 \x03(\x0b\x32\x1a.fennel.proto.spec.PreSpec\x12\x12\n\x05\x61long\x18\x05 \x01(\tH\x00\x88\x01\x01\x12\x39\n\remit_strategy\x18\x06 \x01(\x0e\x32\".fennel.proto.dataset.EmitStrategy\x12\x14\n\x0coperand_name\x18\x04 \x01(\tB\x08\n\x06_along\"\xcd\x03\n\x04Join\x12\x16\n\x0elhs_operand_id\x18\x01 \x01(\t\x12\x1c\n\x14rhs_dsref_operand_id\x18\x02 \x01(\t\x12.\n\x02on\x18\x03 \x03(\x0b\x32\".fennel.proto.dataset.Join.OnEntry\x12\x32\n\nwithin_low\x18\x06 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x88\x01\x01\x12\x33\n\x0bwithin_high\x18\x07 \x01(\x0b\x32\x19.google.protobuf.DurationH\x01\x88\x01\x01\x12\x18\n\x10lhs_operand_name\x18\x04 \x01(\t\x12\x1e\n\x16rhs_dsref_operand_name\x18\x05 \x01(\t\x12+\n\x03how\x18\x08 \x01(\x0e\x32\x1e.fennel.proto.dataset.Join.How\x12\x15\n\tbroadcast\x18\t \x01(\x08\x42\x02\x18\x01\x12\x12\n\nrhs_fields\x18\n \x03(\t\x1a)\n\x07OnEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1a\n\x03How\x12\x08\n\x04Left\x10\x00\x12\t\n\x05Inner\x10\x01\x42\r\n\x0b_within_lowB\x0e\n\x0c_within_high\"\xed\x01\n\tTransform\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12;\n\x06schema\x18\x02 \x03(\x0b\x32+.fennel.proto.dataset.Transform.SchemaEntry\x12+\n\x06pycode\x18\x03 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\x12\x14\n\x0coperand_name\x18\x04 \x01(\t\x1aL\n\x0bSchemaEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType:\x02\x38\x01\"c\n\nFilterExpr\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12+\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"_\n\x06\x46ilter\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12+\n\x06pycode\x18\x02 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"\xa8\x01\n\x06\x41ssign\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12+\n\x06pycode\x18\x02 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCode\x12\x13\n\x0b\x63olumn_name\x18\x03 \x01(\t\x12\x32\n\x0boutput_type\x18\x04 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType\x12\x14\n\x0coperand_name\x18\x05 \x01(\t\"\xdb\x02\n\nAssignExpr\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12:\n\x05\x65xprs\x18\x02 \x03(\x0b\x32+.fennel.proto.dataset.AssignExpr.ExprsEntry\x12G\n\x0coutput_types\x18\x03 \x03(\x0b\x32\x31.fennel.proto.dataset.AssignExpr.OutputTypesEntry\x12\x14\n\x0coperand_name\x18\x05 \x01(\t\x1aK\n\nExprsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr:\x02\x38\x01\x1aQ\n\x10OutputTypesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType:\x02\x38\x01\"E\n\x08\x44ropnull\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x0f\n\x07\x63olumns\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"B\n\x04\x44rop\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x10\n\x08\x64ropcols\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"\xa5\x01\n\x06Rename\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12?\n\ncolumn_map\x18\x02 \x03(\x0b\x32+.fennel.proto.dataset.Rename.ColumnMapEntry\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\x1a\x30\n\x0e\x43olumnMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"3\n\x05Union\x12\x13\n\x0boperand_ids\x18\x01 \x03(\t\x12\x15\n\roperand_names\x18\x02 \x03(\t\"B\n\x05\x44\x65\x64up\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x0f\n\x07\x63olumns\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"D\n\x07\x45xplode\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x0f\n\x07\x63olumns\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"=\n\x05\x46irst\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\n\n\x02\x62y\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\">\n\x06Latest\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\n\n\x02\x62y\x18\x02 \x03(\t\x12\x14\n\x0coperand_name\x18\x03 \x01(\t\"[\n\tChangelog\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x13\n\x0bkind_column\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65letes\x18\x03 \x01(\x08\x12\x14\n\x0coperand_name\x18\x04 \x01(\t\"\xcb\x01\n\x12WindowOperatorKind\x12\x12\n\noperand_id\x18\x01 \x01(\t\x12\x30\n\x0bwindow_type\x18\x02 \x01(\x0b\x32\x1b.fennel.proto.window.Window\x12\n\n\x02\x62y\x18\x03 \x03(\t\x12\r\n\x05\x66ield\x18\x04 \x01(\t\x12\x32\n\x07summary\x18\x06 \x01(\x0b\x32\x1c.fennel.proto.window.SummaryH\x00\x88\x01\x01\x12\x14\n\x0coperand_name\x18\x05 \x01(\tB\n\n\x08_summary\",\n\nDatasetRef\x12\x1e\n\x16referring_dataset_name\x18\x01 \x01(\t\"\x80\x02\n\x08\x44\x61taflow\x12\x16\n\x0c\x64\x61taset_name\x18\x01 \x01(\tH\x00\x12L\n\x11pipeline_dataflow\x18\x02 \x01(\x0b\x32/.fennel.proto.dataset.Dataflow.PipelineDataflowH\x00\x12\x0c\n\x04tags\x18\x03 \x03(\t\x1ax\n\x10PipelineDataflow\x12\x14\n\x0c\x64\x61taset_name\x18\x01 \x01(\t\x12\x15\n\rpipeline_name\x18\x02 \x01(\t\x12\x37\n\x0finput_dataflows\x18\x03 \x03(\x0b\x32\x1e.fennel.proto.dataset.DataflowB\x06\n\x04kind\"\x9c\x01\n\x10PipelineLineages\x12\x14\n\x0c\x64\x61taset_name\x18\x01 \x01(\t\x12\x15\n\rpipeline_name\x18\x02 \x01(\t\x12=\n\x0einput_datasets\x18\x03 \x03(\x0b\x32%.fennel.proto.dataset.DatasetLineages\x12\x0e\n\x06\x61\x63tive\x18\x04 \x01(\x08\x12\x0c\n\x04tags\x18\x05 \x03(\t\"\\\n\x17\x44\x61tasetPipelineLineages\x12\x41\n\x11pipeline_lineages\x18\x02 \x03(\x0b\x32&.fennel.proto.dataset.PipelineLineages\"\x8b\x01\n\x0f\x44\x61tasetLineages\x12\x18\n\x0esource_dataset\x18\x01 \x01(\tH\x00\x12H\n\x0f\x64\x65rived_dataset\x18\x02 \x01(\x0b\x32-.fennel.proto.dataset.DatasetPipelineLineagesH\x00\x12\x0c\n\x04tags\x18\x03 \x03(\tB\x06\n\x04kind*$\n\x0c\x45mitStrategy\x12\t\n\x05\x45\x61ger\x10\x00\x12\t\n\x05\x46inal\x10\x01\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dataset_pb2', globals()) @@ -41,8 +41,8 @@ _ASSIGNEXPR_OUTPUTTYPESENTRY._serialized_options = b'8\001' _RENAME_COLUMNMAPENTRY._options = None _RENAME_COLUMNMAPENTRY._serialized_options = b'8\001' - _EMITSTRATEGY._serialized_start=5219 - _EMITSTRATEGY._serialized_end=5255 + _EMITSTRATEGY._serialized_start=5234 + _EMITSTRATEGY._serialized_end=5270 _COREDATASET._serialized_start=160 _COREDATASET._serialized_end=645 _COREDATASET_FIELDMETADATAENTRY._serialized_start=560 @@ -96,19 +96,19 @@ _LATEST._serialized_start=4171 _LATEST._serialized_end=4233 _CHANGELOG._serialized_start=4235 - _CHANGELOG._serialized_end=4311 - _WINDOWOPERATORKIND._serialized_start=4314 - _WINDOWOPERATORKIND._serialized_end=4517 - _DATASETREF._serialized_start=4519 - _DATASETREF._serialized_end=4563 - _DATAFLOW._serialized_start=4566 - _DATAFLOW._serialized_end=4822 - _DATAFLOW_PIPELINEDATAFLOW._serialized_start=4694 - _DATAFLOW_PIPELINEDATAFLOW._serialized_end=4814 - _PIPELINELINEAGES._serialized_start=4825 - _PIPELINELINEAGES._serialized_end=4981 - _DATASETPIPELINELINEAGES._serialized_start=4983 - _DATASETPIPELINELINEAGES._serialized_end=5075 - _DATASETLINEAGES._serialized_start=5078 - _DATASETLINEAGES._serialized_end=5217 + _CHANGELOG._serialized_end=4326 + _WINDOWOPERATORKIND._serialized_start=4329 + _WINDOWOPERATORKIND._serialized_end=4532 + _DATASETREF._serialized_start=4534 + _DATASETREF._serialized_end=4578 + _DATAFLOW._serialized_start=4581 + _DATAFLOW._serialized_end=4837 + _DATAFLOW_PIPELINEDATAFLOW._serialized_start=4709 + _DATAFLOW_PIPELINEDATAFLOW._serialized_end=4829 + _PIPELINELINEAGES._serialized_start=4840 + _PIPELINELINEAGES._serialized_end=4996 + _DATASETPIPELINELINEAGES._serialized_start=4998 + _DATASETPIPELINELINEAGES._serialized_end=5090 + _DATASETLINEAGES._serialized_start=5093 + _DATASETLINEAGES._serialized_end=5232 # @@protoc_insertion_point(module_scope) diff --git a/fennel/gen/dataset_pb2.pyi b/fennel/gen/dataset_pb2.pyi index 602fd4de7..7f38b3279 100644 --- a/fennel/gen/dataset_pb2.pyi +++ b/fennel/gen/dataset_pb2.pyi @@ -832,10 +832,17 @@ class Changelog(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor OPERAND_ID_FIELD_NUMBER: builtins.int - DELETE_COLUMN_FIELD_NUMBER: builtins.int + KIND_COLUMN_FIELD_NUMBER: builtins.int + DELETES_FIELD_NUMBER: builtins.int OPERAND_NAME_FIELD_NUMBER: builtins.int operand_id: builtins.str - delete_column: builtins.str + kind_column: builtins.str + """Name of column which will store kind value as a boolean""" + deletes: builtins.bool + """If deletes is true then the kind column will store true + for deletes and false for inserts. If deletes is false then the + kind column will store true for inserts and false for deletes. + """ operand_name: builtins.str """NOTE: FOLLOWING PROPERTIES ARE SET BY THE SERVER AND WILL BE IGNORED BY THE CLIENT @@ -844,10 +851,11 @@ class Changelog(google.protobuf.message.Message): self, *, operand_id: builtins.str = ..., - delete_column: builtins.str = ..., + kind_column: builtins.str = ..., + deletes: builtins.bool = ..., operand_name: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["delete_column", b"delete_column", "operand_id", b"operand_id", "operand_name", b"operand_name"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["deletes", b"deletes", "kind_column", b"kind_column", "operand_id", b"operand_id", "operand_name", b"operand_name"]) -> None: ... global___Changelog = Changelog diff --git a/fennel/gen/expression_pb2.py b/fennel/gen/expression_pb2.py index f744e3e90..ab582c61c 100644 --- a/fennel/gen/expression_pb2.py +++ b/fennel/gen/expression_pb2.py @@ -14,7 +14,7 @@ import fennel.gen.schema_pb2 as schema__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x65xpression.proto\x12\x17\x66\x65nnel.proto.expression\x1a\x0cschema.proto\"9\n\x0b\x45valContext\x12\x19\n\x0cnow_col_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0f\n\r_now_col_name\"\xd4\x07\n\x04\x45xpr\x12+\n\x03ref\x18\x01 \x01(\x0b\x32\x1c.fennel.proto.expression.RefH\x00\x12<\n\x0cjson_literal\x18\x02 \x01(\x0b\x32$.fennel.proto.expression.JsonLiteralH\x00\x12/\n\x05unary\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.expression.UnaryH\x00\x12-\n\x04\x63\x61se\x18\x05 \x01(\x0b\x32\x1d.fennel.proto.expression.CaseH\x00\x12\x31\n\x06\x62inary\x18\x06 \x01(\x0b\x32\x1f.fennel.proto.expression.BinaryH\x00\x12\x31\n\x06isnull\x18\x07 \x01(\x0b\x32\x1f.fennel.proto.expression.IsNullH\x00\x12\x35\n\x08\x66illnull\x18\x08 \x01(\x0b\x32!.fennel.proto.expression.FillNullH\x00\x12\x32\n\x07list_fn\x18\t \x01(\x0b\x32\x1f.fennel.proto.expression.ListFnH\x00\x12\x32\n\x07math_fn\x18\n \x01(\x0b\x32\x1f.fennel.proto.expression.MathFnH\x00\x12\x36\n\tstruct_fn\x18\x0b \x01(\x0b\x32!.fennel.proto.expression.StructFnH\x00\x12\x32\n\x07\x64ict_fn\x18\x0c \x01(\x0b\x32\x1f.fennel.proto.expression.DictFnH\x00\x12\x36\n\tstring_fn\x18\r \x01(\x0b\x32!.fennel.proto.expression.StringFnH\x00\x12:\n\x0b\x64\x61tetime_fn\x18\x0e \x01(\x0b\x32#.fennel.proto.expression.DateTimeFnH\x00\x12\x44\n\x10\x64\x61tetime_literal\x18\x0f \x01(\x0b\x32(.fennel.proto.expression.DatetimeLiteralH\x00\x12:\n\x0bmake_struct\x18\x10 \x01(\x0b\x32#.fennel.proto.expression.MakeStructH\x00\x12\x38\n\nfrom_epoch\x18\x11 \x01(\x0b\x32\".fennel.proto.expression.FromEpochH\x00\x12+\n\x03var\x18\x12 \x01(\x0b\x32\x1c.fennel.proto.expression.VarH\x00\x12+\n\x03now\x18\x13 \x01(\x0b\x32\x1c.fennel.proto.expression.NowH\x00\x42\x06\n\x04node\"\x05\n\x03Now\"\x13\n\x03Var\x12\x0c\n\x04name\x18\x01 \x01(\t\"m\n\tFromEpoch\x12/\n\x08\x64uration\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12/\n\x04unit\x18\x02 \x01(\x0e\x32!.fennel.proto.expression.TimeUnit\"\xb3\x01\n\x0f\x44\x61tetimeLiteral\x12\x0c\n\x04year\x18\x01 \x01(\r\x12\r\n\x05month\x18\x02 \x01(\r\x12\x0b\n\x03\x64\x61y\x18\x03 \x01(\r\x12\x0c\n\x04hour\x18\x04 \x01(\r\x12\x0e\n\x06minute\x18\x05 \x01(\r\x12\x0e\n\x06second\x18\x06 \x01(\r\x12\x13\n\x0bmicrosecond\x18\x07 \x01(\r\x12\x33\n\x08timezone\x18\x08 \x01(\x0b\x32!.fennel.proto.expression.Timezone\"\xd1\x01\n\nMakeStruct\x12\x34\n\x0bstruct_type\x18\x01 \x01(\x0b\x32\x1f.fennel.proto.schema.StructType\x12?\n\x06\x66ields\x18\x02 \x03(\x0b\x32/.fennel.proto.expression.MakeStruct.FieldsEntry\x1aL\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr:\x02\x38\x01\"L\n\x0bJsonLiteral\x12\x0f\n\x07literal\x18\x01 \x01(\t\x12,\n\x05\x64type\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType\"\x13\n\x03Ref\x12\x0c\n\x04name\x18\x01 \x01(\t\"e\n\x05Unary\x12,\n\x02op\x18\x01 \x01(\x0e\x32 .fennel.proto.expression.UnaryOp\x12.\n\x07operand\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"\x8f\x01\n\x06\x42inary\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12,\n\x05right\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12*\n\x02op\x18\x03 \x01(\x0e\x32\x1e.fennel.proto.expression.BinOp\"n\n\x04\x43\x61se\x12\x34\n\twhen_then\x18\x01 \x03(\x0b\x32!.fennel.proto.expression.WhenThen\x12\x30\n\totherwise\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"d\n\x08WhenThen\x12+\n\x04when\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x04then\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"8\n\x06IsNull\x12.\n\x07operand\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"g\n\x08\x46illNull\x12.\n\x07operand\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x04\x66ill\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"\xeb\x04\n\x06ListOp\x12+\n\x03len\x18\x01 \x01(\x0b\x32\x1c.fennel.proto.expression.LenH\x00\x12,\n\x03get\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.ExprH\x00\x12\x35\n\x08\x63ontains\x18\x03 \x01(\x0b\x32!.fennel.proto.expression.ContainsH\x00\x12\x34\n\x08has_null\x18\x04 \x01(\x0b\x32 .fennel.proto.expression.HasNullH\x00\x12/\n\x03sum\x18\x05 \x01(\x0b\x32 .fennel.proto.expression.ListSumH\x00\x12/\n\x03min\x18\x06 \x01(\x0b\x32 .fennel.proto.expression.ListMinH\x00\x12/\n\x03max\x18\x07 \x01(\x0b\x32 .fennel.proto.expression.ListMaxH\x00\x12/\n\x03\x61ll\x18\x08 \x01(\x0b\x32 .fennel.proto.expression.ListAllH\x00\x12/\n\x03\x61ny\x18\t \x01(\x0b\x32 .fennel.proto.expression.ListAnyH\x00\x12\x31\n\x04mean\x18\n \x01(\x0b\x32!.fennel.proto.expression.ListMeanH\x00\x12\x35\n\x06\x66ilter\x18\x0b \x01(\x0b\x32#.fennel.proto.expression.ListFilterH\x00\x12/\n\x03map\x18\x0c \x01(\x0b\x32 .fennel.proto.expression.ListMapH\x00\x42\t\n\x07\x66n_type\"K\n\nListFilter\x12\x0b\n\x03var\x18\x01 \x01(\t\x12\x30\n\tpredicate\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"G\n\x07ListMap\x12\x0b\n\x03var\x18\x01 \x01(\t\x12/\n\x08map_expr\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"\t\n\x07ListSum\"\t\n\x07ListMin\"\n\n\x08ListMean\"\t\n\x07ListMax\"\t\n\x07ListAll\"\t\n\x07ListAny\"\x05\n\x03Len\"\t\n\x07HasNull\":\n\x08\x43ontains\x12.\n\x07\x65lement\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"b\n\x06ListFn\x12+\n\x04list\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x02\x66n\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.expression.ListOp\"\xd1\x01\n\x06MathOp\x12/\n\x05round\x18\x01 \x01(\x0b\x32\x1e.fennel.proto.expression.RoundH\x00\x12+\n\x03\x61\x62s\x18\x02 \x01(\x0b\x32\x1c.fennel.proto.expression.AbsH\x00\x12-\n\x04\x63\x65il\x18\x03 \x01(\x0b\x32\x1d.fennel.proto.expression.CeilH\x00\x12/\n\x05\x66loor\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.expression.FloorH\x00\x42\t\n\x07\x66n_type\"\x1a\n\x05Round\x12\x11\n\tprecision\x18\x01 \x01(\x05\"\x05\n\x03\x41\x62s\"\x06\n\x04\x43\x65il\"\x07\n\x05\x46loor\"e\n\x06MathFn\x12.\n\x07operand\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x02\x66n\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.expression.MathOp\"&\n\x08StructOp\x12\x0f\n\x05\x66ield\x18\x01 \x01(\tH\x00\x42\t\n\x07\x66n_type\"h\n\x08StructFn\x12-\n\x06struct\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12-\n\x02\x66n\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.StructOp\"m\n\x07\x44ictGet\x12,\n\x05\x66ield\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12\x34\n\rdefault_value\x18\x03 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"\xa8\x01\n\x06\x44ictOp\x12+\n\x03len\x18\x01 \x01(\x0b\x32\x1c.fennel.proto.expression.LenH\x00\x12/\n\x03get\x18\x02 \x01(\x0b\x32 .fennel.proto.expression.DictGetH\x00\x12\x35\n\x08\x63ontains\x18\x03 \x01(\x0b\x32!.fennel.proto.expression.ContainsH\x00\x42\t\n\x07\x66n_type\"b\n\x06\x44ictFn\x12+\n\x04\x64ict\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x02\x66n\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.expression.DictOp\"\xfb\x03\n\x08StringOp\x12+\n\x03len\x18\x01 \x01(\x0b\x32\x1c.fennel.proto.expression.LenH\x00\x12\x33\n\x07tolower\x18\x02 \x01(\x0b\x32 .fennel.proto.expression.ToLowerH\x00\x12\x33\n\x07toupper\x18\x03 \x01(\x0b\x32 .fennel.proto.expression.ToUpperH\x00\x12\x35\n\x08\x63ontains\x18\x04 \x01(\x0b\x32!.fennel.proto.expression.ContainsH\x00\x12\x39\n\nstartswith\x18\x05 \x01(\x0b\x32#.fennel.proto.expression.StartsWithH\x00\x12\x35\n\x08\x65ndswith\x18\x06 \x01(\x0b\x32!.fennel.proto.expression.EndsWithH\x00\x12\x31\n\x06\x63oncat\x18\x07 \x01(\x0b\x32\x1f.fennel.proto.expression.ConcatH\x00\x12\x35\n\x08strptime\x18\x08 \x01(\x0b\x32!.fennel.proto.expression.StrptimeH\x00\x12:\n\x0bjson_decode\x18\t \x01(\x0b\x32#.fennel.proto.expression.JsonDecodeH\x00\x42\t\n\x07\x66n_type\"\x1c\n\x08Timezone\x12\x10\n\x08timezone\x18\x01 \x01(\t\":\n\nJsonDecode\x12,\n\x05\x64type\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType\"O\n\x08Strptime\x12\x0e\n\x06\x66ormat\x18\x01 \x01(\t\x12\x33\n\x08timezone\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.Timezone\"\t\n\x07ToLower\"\t\n\x07ToUpper\"8\n\nStartsWith\x12*\n\x03key\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"6\n\x08\x45ndsWith\x12*\n\x03key\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"6\n\x06\x43oncat\x12,\n\x05other\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"h\n\x08StringFn\x12-\n\x06string\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12-\n\x02\x66n\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.StringOp\"n\n\nDateTimeFn\x12/\n\x08\x64\x61tetime\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12/\n\x02\x66n\x18\x02 \x01(\x0b\x32#.fennel.proto.expression.DateTimeOp\"\xea\x01\n\nDateTimeOp\x12/\n\x05since\x18\x01 \x01(\x0b\x32\x1e.fennel.proto.expression.SinceH\x00\x12:\n\x0bsince_epoch\x18\x02 \x01(\x0b\x32#.fennel.proto.expression.SinceEpochH\x00\x12\x35\n\x08strftime\x18\x03 \x01(\x0b\x32!.fennel.proto.expression.StrftimeH\x00\x12-\n\x04part\x18\x04 \x01(\x0b\x32\x1d.fennel.proto.expression.PartH\x00\x42\t\n\x07\x66n_type\"f\n\x05Since\x12,\n\x05other\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12/\n\x04unit\x18\x02 \x01(\x0e\x32!.fennel.proto.expression.TimeUnit\"=\n\nSinceEpoch\x12/\n\x04unit\x18\x01 \x01(\x0e\x32!.fennel.proto.expression.TimeUnit\"O\n\x08Strftime\x12\x0e\n\x06\x66ormat\x18\x01 \x01(\t\x12\x33\n\x08timezone\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.Timezone\"l\n\x04Part\x12/\n\x04unit\x18\x01 \x01(\x0e\x32!.fennel.proto.expression.TimeUnit\x12\x33\n\x08timezone\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.Timezone*\x1b\n\x07UnaryOp\x12\x07\n\x03NEG\x10\x00\x12\x07\n\x03NOT\x10\x01*\x86\x01\n\x05\x42inOp\x12\x07\n\x03\x41\x44\x44\x10\x00\x12\x07\n\x03SUB\x10\x01\x12\x07\n\x03MUL\x10\x02\x12\x07\n\x03\x44IV\x10\x03\x12\x07\n\x03MOD\x10\x04\x12\r\n\tFLOOR_DIV\x10\x05\x12\x06\n\x02\x45Q\x10\x06\x12\x06\n\x02NE\x10\x07\x12\x06\n\x02GT\x10\x08\x12\x07\n\x03GTE\x10\t\x12\x06\n\x02LT\x10\n\x12\x07\n\x03LTE\x10\x0b\x12\x07\n\x03\x41ND\x10\x0c\x12\x06\n\x02OR\x10\r*\x83\x01\n\x08TimeUnit\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06SECOND\x10\x01\x12\n\n\x06MINUTE\x10\x02\x12\x08\n\x04HOUR\x10\x03\x12\x07\n\x03\x44\x41Y\x10\x04\x12\x08\n\x04WEEK\x10\x05\x12\t\n\x05MONTH\x10\x06\x12\x08\n\x04YEAR\x10\x07\x12\x0f\n\x0bMICROSECOND\x10\x08\x12\x0f\n\x0bMILLISECOND\x10\tb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x65xpression.proto\x12\x17\x66\x65nnel.proto.expression\x1a\x0cschema.proto\"9\n\x0b\x45valContext\x12\x19\n\x0cnow_col_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0f\n\r_now_col_name\"\xd4\x07\n\x04\x45xpr\x12+\n\x03ref\x18\x01 \x01(\x0b\x32\x1c.fennel.proto.expression.RefH\x00\x12<\n\x0cjson_literal\x18\x02 \x01(\x0b\x32$.fennel.proto.expression.JsonLiteralH\x00\x12/\n\x05unary\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.expression.UnaryH\x00\x12-\n\x04\x63\x61se\x18\x05 \x01(\x0b\x32\x1d.fennel.proto.expression.CaseH\x00\x12\x31\n\x06\x62inary\x18\x06 \x01(\x0b\x32\x1f.fennel.proto.expression.BinaryH\x00\x12\x31\n\x06isnull\x18\x07 \x01(\x0b\x32\x1f.fennel.proto.expression.IsNullH\x00\x12\x35\n\x08\x66illnull\x18\x08 \x01(\x0b\x32!.fennel.proto.expression.FillNullH\x00\x12\x32\n\x07list_fn\x18\t \x01(\x0b\x32\x1f.fennel.proto.expression.ListFnH\x00\x12\x32\n\x07math_fn\x18\n \x01(\x0b\x32\x1f.fennel.proto.expression.MathFnH\x00\x12\x36\n\tstruct_fn\x18\x0b \x01(\x0b\x32!.fennel.proto.expression.StructFnH\x00\x12\x32\n\x07\x64ict_fn\x18\x0c \x01(\x0b\x32\x1f.fennel.proto.expression.DictFnH\x00\x12\x36\n\tstring_fn\x18\r \x01(\x0b\x32!.fennel.proto.expression.StringFnH\x00\x12:\n\x0b\x64\x61tetime_fn\x18\x0e \x01(\x0b\x32#.fennel.proto.expression.DateTimeFnH\x00\x12\x44\n\x10\x64\x61tetime_literal\x18\x0f \x01(\x0b\x32(.fennel.proto.expression.DatetimeLiteralH\x00\x12:\n\x0bmake_struct\x18\x10 \x01(\x0b\x32#.fennel.proto.expression.MakeStructH\x00\x12\x38\n\nfrom_epoch\x18\x11 \x01(\x0b\x32\".fennel.proto.expression.FromEpochH\x00\x12+\n\x03var\x18\x12 \x01(\x0b\x32\x1c.fennel.proto.expression.VarH\x00\x12+\n\x03now\x18\x13 \x01(\x0b\x32\x1c.fennel.proto.expression.NowH\x00\x42\x06\n\x04node\"\x05\n\x03Now\"\x13\n\x03Var\x12\x0c\n\x04name\x18\x01 \x01(\t\"m\n\tFromEpoch\x12/\n\x08\x64uration\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12/\n\x04unit\x18\x02 \x01(\x0e\x32!.fennel.proto.expression.TimeUnit\"\xb3\x01\n\x0f\x44\x61tetimeLiteral\x12\x0c\n\x04year\x18\x01 \x01(\r\x12\r\n\x05month\x18\x02 \x01(\r\x12\x0b\n\x03\x64\x61y\x18\x03 \x01(\r\x12\x0c\n\x04hour\x18\x04 \x01(\r\x12\x0e\n\x06minute\x18\x05 \x01(\r\x12\x0e\n\x06second\x18\x06 \x01(\r\x12\x13\n\x0bmicrosecond\x18\x07 \x01(\r\x12\x33\n\x08timezone\x18\x08 \x01(\x0b\x32!.fennel.proto.expression.Timezone\"\xd1\x01\n\nMakeStruct\x12\x34\n\x0bstruct_type\x18\x01 \x01(\x0b\x32\x1f.fennel.proto.schema.StructType\x12?\n\x06\x66ields\x18\x02 \x03(\x0b\x32/.fennel.proto.expression.MakeStruct.FieldsEntry\x1aL\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr:\x02\x38\x01\"L\n\x0bJsonLiteral\x12\x0f\n\x07literal\x18\x01 \x01(\t\x12,\n\x05\x64type\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType\"\x13\n\x03Ref\x12\x0c\n\x04name\x18\x01 \x01(\t\"e\n\x05Unary\x12,\n\x02op\x18\x01 \x01(\x0e\x32 .fennel.proto.expression.UnaryOp\x12.\n\x07operand\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"\x8f\x01\n\x06\x42inary\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12,\n\x05right\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12*\n\x02op\x18\x03 \x01(\x0e\x32\x1e.fennel.proto.expression.BinOp\"n\n\x04\x43\x61se\x12\x34\n\twhen_then\x18\x01 \x03(\x0b\x32!.fennel.proto.expression.WhenThen\x12\x30\n\totherwise\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"d\n\x08WhenThen\x12+\n\x04when\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x04then\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"8\n\x06IsNull\x12.\n\x07operand\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"g\n\x08\x46illNull\x12.\n\x07operand\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x04\x66ill\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"\xeb\x04\n\x06ListOp\x12+\n\x03len\x18\x01 \x01(\x0b\x32\x1c.fennel.proto.expression.LenH\x00\x12,\n\x03get\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.ExprH\x00\x12\x35\n\x08\x63ontains\x18\x03 \x01(\x0b\x32!.fennel.proto.expression.ContainsH\x00\x12\x34\n\x08has_null\x18\x04 \x01(\x0b\x32 .fennel.proto.expression.HasNullH\x00\x12/\n\x03sum\x18\x05 \x01(\x0b\x32 .fennel.proto.expression.ListSumH\x00\x12/\n\x03min\x18\x06 \x01(\x0b\x32 .fennel.proto.expression.ListMinH\x00\x12/\n\x03max\x18\x07 \x01(\x0b\x32 .fennel.proto.expression.ListMaxH\x00\x12/\n\x03\x61ll\x18\x08 \x01(\x0b\x32 .fennel.proto.expression.ListAllH\x00\x12/\n\x03\x61ny\x18\t \x01(\x0b\x32 .fennel.proto.expression.ListAnyH\x00\x12\x31\n\x04mean\x18\n \x01(\x0b\x32!.fennel.proto.expression.ListMeanH\x00\x12\x35\n\x06\x66ilter\x18\x0b \x01(\x0b\x32#.fennel.proto.expression.ListFilterH\x00\x12/\n\x03map\x18\x0c \x01(\x0b\x32 .fennel.proto.expression.ListMapH\x00\x42\t\n\x07\x66n_type\"K\n\nListFilter\x12\x0b\n\x03var\x18\x01 \x01(\t\x12\x30\n\tpredicate\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"G\n\x07ListMap\x12\x0b\n\x03var\x18\x01 \x01(\t\x12/\n\x08map_expr\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"\t\n\x07ListSum\"\t\n\x07ListMin\"\n\n\x08ListMean\"\t\n\x07ListMax\"\t\n\x07ListAll\"\t\n\x07ListAny\"\x05\n\x03Len\"\t\n\x07HasNull\":\n\x08\x43ontains\x12.\n\x07\x65lement\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"b\n\x06ListFn\x12+\n\x04list\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x02\x66n\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.expression.ListOp\"\xd1\x01\n\x06MathOp\x12/\n\x05round\x18\x01 \x01(\x0b\x32\x1e.fennel.proto.expression.RoundH\x00\x12+\n\x03\x61\x62s\x18\x02 \x01(\x0b\x32\x1c.fennel.proto.expression.AbsH\x00\x12-\n\x04\x63\x65il\x18\x03 \x01(\x0b\x32\x1d.fennel.proto.expression.CeilH\x00\x12/\n\x05\x66loor\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.expression.FloorH\x00\x42\t\n\x07\x66n_type\"\x1a\n\x05Round\x12\x11\n\tprecision\x18\x01 \x01(\x05\"\x05\n\x03\x41\x62s\"\x06\n\x04\x43\x65il\"\x07\n\x05\x46loor\"e\n\x06MathFn\x12.\n\x07operand\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x02\x66n\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.expression.MathOp\"&\n\x08StructOp\x12\x0f\n\x05\x66ield\x18\x01 \x01(\tH\x00\x42\t\n\x07\x66n_type\"h\n\x08StructFn\x12-\n\x06struct\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12-\n\x02\x66n\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.StructOp\"m\n\x07\x44ictGet\x12,\n\x05\x66ield\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12\x34\n\rdefault_value\x18\x03 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"\xa8\x01\n\x06\x44ictOp\x12+\n\x03len\x18\x01 \x01(\x0b\x32\x1c.fennel.proto.expression.LenH\x00\x12/\n\x03get\x18\x02 \x01(\x0b\x32 .fennel.proto.expression.DictGetH\x00\x12\x35\n\x08\x63ontains\x18\x03 \x01(\x0b\x32!.fennel.proto.expression.ContainsH\x00\x42\t\n\x07\x66n_type\"b\n\x06\x44ictFn\x12+\n\x04\x64ict\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12+\n\x02\x66n\x18\x02 \x01(\x0b\x32\x1f.fennel.proto.expression.DictOp\"\x9c\x05\n\x08StringOp\x12+\n\x03len\x18\x01 \x01(\x0b\x32\x1c.fennel.proto.expression.LenH\x00\x12\x33\n\x07tolower\x18\x02 \x01(\x0b\x32 .fennel.proto.expression.ToLowerH\x00\x12\x33\n\x07toupper\x18\x03 \x01(\x0b\x32 .fennel.proto.expression.ToUpperH\x00\x12\x35\n\x08\x63ontains\x18\x04 \x01(\x0b\x32!.fennel.proto.expression.ContainsH\x00\x12\x39\n\nstartswith\x18\x05 \x01(\x0b\x32#.fennel.proto.expression.StartsWithH\x00\x12\x35\n\x08\x65ndswith\x18\x06 \x01(\x0b\x32!.fennel.proto.expression.EndsWithH\x00\x12\x31\n\x06\x63oncat\x18\x07 \x01(\x0b\x32\x1f.fennel.proto.expression.ConcatH\x00\x12\x35\n\x08strptime\x18\x08 \x01(\x0b\x32!.fennel.proto.expression.StrptimeH\x00\x12:\n\x0bjson_decode\x18\t \x01(\x0b\x32#.fennel.proto.expression.JsonDecodeH\x00\x12/\n\x05split\x18\n \x01(\x0b\x32\x1e.fennel.proto.expression.SplitH\x00\x12<\n\x0cjson_extract\x18\x0b \x01(\x0b\x32$.fennel.proto.expression.JsonExtractH\x00\x12\x30\n\x06to_int\x18\x0c \x01(\x0b\x32\x1e.fennel.proto.expression.ToIntH\x00\x42\t\n\x07\x66n_type\"\x1c\n\x08Timezone\x12\x10\n\x08timezone\x18\x01 \x01(\t\":\n\nJsonDecode\x12,\n\x05\x64type\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.schema.DataType\"O\n\x08Strptime\x12\x0e\n\x06\x66ormat\x18\x01 \x01(\t\x12\x33\n\x08timezone\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.Timezone\"\t\n\x07ToLower\"\t\n\x07ToUpper\"8\n\nStartsWith\x12*\n\x03key\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"6\n\x08\x45ndsWith\x12*\n\x03key\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"6\n\x06\x43oncat\x12,\n\x05other\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\"h\n\x08StringFn\x12-\n\x06string\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12-\n\x02\x66n\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.StringOp\"n\n\nDateTimeFn\x12/\n\x08\x64\x61tetime\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12/\n\x02\x66n\x18\x02 \x01(\x0b\x32#.fennel.proto.expression.DateTimeOp\"\xea\x01\n\nDateTimeOp\x12/\n\x05since\x18\x01 \x01(\x0b\x32\x1e.fennel.proto.expression.SinceH\x00\x12:\n\x0bsince_epoch\x18\x02 \x01(\x0b\x32#.fennel.proto.expression.SinceEpochH\x00\x12\x35\n\x08strftime\x18\x03 \x01(\x0b\x32!.fennel.proto.expression.StrftimeH\x00\x12-\n\x04part\x18\x04 \x01(\x0b\x32\x1d.fennel.proto.expression.PartH\x00\x42\t\n\x07\x66n_type\"f\n\x05Since\x12,\n\x05other\x18\x01 \x01(\x0b\x32\x1d.fennel.proto.expression.Expr\x12/\n\x04unit\x18\x02 \x01(\x0e\x32!.fennel.proto.expression.TimeUnit\"=\n\nSinceEpoch\x12/\n\x04unit\x18\x01 \x01(\x0e\x32!.fennel.proto.expression.TimeUnit\"O\n\x08Strftime\x12\x0e\n\x06\x66ormat\x18\x01 \x01(\t\x12\x33\n\x08timezone\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.Timezone\"l\n\x04Part\x12/\n\x04unit\x18\x01 \x01(\x0e\x32!.fennel.proto.expression.TimeUnit\x12\x33\n\x08timezone\x18\x02 \x01(\x0b\x32!.fennel.proto.expression.Timezone\"\x14\n\x05Split\x12\x0b\n\x03sep\x18\x01 \x01(\t\"\x1b\n\x0bJsonExtract\x12\x0c\n\x04path\x18\x01 \x01(\t\"\x07\n\x05ToInt*\x1b\n\x07UnaryOp\x12\x07\n\x03NEG\x10\x00\x12\x07\n\x03NOT\x10\x01*\x86\x01\n\x05\x42inOp\x12\x07\n\x03\x41\x44\x44\x10\x00\x12\x07\n\x03SUB\x10\x01\x12\x07\n\x03MUL\x10\x02\x12\x07\n\x03\x44IV\x10\x03\x12\x07\n\x03MOD\x10\x04\x12\r\n\tFLOOR_DIV\x10\x05\x12\x06\n\x02\x45Q\x10\x06\x12\x06\n\x02NE\x10\x07\x12\x06\n\x02GT\x10\x08\x12\x07\n\x03GTE\x10\t\x12\x06\n\x02LT\x10\n\x12\x07\n\x03LTE\x10\x0b\x12\x07\n\x03\x41ND\x10\x0c\x12\x06\n\x02OR\x10\r*\x83\x01\n\x08TimeUnit\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06SECOND\x10\x01\x12\n\n\x06MINUTE\x10\x02\x12\x08\n\x04HOUR\x10\x03\x12\x07\n\x03\x44\x41Y\x10\x04\x12\x08\n\x04WEEK\x10\x05\x12\t\n\x05MONTH\x10\x06\x12\x08\n\x04YEAR\x10\x07\x12\x0f\n\x0bMICROSECOND\x10\x08\x12\x0f\n\x0bMILLISECOND\x10\tb\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'expression_pb2', globals()) @@ -23,12 +23,12 @@ DESCRIPTOR._options = None _MAKESTRUCT_FIELDSENTRY._options = None _MAKESTRUCT_FIELDSENTRY._serialized_options = b'8\001' - _UNARYOP._serialized_start=5957 - _UNARYOP._serialized_end=5984 - _BINOP._serialized_start=5987 - _BINOP._serialized_end=6121 - _TIMEUNIT._serialized_start=6124 - _TIMEUNIT._serialized_end=6255 + _UNARYOP._serialized_start=6178 + _UNARYOP._serialized_end=6205 + _BINOP._serialized_start=6208 + _BINOP._serialized_end=6342 + _TIMEUNIT._serialized_start=6345 + _TIMEUNIT._serialized_end=6476 _EVALCONTEXT._serialized_start=59 _EVALCONTEXT._serialized_end=116 _EXPR._serialized_start=119 @@ -110,35 +110,41 @@ _DICTFN._serialized_start=4171 _DICTFN._serialized_end=4269 _STRINGOP._serialized_start=4272 - _STRINGOP._serialized_end=4779 - _TIMEZONE._serialized_start=4781 - _TIMEZONE._serialized_end=4809 - _JSONDECODE._serialized_start=4811 - _JSONDECODE._serialized_end=4869 - _STRPTIME._serialized_start=4871 - _STRPTIME._serialized_end=4950 - _TOLOWER._serialized_start=4952 - _TOLOWER._serialized_end=4961 - _TOUPPER._serialized_start=4963 - _TOUPPER._serialized_end=4972 - _STARTSWITH._serialized_start=4974 - _STARTSWITH._serialized_end=5030 - _ENDSWITH._serialized_start=5032 - _ENDSWITH._serialized_end=5086 - _CONCAT._serialized_start=5088 - _CONCAT._serialized_end=5142 - _STRINGFN._serialized_start=5144 - _STRINGFN._serialized_end=5248 - _DATETIMEFN._serialized_start=5250 - _DATETIMEFN._serialized_end=5360 - _DATETIMEOP._serialized_start=5363 - _DATETIMEOP._serialized_end=5597 - _SINCE._serialized_start=5599 - _SINCE._serialized_end=5701 - _SINCEEPOCH._serialized_start=5703 - _SINCEEPOCH._serialized_end=5764 - _STRFTIME._serialized_start=5766 - _STRFTIME._serialized_end=5845 - _PART._serialized_start=5847 - _PART._serialized_end=5955 + _STRINGOP._serialized_end=4940 + _TIMEZONE._serialized_start=4942 + _TIMEZONE._serialized_end=4970 + _JSONDECODE._serialized_start=4972 + _JSONDECODE._serialized_end=5030 + _STRPTIME._serialized_start=5032 + _STRPTIME._serialized_end=5111 + _TOLOWER._serialized_start=5113 + _TOLOWER._serialized_end=5122 + _TOUPPER._serialized_start=5124 + _TOUPPER._serialized_end=5133 + _STARTSWITH._serialized_start=5135 + _STARTSWITH._serialized_end=5191 + _ENDSWITH._serialized_start=5193 + _ENDSWITH._serialized_end=5247 + _CONCAT._serialized_start=5249 + _CONCAT._serialized_end=5303 + _STRINGFN._serialized_start=5305 + _STRINGFN._serialized_end=5409 + _DATETIMEFN._serialized_start=5411 + _DATETIMEFN._serialized_end=5521 + _DATETIMEOP._serialized_start=5524 + _DATETIMEOP._serialized_end=5758 + _SINCE._serialized_start=5760 + _SINCE._serialized_end=5862 + _SINCEEPOCH._serialized_start=5864 + _SINCEEPOCH._serialized_end=5925 + _STRFTIME._serialized_start=5927 + _STRFTIME._serialized_end=6006 + _PART._serialized_start=6008 + _PART._serialized_end=6116 + _SPLIT._serialized_start=6118 + _SPLIT._serialized_end=6138 + _JSONEXTRACT._serialized_start=6140 + _JSONEXTRACT._serialized_end=6167 + _TOINT._serialized_start=6169 + _TOINT._serialized_end=6176 # @@protoc_insertion_point(module_scope) diff --git a/fennel/gen/expression_pb2.pyi b/fennel/gen/expression_pb2.pyi index 9d6c2d73b..e5b03387e 100644 --- a/fennel/gen/expression_pb2.pyi +++ b/fennel/gen/expression_pb2.pyi @@ -933,6 +933,9 @@ class StringOp(google.protobuf.message.Message): CONCAT_FIELD_NUMBER: builtins.int STRPTIME_FIELD_NUMBER: builtins.int JSON_DECODE_FIELD_NUMBER: builtins.int + SPLIT_FIELD_NUMBER: builtins.int + JSON_EXTRACT_FIELD_NUMBER: builtins.int + TO_INT_FIELD_NUMBER: builtins.int @property def len(self) -> global___Len: ... @property @@ -951,6 +954,12 @@ class StringOp(google.protobuf.message.Message): def strptime(self) -> global___Strptime: ... @property def json_decode(self) -> global___JsonDecode: ... + @property + def split(self) -> global___Split: ... + @property + def json_extract(self) -> global___JsonExtract: ... + @property + def to_int(self) -> global___ToInt: ... def __init__( self, *, @@ -963,10 +972,13 @@ class StringOp(google.protobuf.message.Message): concat: global___Concat | None = ..., strptime: global___Strptime | None = ..., json_decode: global___JsonDecode | None = ..., + split: global___Split | None = ..., + json_extract: global___JsonExtract | None = ..., + to_int: global___ToInt | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["concat", b"concat", "contains", b"contains", "endswith", b"endswith", "fn_type", b"fn_type", "json_decode", b"json_decode", "len", b"len", "startswith", b"startswith", "strptime", b"strptime", "tolower", b"tolower", "toupper", b"toupper"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["concat", b"concat", "contains", b"contains", "endswith", b"endswith", "fn_type", b"fn_type", "json_decode", b"json_decode", "len", b"len", "startswith", b"startswith", "strptime", b"strptime", "tolower", b"tolower", "toupper", b"toupper"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["fn_type", b"fn_type"]) -> typing_extensions.Literal["len", "tolower", "toupper", "contains", "startswith", "endswith", "concat", "strptime", "json_decode"] | None: ... + def HasField(self, field_name: typing_extensions.Literal["concat", b"concat", "contains", b"contains", "endswith", b"endswith", "fn_type", b"fn_type", "json_decode", b"json_decode", "json_extract", b"json_extract", "len", b"len", "split", b"split", "startswith", b"startswith", "strptime", b"strptime", "to_int", b"to_int", "tolower", b"tolower", "toupper", b"toupper"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["concat", b"concat", "contains", b"contains", "endswith", b"endswith", "fn_type", b"fn_type", "json_decode", b"json_decode", "json_extract", b"json_extract", "len", b"len", "split", b"split", "startswith", b"startswith", "strptime", b"strptime", "to_int", b"to_int", "tolower", b"tolower", "toupper", b"toupper"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["fn_type", b"fn_type"]) -> typing_extensions.Literal["len", "tolower", "toupper", "contains", "startswith", "endswith", "concat", "strptime", "json_decode", "split", "json_extract", "to_int"] | None: ... global___StringOp = StringOp @@ -1239,3 +1251,43 @@ class Part(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["timezone", b"timezone", "unit", b"unit"]) -> None: ... global___Part = Part + +@typing_extensions.final +class Split(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEP_FIELD_NUMBER: builtins.int + sep: builtins.str + def __init__( + self, + *, + sep: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["sep", b"sep"]) -> None: ... + +global___Split = Split + +@typing_extensions.final +class JsonExtract(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PATH_FIELD_NUMBER: builtins.int + path: builtins.str + def __init__( + self, + *, + path: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["path", b"path"]) -> None: ... + +global___JsonExtract = JsonExtract + +@typing_extensions.final +class ToInt(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___ToInt = ToInt diff --git a/fennel/internal_lib/to_proto/serializer.py b/fennel/internal_lib/to_proto/serializer.py index 95ff21cfd..2f4da56c1 100644 --- a/fennel/internal_lib/to_proto/serializer.py +++ b/fennel/internal_lib/to_proto/serializer.py @@ -386,6 +386,7 @@ def visitChangelog(self, obj): ds_version=self.dataset_version, changelog=proto.Changelog( operand_id=self.visit(obj.node), - delete_column=obj.delete_column, + kind_column=obj.kind_column, + deletes=obj.deletes, ), ) diff --git a/fennel/testing/executor.py b/fennel/testing/executor.py index 2449b4cd8..b3ad9c111 100644 --- a/fennel/testing/executor.py +++ b/fennel/testing/executor.py @@ -954,8 +954,8 @@ def visitChangelog(self, obj): delete_df[input_ret.timestamp_field] = delete_df[ FENNEL_DELETE_TIMESTAMP ] - df[obj.delete_column] = False - delete_df[obj.delete_column] = True + df[obj.kind_column] = False if obj.deletes else True + delete_df[obj.kind_column] = True if obj.deletes else False df = pd.concat([df, delete_df]) df = df.drop(columns=[FENNEL_DELETE_TIMESTAMP]) # Sort the dataframe by timestamp diff --git a/pyproject.toml b/pyproject.toml index d175fa58f..96e647542 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fennel-ai" -version = "1.5.39" +version = "1.5.40" description = "The modern realtime feature engineering platform" authors = ["Fennel AI "] packages = [{ include = "fennel" }]