Merge branch 'main' into rk/fix-stop-url-for-algolia

docs/api.yml

@@ -147,6 +147,7 @@ sidebar:
       - "api-reference/expressions/num/ceil"
       - "api-reference/expressions/num/floor"
       - "api-reference/expressions/num/round"
+      - "api-reference/expressions/num/to_string"
 
   - slug: "api-reference/expressions/str"
     title: "String Expressions"

docs/examples/api-reference/aggregations/lastk.py

@@ -41,6 +41,7 @@ def lastk_pipeline(cls, ds: Dataset):
                     of="amount",
                     limit=10,
                     dedup=False,
+                    dropnull=False,
                     window=Continuous("1d"),
                 ),
                 # docsnip-highlight end
@@ -142,6 +143,7 @@ def bad_pipeline(cls, ds: Dataset):
                         of="amount",
                         limit=10,
                         dedup=False,
+                        dropnull=False,
                         window=Continuous("1d"),
                     ),
                     # docsnip-highlight end

docs/examples/api-reference/expressions/num.py

@@ -123,3 +123,26 @@ def test_round():
 
     with pytest.raises(Exception):
         expr = col("x").round(1.1)
+
+
+def test_to_string():
+    # docsnip to_string
+    from fennel.expr import col
+
+    # docsnip-highlight next-line
+    expr = col("x").num.to_string()
+
+    # type is str or optional str
+    assert expr.typeof(schema={"x": int}) == str
+    assert expr.typeof(schema={"x": Optional[int]}) == Optional[str]
+    assert expr.typeof(schema={"x": float}) == str
+    assert expr.typeof(schema={"x": Optional[float]}) == Optional[str]
+
+    # can be evaluated with a dataframe
+    df = pd.DataFrame({"x": pd.Series([1.1, -2.3, None])})
+    assert expr.eval(df, schema={"x": Optional[float]}).tolist() == [
+        "1.1",
+        "-2.3",
+        pd.NA,
+    ]
+    # /docsnip

docs/pages/api-reference/aggregations/average.md

@@ -22,28 +22,30 @@ The name of the field in the output dataset that should store the result of this
 aggregation. This field is expected to be of type `float`.
 </Expandable>
 
-<Expandable title="default" type="float">
+<Expandable title="default" type="Optional[float]">
 Average over an empty set of rows isn't well defined - Fennel returns `default`
-in such cases.
+in such cases. If the default is not set or is None, Fennel returns None and 
+in that case, the expected type of `into_field` must be `Optional[float]`.
 </Expandable>
 
 <pre snippet="api-reference/aggregations/avg#basic" status="success" 
     message="Average in rolling window of 1 day & 1 week">
 </pre>
 
 #### Returns
-<Expandable type="float">
+<Expandable type="Union[float, Optional[float]]">
 Stores the result of the aggregation in the appropriate field of the output 
 dataset. If there are no rows in the aggregation window, `default` is used.
 </Expandable>
 
 
 #### Errors
-<Expandable title="Average on non int/float types">
-The input column denoted by `of` must either be of `int` or `float` types. 
+<Expandable title="Average on non int/float/decimal types">
+The input column denoted by `of` must either be of `int` or `float` or 
+`decimal` types.
 
-Note that unlike SQL, even aggregations over `Optional[int]` or `Optional[float]` 
-aren't allowed.
+Note that like SQL, aggregations over `Optional[int]` or `Optional[float]` 
+are allowed.
 </Expandable>
 
 <Expandable title="Output and/or default aren't float">
@@ -52,7 +54,7 @@ The type of the field denoted by `into_field` in the output dataset and that of
 </Expandable>
 
 <pre snippet="api-reference/aggregations/avg#incorrect_type" status="error" 
-    message="Can not take average over string, only int or float">
+    message="Can not take average over string, only int or float or decimal">
 </pre>
 <pre snippet="api-reference/aggregations/avg#non_matching_types" status="error" 
     message="Invalid type: ret is int but should be float">

docs/pages/api-reference/aggregations/exponential-decay-sum.md

@@ -9,7 +9,7 @@ Aggregation to compute a rolling exponential decay for each group within a windo
 #### Parameters
 <Expandable title="of" type="str">
 Name of the field in the input dataset over which the decayed sum should be computed. 
-This field can only either be `int` or `float.
+This field can only either be `int` or `float` or `decimal`.
 </Expandable>
 
 <Expandable title="window" type="Window">
@@ -46,8 +46,8 @@ are no rows to count, by default, it returns 0.0
 The input column denoted by `of` must either be of `int` or `float` types. 
 The output field denoted by `into_field` must always be of type `float`.
 
-Note that unlike SQL, even aggregations over `Optional[int]` or `Optional[float]` 
-aren't allowed.
+Note that like SQL, aggregations over `Optional[int]` or `Optional[float]` 
+are allowed.
 </Expandable>
 
 <pre snippet="api-reference/aggregations/exp-decay-sum#incorrect_type_exp_decay" status="error" 

docs/pages/api-reference/aggregations/lastk.md

@@ -35,6 +35,11 @@ If set to True, only distinct values are stored else values stored in the last
 can have duplicates too.
 </Expandable>
 
+<Expandable title="dropnull" type="bool">
+If set to True, None values are dropped from the result. It expects `of` field
+to be of type `Optional[T]` and `into_field` gets the type `List[T]`.
+</Expandable>
+
 <pre snippet="api-reference/aggregations/lastk#basic" status="success" 
     message="LastK in window of 1 day">
 </pre>

docs/pages/api-reference/aggregations/max.md

@@ -24,10 +24,11 @@ aggregation. This field is expected to be of type `int`, `float`, `date` or
 `of`.
 </Expandable>
 
-<Expandable title="default" type="Union[int, float]">
+<Expandable title="default" type="Optional[Union[int, float, Decimal, datetime, date]]">
 Max over an empty set of rows isn't well defined - Fennel returns `default`
 in such cases. The type of `default` must be same as that of `of` in the input
-dataset.
+dataset. If the default is not set or is None, Fennel returns None and in that case, 
+the expected type of `into_field` must be `Optional[T]`.
 </Expandable>
 
 <pre snippet="api-reference/aggregations/max#basic" status="success" 
@@ -43,11 +44,11 @@ dataset. If there are no rows in the aggregation window, `default` is used.
 
 #### Errors
 <Expandable title="Max on other types">
-The input column denoted by `of` must be of `int`, `float`, `date` or `datetime`
-types. 
+The input column denoted by `of` must be of `int`, `float`, `decimal`, 
+`date` or `datetime` types. 
 
-Note that unlike SQL, even aggregations over `Optional[int]` or `Optional[float]` 
-aren't allowed.
+Note that like SQL, aggregations over `Optional[int]` or `Optional[float]` 
+are allowed.
 </Expandable>
 
 <Expandable title="Types of input, output & default don't match">

docs/pages/api-reference/aggregations/min.md

@@ -24,10 +24,11 @@ aggregation. This field is expected to be of type `int`, `float`, `date` or
 `of`.
 </Expandable>
 
-<Expandable title="default" type="Union[int, float]">
+<Expandable title="default" type="Optional[Union[int, float, Decimal, datetime, date]]">
 Min over an empty set of rows isn't well defined - Fennel returns `default`
 in such cases. The type of `default` must be same as that of `of` in the input
-dataset.
+dataset. If the default is not set or is None, Fennel returns None and in that case, 
+the expected type of `into_field` must be `Optional[T]`.
 </Expandable>
 
 <pre snippet="api-reference/aggregations/min#basic" status="success" 
@@ -43,11 +44,11 @@ dataset. If there are no rows in the aggregation window, `default` is used.
 
 #### Errors
 <Expandable title="Min on other types">
-The input column denoted by `of` must be of `int`, `float`, `date` or `datetime`
-types. 
+The input column denoted by `of` must be of `int`, `float`, `decimal`, 
+`date` or `datetime` types. 
 
-Note that unlike SQL, even aggregations over `Optional[int]` or `Optional[float]` 
-aren't allowed.
+Note that like SQL, aggregations over `Optional[int]` or `Optional[float]` 
+are allowed.
 </Expandable>
 
 <Expandable title="Types of input, output & default don't match">

docs/pages/api-reference/aggregations/quantile.md

@@ -58,10 +58,11 @@ dataset. If there are no rows in the aggregation window, `default` is used.
 
 #### Errors
 <Expandable title="Quantile on non int/float types">
-The input column denoted by `of` must either be of `int` or `float` types. 
+The input column denoted by `of` must either be of `int` or `float` or 
+`decimal` types.
 
-Note that unlike SQL, even aggregations over `Optional[int]` or `Optional[float]` 
-aren't allowed.
+Note that like SQL, aggregations over `Optional[int]` or `Optional[float]` 
+are allowed.
 </Expandable>
 
 <Expandable title="Types of output & default don't match">
@@ -81,7 +82,7 @@ right expectations and be compatible with future addition of exact quantiles.
 </Expandable>
 
 <pre snippet="api-reference/aggregations/quantile#incorrect_type" status="error" 
-    message="Can not take quantile over string, only int or float">
+    message="Can not take quantile over string, only int or float or decimal">
 </pre>
 <pre snippet="api-reference/aggregations/quantile#invalid_default" status="error" 
     message="Default is not specified, so the output field should be Optional[float]">

docs/pages/api-reference/aggregations/stddev.md

@@ -22,28 +22,31 @@ The name of the field in the output dataset that should store the result of this
 aggregation. This field is expected to be of type `float`.
 </Expandable>
 
-<Expandable title="default" type="float">
+<Expandable title="default" type="Optional[float]">
 Standard deviation over an empty set of rows isn't well defined - Fennel 
-returns `default` in such cases.
+returns `default` in such cases. If the default is not set or is None, 
+Fennel returns None and in that case, the expected type of `into_field` 
+must be `Optional[float]`.
 </Expandable>
 
 <pre snippet="api-reference/aggregations/stddev#basic" status="success" 
     message="Standard deviation in window of 1 day & week">
 </pre>
 
 #### Returns
-<Expandable type="float">
+<Expandable type="Union[float, Optional[float]]">
 Stores the result of the aggregation in the appropriate field of the output 
 dataset. If there are no rows in the aggregation window, `default` is used.
 </Expandable>
 
 
 #### Errors
 <Expandable title="Stddev on non int/float types">
-The input column denoted by `of` must either be of `int` or `float` types. 
+The input column denoted by `of` must either be of `int` or `float` or 
+`decimal` types.
 
-Note that unlike SQL, even aggregations over `Optional[int]` or `Optional[float]` 
-aren't allowed.
+Note that like SQL, aggregations over `Optional[int]` or `Optional[float]` 
+are allowed.
 </Expandable>
 
 <Expandable title="Output and/or default aren't float">
@@ -52,7 +55,7 @@ The type of the field denoted by `into_field` in the output dataset and that of
 </Expandable>
 
 <pre snippet="api-reference/aggregations/stddev#incorrect_type" status="error" 
-    message="Can not take stddev over string, only int or float">
+    message="Can not take stddev over string, only int or float or decimal">
 </pre>
 
 <pre snippet="api-reference/aggregations/stddev#non_matching_types" status="error" 

docs/pages/api-reference/aggregations/sum.md

@@ -28,18 +28,19 @@ type of the field in the input dataset corresponding to `of`.
 </pre>
 
 #### Returns
-<Expandable type="Union[int, float]">
+<Expandable type="Union[int, float, Decimal]">
 Accumulates the count in the appropriate field of the output dataset. If there 
 are no rows to count, by default, it returns 0 (or 0.0 if `of` is float).
 </Expandable>
 
 
 #### Errors
 <Expandable title="Sum on non int/float types">
-The input column denoted by `of` must either be of `int` or `float` types. 
+The input column denoted by `of` must either be of `int` or `float` 
+or `decimal` types.
 
-Note that unlike SQL, even aggregations over `Optional[int]` or `Optional[float]` 
-aren't allowed.
+Note that like SQL, aggregations over `Optional[int]` or `Optional[float]` 
+are allowed.
 </Expandable>
 
 <pre snippet="api-reference/aggregations/sum#incorrect_type" status="error" 

docs/pages/api-reference/decorators/source.md

@@ -120,7 +120,7 @@ As of right now, there are three kinds of values of preproc:
   a constant value.
 
 :::info
-Fennel supports preproc ref(str) values of type A[B][C] only for the JSON and Protobuf formats, and 
+Fennel supports preproc ref(str) values of type A[B][C] only for the JSON, Avro and Protobuf formats, and 
 A, B should be struct types. If you have data in other format or require indirection
 for other parent types apart from struct, please reach out to Fennel support.
 :::

docs/pages/api-reference/expressions/num/to_string.md

@@ -0,0 +1,26 @@
+---
+title: To String
+order: 0
+status: published
+---
+
+### To String
+
+Function in `num` namespace to convert a number to a string.
+
+#### Returns
+<Expandable type="Expr">
+Returns an expression object denoting the string value of the input data. The
+data type of the resulting expression is `str` (or `Optional[str]` if the input
+is an optional number).
+</Expandable>
+
+<pre snippet="api-reference/expressions/num#to_string"
+status="success" message="Converting a number to a string using Fennel expressions">
+</pre>
+
+#### Errors
+<Expandable title="Invoking on a non-numeric type">
+Error during `typeof` or `eval` if the input expression is not of type int, 
+float, optional int or optional float.
+</Expandable>

examples/plaid/commit.py

@@ -0,0 +1,12 @@
+from features import PlaidReport
+
+TOKEN = "9nWHJMVT6YyaImFotfdCzhzHD30WE3ywh3CQIF0_zUU"
+URL = "https://hoang.aws.fennel.ai/"
+BRANCH = "plaid_test"
+
+if __name__ == "__main__":
+    from fennel.client import Client
+
+    client = Client(url=URL, token=TOKEN)
+    client.checkout(BRANCH, init=True)
+    client.commit("add plaid report features", featuresets=[PlaidReport])