diff --git a/hogvm/__tests__/__snapshots__/catch.hoge b/hogvm/__tests__/__snapshots__/catch.hoge index e3588d7a1cdf9..9bde1eae9b68c 100644 --- a/hogvm/__tests__/__snapshots__/catch.hoge +++ b/hogvm/__tests__/__snapshots__/catch.hoge @@ -1,11 +1,11 @@ -["_H", 1, 41, "FishError", 1, 9, 32, "FishError", 36, 0, 31, 2, "HogError", 3, 38, 41, "FoodError", 1, 9, 32, -"FoodError", 36, 0, 31, 2, "HogError", 3, 38, 50, 10, 32, "You forgot to feed your fish", 2, "FishError", 1, 49, 51, 39, -55, 36, 0, 32, "type", 45, 32, "FoodError", 36, 1, 11, 40, 16, 32, "Problem with your food: ", 36, 0, 32, "message", 45, -2, "concat", 2, 2, "print", 1, 35, 39, 25, 32, "FishError", 36, 1, 11, 40, 16, 32, "Problem with your fish: ", 36, 0, -32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 50, 10, 32, "Your fish are hungry", 2, -"FoodError", 1, 49, 51, 39, 55, 36, 0, 32, "type", 45, 32, "FoodError", 36, 1, 11, 40, 16, 32, -"Problem with your food: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 25, 32, "FishError", 36, 1, -11, 40, 16, 32, "Problem with your fish: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, -35, 35, 50, 11, 32, "Your fish are hungry", 31, 2, "NotImplementedError", 2, 49, 51, 39, 45, 36, 0, 32, "type", 45, 32, -"FoodError", 36, 1, 11, 40, 16, 32, "Problem with your food: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, -35, 39, 15, 32, "Unknown problem: ", 36, 0, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35] +["_H", 1, 52, "FishError", 1, 0, 8, 32, "FishError", 36, 0, 2, "HogError", 2, 38, 53, 0, 52, "FoodError", 1, 0, 8, 32, +"FoodError", 36, 0, 2, "HogError", 2, 38, 53, 0, 50, 11, 32, "You forgot to feed your fish", 36, 0, 54, 1, 49, 51, 39, +55, 36, 2, 32, "type", 45, 32, "FoodError", 36, 3, 11, 40, 16, 32, "Problem with your food: ", 36, 2, 32, "message", 45, +2, "concat", 2, 2, "print", 1, 35, 39, 25, 32, "FishError", 36, 3, 11, 40, 16, 32, "Problem with your fish: ", 36, 2, +32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 50, 11, 32, "Your fish are hungry", 36, 1, +54, 1, 49, 51, 39, 55, 36, 2, 32, "type", 45, 32, "FoodError", 36, 3, 11, 40, 16, 32, "Problem with your food: ", 36, 2, +32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 25, 32, "FishError", 36, 3, 11, 40, 16, 32, +"Problem with your fish: ", 36, 2, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 50, 10, +32, "Your fish are hungry", 2, "NotImplementedError", 1, 49, 51, 39, 45, 36, 2, 32, "type", 45, 32, "FoodError", 36, 3, +11, 40, 16, 32, "Problem with your food: ", 36, 2, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 15, 32, +"Unknown problem: ", 36, 2, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/catch2.hoge b/hogvm/__tests__/__snapshots__/catch2.hoge index d99e3cd75ae3a..ce7f940a43a20 100644 --- a/hogvm/__tests__/__snapshots__/catch2.hoge +++ b/hogvm/__tests__/__snapshots__/catch2.hoge @@ -1,15 +1,15 @@ -["_H", 1, 50, 50, 50, 13, 32, "FishError", 32, "You forgot to feed your fish", 31, 2, "HogError", 3, 49, 51, 39, 32, 36, -0, 32, "type", 45, 32, "FoodError", 36, 1, 11, 40, 16, 32, "Problem with your food: ", 36, 0, 32, "message", 45, 2, +["_H", 1, 50, 49, 50, 12, 32, "FishError", 32, "You forgot to feed your fish", 2, "HogError", 2, 49, 51, 39, 32, 36, 0, +32, "type", 45, 32, "FoodError", 36, 1, 11, 40, 16, 32, "Problem with your food: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 51, 39, 48, 36, 0, 32, "type", 45, 32, "FishError", 36, 1, 11, 40, 16, 32, "FishError: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 18, 32, "Error: ", 36, 0, -32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 50, 50, 50, 13, 32, "FunkyError", 32, -"You forgot to feed your fish", 31, 2, "HogError", 3, 49, 51, 39, 32, 36, 0, 32, "type", 45, 32, "FoodError", 36, 1, 11, -40, 16, 32, "Problem with your food: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, -35, 51, 39, 55, 36, 0, 32, "type", 45, 32, "FishError", 36, 1, 11, 40, 16, 32, "FishError: ", 36, 0, 32, "message", 45, -2, "concat", 2, 2, "print", 1, 35, 39, 25, 32, "Error of type ", 36, 0, 32, "name", 45, 32, ": ", 36, 0, 32, "message", -45, 2, "concat", 4, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 50, 50, 50, 13, 32, "FishError", 32, -"You forgot to feed your fish", 31, 2, "HogError", 3, 49, 51, 39, 32, 36, 0, 32, "type", 45, 32, "FoodError", 36, 1, 11, -40, 16, 32, "Problem with your food: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, -35, 51, 39, 55, 36, 0, 32, "type", 45, 32, "Error of type ", 36, 0, 32, "name", 45, 32, ": ", 36, 0, 32, "message", 45, -2, "concat", 4, 2, "print", 1, 35, 39, 25, 32, "FishError", 36, 1, 11, 40, 16, 32, "FishError: ", 36, 0, 32, "message", -45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35] +32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 50, 49, 50, 12, 32, "FunkyError", 32, +"You forgot to feed your fish", 2, "HogError", 2, 49, 51, 39, 32, 36, 0, 32, "type", 45, 32, "FoodError", 36, 1, 11, 40, +16, 32, "Problem with your food: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, +51, 39, 55, 36, 0, 32, "type", 45, 32, "FishError", 36, 1, 11, 40, 16, 32, "FishError: ", 36, 0, 32, "message", 45, 2, +"concat", 2, 2, "print", 1, 35, 39, 25, 32, "Error of type ", 36, 0, 32, "name", 45, 32, ": ", 36, 0, 32, "message", 45, +2, "concat", 4, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 50, 49, 50, 12, 32, "FishError", 32, +"You forgot to feed your fish", 2, "HogError", 2, 49, 51, 39, 32, 36, 0, 32, "type", 45, 32, "FoodError", 36, 1, 11, 40, +16, 32, "Problem with your food: ", 36, 0, 32, "message", 45, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, +51, 39, 55, 36, 0, 32, "type", 45, 32, "Error of type ", 36, 0, 32, "name", 45, 32, ": ", 36, 0, 32, "message", 45, 2, +"concat", 4, 2, "print", 1, 35, 39, 25, 32, "FishError", 36, 1, 11, 40, 16, 32, "FishError: ", 36, 0, 32, "message", 45, +2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/exceptions.hoge b/hogvm/__tests__/__snapshots__/exceptions.hoge index a4a5b1d16a4b2..8d20794550565 100644 --- a/hogvm/__tests__/__snapshots__/exceptions.hoge +++ b/hogvm/__tests__/__snapshots__/exceptions.hoge @@ -2,13 +2,14 @@ 0, 32, " was the exception", 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 32, "------------------", 2, "print", 1, 35, 32, "start", 2, "print", 1, 35, 50, 10, 32, "try", 2, "print", 1, 35, 51, 39, 17, 36, 0, 32, "type", 45, 32, "No var for error, but no error", 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 32, "------------------", 2, "print", 1, -35, 50, 16, 32, "try again", 2, "print", 1, 35, 31, 31, 2, "Error", 2, 49, 51, 39, 22, 36, 0, 32, "type", 45, 36, 0, 32, +35, 50, 14, 32, "try again", 2, "print", 1, 35, 2, "Error", 0, 49, 51, 39, 22, 36, 0, 32, "type", 45, 36, 0, 32, " was the exception", 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 32, "------------------", 2, "print", 1, -35, 50, 16, 32, "try again", 2, "print", 1, 35, 31, 31, 2, "Error", 2, 49, 51, 39, 17, 36, 0, 32, "type", 45, 32, -"No var for error", 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 32, "------------------", 2, "print", 1, 35, 41, "third", -0, 15, 32, "Throwing in third", 2, "print", 1, 35, 32, "Threw in third", 31, 2, "Error", 2, 49, 31, 38, 41, "second", 0, -12, 32, "second", 2, "print", 1, 35, 2, "third", 0, 35, 31, 38, 41, "first", 0, 12, 32, "first", 2, "print", 1, 35, 2, -"second", 0, 35, 31, 38, 41, "base", 0, 42, 32, "base", 2, "print", 1, 35, 50, 8, 2, "first", 0, 35, 51, 39, 25, 36, 0, -32, "type", 45, 32, "Caught in base: ", 36, 0, 2, "concat", 2, 2, "print", 1, 35, 36, 0, 49, 39, 2, 35, 49, 35, 35, 31, -38, 50, 8, 2, "base", 0, 35, 51, 39, 22, 36, 0, 32, "type", 45, 32, "Caught in root: ", 36, 0, 2, "concat", 2, 2, -"print", 1, 35, 39, 2, 35, 49, 35, 35, 32, "The end", 2, "print", 1, 35] +35, 50, 14, 32, "try again", 2, "print", 1, 35, 2, "Error", 0, 49, 51, 39, 17, 36, 0, 32, "type", 45, 32, +"No var for error", 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 32, "------------------", 2, "print", 1, 35, 52, "third", +0, 0, 14, 32, "Throwing in third", 2, "print", 1, 35, 32, "Threw in third", 2, "Error", 1, 49, 31, 38, 53, 0, 52, +"second", 0, 1, 13, 32, "second", 2, "print", 1, 35, 55, 0, 54, 0, 35, 31, 38, 53, 1, true, 0, 52, "first", 0, 1, 13, +32, "first", 2, "print", 1, 35, 55, 0, 54, 0, 35, 31, 38, 53, 1, true, 1, 52, "base", 0, 1, 43, 32, "base", 2, "print", +1, 35, 50, 9, 55, 0, 54, 0, 35, 51, 39, 25, 36, 0, 32, "type", 45, 32, "Caught in base: ", 36, 0, 2, "concat", 2, 2, +"print", 1, 35, 36, 0, 49, 39, 2, 35, 49, 35, 35, 31, 38, 53, 1, true, 2, 50, 9, 36, 3, 54, 0, 35, 51, 39, 22, 36, 4, +32, "type", 45, 32, "Caught in root: ", 36, 4, 2, "concat", 2, 2, "print", 1, 35, 39, 2, 35, 49, 35, 35, 32, "The end", +2, "print", 1, 35, 35, 57, 57, 57] diff --git a/hogvm/__tests__/__snapshots__/functionVars.hoge b/hogvm/__tests__/__snapshots__/functionVars.hoge new file mode 100644 index 0000000000000..78fc29dae44b8 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/functionVars.hoge @@ -0,0 +1,10 @@ +["_H", 1, 52, "execFunction", 0, 0, 8, 32, "execFunction", 2, "print", 1, 35, 31, 38, 53, 0, 52, "execFunctionNested", +0, 0, 29, 52, "execFunction", 0, 0, 8, 32, "execFunctionNew", 2, "print", 1, 35, 31, 38, 53, 0, 32, +"execFunctionNested", 2, "print", 1, 35, 36, 0, 54, 0, 35, 31, 38, 35, 53, 0, 36, 0, 54, 0, 35, 36, 1, 54, 0, 35, 36, 0, +54, 0, 35, 32, "--------", 2, "print", 1, 35, 52, "secondExecFunction", 0, 0, 8, 32, "secondExecFunction", 2, "print", +1, 35, 31, 38, 53, 0, 52, "secondExecFunctionNested", 0, 1, 13, 32, "secondExecFunctionNested", 2, "print", 1, 35, 55, +0, 54, 0, 35, 31, 38, 53, 1, true, 2, 36, 2, 54, 0, 35, 36, 3, 54, 0, 35, 36, 2, 54, 0, 35, 32, "--------", 2, "print", +1, 35, 52, "lambda", 0, 0, 5, 32, "base64Decode", 1, 1, 38, 53, 0, 32, "base64Encode", 1, 1, 32, +"http://www.google.com", 36, 5, 54, 1, 2, "print", 1, 35, 32, "http://www.google.com", 36, 5, 54, 1, 36, 4, 54, 0, 54, +1, 2, "print", 1, 35, 32, "http://www.google.com", 36, 5, 54, 1, 36, 4, 54, 0, 54, 1, 2, "print", 1, 35, 35, 35, 35, 57, +35, 35] diff --git a/hogvm/__tests__/__snapshots__/functionVars.stdout b/hogvm/__tests__/__snapshots__/functionVars.stdout new file mode 100644 index 0000000000000..f8f446e9ae530 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/functionVars.stdout @@ -0,0 +1,13 @@ +execFunction +execFunctionNested +execFunctionNew +execFunction +-------- +secondExecFunction +secondExecFunctionNested +secondExecFunction +secondExecFunction +-------- +aHR0cDovL3d3dy5nb29nbGUuY29t +http://www.google.com +http://www.google.com diff --git a/hogvm/__tests__/__snapshots__/functions.hoge b/hogvm/__tests__/__snapshots__/functions.hoge index 0351cb7ce179d..ae2b3ec50febf 100644 --- a/hogvm/__tests__/__snapshots__/functions.hoge +++ b/hogvm/__tests__/__snapshots__/functions.hoge @@ -1,14 +1,21 @@ -["_H", 1, 32, "-- test functions --", 2, "print", 1, 35, 41, "add", 2, 6, 36, 1, 36, 0, 6, 38, 41, "add2", 2, 9, 36, 1, -36, 0, 6, 36, 2, 38, 35, 41, "mult", 2, 6, 36, 1, 36, 0, 8, 38, 41, "noArgs", 0, 12, 32, "basdfasdf", 33, 3, 33, 2, 6, -36, 1, 38, 35, 35, 41, "empty", 0, 2, 31, 38, 41, "empty2", 0, 2, 31, 38, 41, "empty3", 0, 2, 31, 38, 41, "noReturn", 0, -14, 33, 1, 33, 2, 36, 1, 36, 0, 6, 31, 38, 35, 35, 35, 41, "emptyReturn", 0, 2, 31, 38, 41, -"emptyReturnBeforeOtherStuff", 0, 10, 31, 38, 33, 2, 33, 2, 6, 35, 31, 38, 41, "emptyReturnBeforeOtherStuffNoSemicolon", -0, 6, 33, 2, 33, 2, 6, 38, 41, "ifThenReturn", 0, 8, 30, 40, 2, 31, 38, 33, 4, 38, 33, 3, 33, 4, 2, "add", 2, 2, -"print", 1, 35, 33, 1, 33, 1, 2, "add", 2, 33, 100, 33, 3, 33, 4, 2, "add", 2, 6, 6, 2, "print", 1, 35, 2, "noArgs", 0, -47, 3, 35, 33, -1, 2, "print", 1, 35, 2, "empty", 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 2, "empty2", 0, 47, 3, 35, -33, -1, 2, "print", 1, 35, 2, "empty3", 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 2, "noReturn", 0, 47, 3, 35, 33, -1, 2, -"print", 1, 35, 2, "emptyReturn", 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 2, "emptyReturnBeforeOtherStuff", 0, 47, 3, -35, 33, -1, 2, "print", 1, 35, 2, "emptyReturnBeforeOtherStuffNoSemicolon", 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 2, -"ifThenReturn", 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 33, 2, 33, 1, 2, "add", 2, 33, 100, 33, 3, 33, 4, 2, "add", 2, -6, 6, 33, 2, 2, "mult", 2, 2, "print", 1, 35, 33, 2, 33, 1, 2, "add2", 2, 33, 100, 33, 3, 33, 4, 2, "add2", 2, 6, 6, 33, -10, 2, "mult", 2, 2, "print", 1, 35] +["_H", 1, 32, "-- test functions --", 2, "print", 1, 35, 52, "add", 2, 0, 6, 36, 1, 36, 0, 6, 38, 53, 0, 36, 0, 2, +"print", 1, 35, 52, "add2", 2, 0, 9, 36, 1, 36, 0, 6, 36, 2, 38, 35, 53, 0, 36, 1, 2, "print", 1, 35, 52, "mult", 2, 0, +6, 36, 1, 36, 0, 8, 38, 53, 0, 36, 2, 2, "print", 1, 35, 52, "noArgs", 0, 0, 12, 32, "basdfasdf", 33, 3, 33, 2, 6, 36, +1, 38, 35, 35, 53, 0, 36, 3, 2, "print", 1, 35, 52, "empty", 0, 0, 2, 31, 38, 53, 0, 52, "empty2", 0, 0, 2, 31, 38, 53, +0, 52, "empty3", 0, 0, 2, 31, 38, 53, 0, 52, "noReturn", 0, 0, 14, 33, 1, 33, 2, 36, 1, 36, 0, 6, 31, 38, 35, 35, 35, +53, 0, 52, "emptyReturn", 0, 0, 2, 31, 38, 53, 0, 52, "emptyReturnBeforeOtherStuff", 0, 0, 10, 31, 38, 33, 2, 33, 2, 6, +35, 31, 38, 53, 0, 52, "emptyReturnBeforeOtherStuffNoSemicolon", 0, 0, 6, 33, 2, 33, 2, 6, 38, 53, 0, 52, +"ifThenReturn", 0, 0, 8, 30, 40, 2, 31, 38, 33, 4, 38, 53, 0, 33, 3, 33, 4, 36, 0, 54, 2, 2, "print", 1, 35, 33, 1, 33, +1, 36, 0, 54, 2, 33, 100, 33, 3, 33, 4, 36, 0, 54, 2, 6, 6, 2, "print", 1, 35, 36, 3, 54, 0, 47, 3, 35, 33, -1, 2, +"print", 1, 35, 36, 4, 54, 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 36, 5, 54, 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, +36, 6, 54, 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 36, 7, 54, 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 36, 8, 54, 0, +47, 3, 35, 33, -1, 2, "print", 1, 35, 36, 9, 54, 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 36, 10, 54, 0, 47, 3, 35, 33, +-1, 2, "print", 1, 35, 36, 11, 54, 0, 47, 3, 35, 33, -1, 2, "print", 1, 35, 33, 2, 33, 1, 36, 0, 54, 2, 33, 100, 33, 3, +33, 4, 36, 0, 54, 2, 6, 6, 33, 2, 36, 2, 54, 2, 2, "print", 1, 35, 33, 2, 33, 1, 36, 1, 54, 2, 33, 100, 33, 3, 33, 4, +36, 1, 54, 2, 6, 6, 33, 10, 36, 2, 54, 2, 2, "print", 1, 35, 52, "printArgs", 7, 0, 20, 36, 0, 36, 1, 36, 2, 36, 3, 36, +4, 36, 5, 36, 6, 2, "print", 7, 35, 31, 38, 53, 0, 52, "lambda", 7, 0, 20, 36, 0, 36, 1, 36, 2, 36, 3, 36, 4, 36, 5, 36, +6, 2, "print", 7, 35, 31, 38, 53, 0, 33, 1, 33, 2, 33, 3, 33, 4, 33, 5, 33, 6, 33, 7, 36, 12, 54, 7, 35, 33, 1, 33, 2, +33, 3, 33, 4, 33, 5, 33, 6, 33, 7, 36, 13, 54, 7, 35, 33, 1, 33, 2, 33, 3, 33, 4, 33, 5, 33, 6, 36, 12, 54, 6, 35, 33, +1, 33, 2, 33, 3, 33, 4, 33, 5, 33, 6, 36, 13, 54, 6, 35, 33, 1, 33, 2, 33, 3, 33, 4, 33, 5, 36, 12, 54, 5, 35, 33, 1, +33, 2, 33, 3, 33, 4, 33, 5, 36, 13, 54, 5, 35, 36, 12, 54, 0, 35, 36, 13, 54, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, +35, 35, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/functions.stdout b/hogvm/__tests__/__snapshots__/functions.stdout index bd80a6f22ca08..03ed7307c9441 100644 --- a/hogvm/__tests__/__snapshots__/functions.stdout +++ b/hogvm/__tests__/__snapshots__/functions.stdout @@ -1,4 +1,8 @@ -- test functions -- +fn +fn +fn +fn 7 109 5 @@ -12,3 +16,11 @@ 4 220 1100 +1 2 3 4 5 6 7 +1 2 3 4 5 6 7 +1 2 3 4 5 6 null +1 2 3 4 5 6 null +1 2 3 4 5 null null +1 2 3 4 5 null null +null null null null null null null +null null null null null null null diff --git a/hogvm/__tests__/__snapshots__/lambdas.hoge b/hogvm/__tests__/__snapshots__/lambdas.hoge new file mode 100644 index 0000000000000..6fd2381e9e968 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/lambdas.hoge @@ -0,0 +1,7 @@ +["_H", 1, 52, "lambda", 1, 0, 6, 33, 2, 36, 0, 8, 38, 53, 0, 36, 0, 2, "print", 1, 35, 33, 2, 36, 0, 54, 1, 2, "print", +1, 35, 33, 8, 36, 0, 54, 1, 2, "print", 1, 35, 32, "--------", 2, "print", 1, 35, 52, "lambda", 1, 0, 6, 33, 2, 36, 0, +8, 38, 53, 0, 36, 1, 43, 1, 33, 2, 36, 1, 54, 1, 2, "print", 1, 35, 33, 2, 36, 2, 33, 1, 45, 54, 1, 2, "print", 1, 35, +33, 2, 52, "lambda", 1, 0, 6, 33, 2, 36, 0, 8, 38, 53, 0, 54, 1, 2, "print", 1, 35, 32, "--------", 2, "print", 1, 35, +52, "lambda", 1, 0, 20, 36, 0, 2, "print", 1, 35, 32, "moo", 2, "print", 1, 35, 32, "cow", 2, "print", 1, 35, 31, 38, +53, 0, 33, 2, 36, 3, 54, 1, 35, 32, "--------", 2, "print", 1, 35, 52, "lambda", 0, 0, 14, 32, "moo", 2, "print", 1, 35, +32, "cow", 2, "print", 1, 35, 31, 38, 53, 0, 36, 4, 54, 0, 35, 35, 35, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/lambdas.stdout b/hogvm/__tests__/__snapshots__/lambdas.stdout new file mode 100644 index 0000000000000..965e1733e2ab6 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/lambdas.stdout @@ -0,0 +1,14 @@ +fn +4 +16 +-------- +4 +4 +4 +-------- +2 +moo +cow +-------- +moo +cow diff --git a/hogvm/__tests__/__snapshots__/mandelbrot.hoge b/hogvm/__tests__/__snapshots__/mandelbrot.hoge index 51895c2281458..d919b81c34dcd 100644 --- a/hogvm/__tests__/__snapshots__/mandelbrot.hoge +++ b/hogvm/__tests__/__snapshots__/mandelbrot.hoge @@ -1,8 +1,8 @@ -["_H", 1, 41, "mandelbrot", 3, 93, 34, 0.0, 34, 0.0, 33, 0, 33, 4, 36, 4, 36, 4, 8, 36, 3, 36, 3, 8, 6, 16, 36, 2, 36, -5, 15, 3, 2, 40, 44, 36, 0, 36, 4, 36, 4, 8, 36, 3, 36, 3, 8, 7, 6, 36, 1, 36, 4, 36, 3, 33, 2, 8, 8, 6, 36, 6, 37, 3, -36, 7, 37, 4, 33, 1, 36, 5, 6, 37, 5, 35, 35, 39, -67, 36, 2, 36, 5, 11, 40, 5, 32, " ", 38, 39, 3, 32, "#", 38, 31, 38, -35, 35, 35, 41, "main", 0, 119, 33, 80, 33, 24, 34, -2.0, 34, 1.0, 34, -1.0, 34, 1.0, 33, 30, 33, 0, 36, 1, 36, 7, 15, -40, 86, 32, "", 33, 0, 36, 0, 36, 9, 15, 40, 58, 36, 2, 36, 2, 36, 3, 7, 36, 0, 36, 9, 9, 8, 6, 36, 4, 36, 4, 36, 5, 7, -36, 1, 36, 7, 9, 8, 6, 36, 10, 36, 11, 36, 6, 2, "mandelbrot", 3, 36, 8, 36, 12, 2, "concat", 2, 37, 8, 33, 1, 36, 9, 6, -37, 9, 35, 35, 35, 39, -65, 36, 8, 2, "print", 1, 35, 33, 1, 36, 7, 6, 37, 7, 35, 35, 39, -93, 31, 38, 35, 35, 35, 35, -35, 35, 35, 35, 2, "main", 0, 35] +["_H", 1, 52, "mandelbrot", 3, 0, 93, 34, 0.0, 34, 0.0, 33, 0, 33, 4, 36, 4, 36, 4, 8, 36, 3, 36, 3, 8, 6, 16, 36, 2, +36, 5, 15, 3, 2, 40, 44, 36, 0, 36, 4, 36, 4, 8, 36, 3, 36, 3, 8, 7, 6, 36, 1, 36, 4, 36, 3, 33, 2, 8, 8, 6, 36, 6, 37, +3, 36, 7, 37, 4, 33, 1, 36, 5, 6, 37, 5, 35, 35, 39, -67, 36, 2, 36, 5, 11, 40, 5, 32, " ", 38, 39, 3, 32, "#", 38, 31, +38, 35, 35, 35, 53, 0, 52, "main", 0, 1, 120, 33, 80, 33, 24, 34, -2.0, 34, 1.0, 34, -1.0, 34, 1.0, 33, 30, 33, 0, 36, +1, 36, 7, 15, 40, 87, 32, "", 33, 0, 36, 0, 36, 9, 15, 40, 59, 36, 2, 36, 2, 36, 3, 7, 36, 0, 36, 9, 9, 8, 6, 36, 4, 36, +4, 36, 5, 7, 36, 1, 36, 7, 9, 8, 6, 36, 10, 36, 11, 36, 6, 55, 0, 54, 3, 36, 8, 36, 12, 2, "concat", 2, 37, 8, 33, 1, +36, 9, 6, 37, 9, 35, 35, 35, 39, -66, 36, 8, 2, "print", 1, 35, 33, 1, 36, 7, 6, 37, 7, 35, 35, 39, -94, 31, 38, 35, 35, +35, 35, 35, 35, 35, 35, 53, 1, true, 0, 36, 1, 54, 0, 35, 35, 57] diff --git a/hogvm/__tests__/__snapshots__/operations.hoge b/hogvm/__tests__/__snapshots__/operations.hoge index 3abc0604f4040..f11d3fa9ea5b5 100644 --- a/hogvm/__tests__/__snapshots__/operations.hoge +++ b/hogvm/__tests__/__snapshots__/operations.hoge @@ -1,23 +1,23 @@ -["_H", 1, 41, "test", 1, 11, 36, 0, 2, "jsonStringify", 1, 2, "print", 1, 35, 31, 38, 32, -"-- test the most common expressions --", 2, "print", 1, 35, 33, 2, 33, 1, 6, 2, "test", 1, 35, 33, 2, 33, 1, 7, 2, -"test", 1, 35, 33, 2, 33, 3, 8, 2, "test", 1, 35, 33, 2, 33, 3, 9, 2, "test", 1, 35, 33, 2, 33, 3, 10, 2, "test", 1, 35, -33, 1, 33, 2, 3, 2, 2, "test", 1, 35, 33, 1, 33, 0, 4, 2, 2, "test", 1, 35, 33, 1, 33, 0, 3, 2, 2, "test", 1, 35, 33, 1, -33, 0, 33, 1, 3, 2, 33, 2, 4, 3, 2, "test", 1, 35, 33, 1, 33, 0, 33, 1, 3, 3, 2, "test", 1, 35, 33, 1, 33, 2, 4, 2, 33, -1, 33, 2, 4, 2, 3, 2, 2, "test", 1, 35, 29, 2, "test", 1, 35, 29, 5, 2, "test", 1, 35, 30, 2, "test", 1, 35, 31, 2, -"test", 1, 35, 34, 3.14, 2, "test", 1, 35, 33, 2, 33, 1, 11, 2, "test", 1, 35, 33, 2, 33, 1, 11, 2, "test", 1, 35, 33, -2, 33, 1, 12, 2, "test", 1, 35, 33, 2, 33, 1, 15, 2, "test", 1, 35, 33, 2, 33, 1, 16, 2, "test", 1, 35, 33, 2, 33, 1, -13, 2, "test", 1, 35, 33, 2, 33, 1, 14, 2, "test", 1, 35, 32, "b", 32, "a", 17, 2, "test", 1, 35, 32, "%a%", 32, "baa", -17, 2, "test", 1, 35, 32, "%x%", 32, "baa", 17, 2, "test", 1, 35, 32, "%A%", 32, "baa", 18, 2, "test", 1, 35, 32, "%C%", -32, "baa", 18, 2, "test", 1, 35, 32, "b", 32, "a", 18, 2, "test", 1, 35, 32, "b", 32, "a", 19, 2, "test", 1, 35, 32, -"b", 32, "a", 20, 2, "test", 1, 35, 32, "car", 32, "a", 21, 2, "test", 1, 35, 32, "foo", 32, "a", 21, 2, "test", 1, 35, -32, "car", 32, "a", 22, 2, "test", 1, 35, 32, "arg", 32, "another", 2, "concat", 2, 2, "test", 1, 35, 33, 1, 31, 2, -"concat", 2, 2, "test", 1, 35, 29, 30, 2, "concat", 2, 2, "test", 1, 35, 32, "test", 32, "e.*", 2, "match", 2, 2, -"test", 1, 35, 32, "test", 32, "^e.*", 2, "match", 2, 2, "test", 1, 35, 32, "test", 32, "x.*", 2, "match", 2, 2, "test", -1, 35, 32, "e.*", 32, "test", 23, 2, "test", 1, 35, 32, "e.*", 32, "test", 24, 2, "test", 1, 35, 32, "^e.*", 32, "test", -23, 2, "test", 1, 35, 32, "^e.*", 32, "test", 24, 2, "test", 1, 35, 32, "x.*", 32, "test", 23, 2, "test", 1, 35, 32, -"x.*", 32, "test", 24, 2, "test", 1, 35, 32, "EST", 32, "test", 25, 2, "test", 1, 35, 32, "EST", 32, "test", 25, 2, -"test", 1, 35, 32, "EST", 32, "test", 26, 2, "test", 1, 35, 33, 1, 2, "toString", 1, 2, "test", 1, 35, 34, 1.5, 2, -"toString", 1, 2, "test", 1, 35, 29, 2, "toString", 1, 2, "test", 1, 35, 31, 2, "toString", 1, 2, "test", 1, 35, 32, -"string", 2, "toString", 1, 2, "test", 1, 35, 32, "1", 2, "toInt", 1, 2, "test", 1, 35, 32, "bla", 2, "toInt", 1, 2, -"test", 1, 35, 32, "1.2", 2, "toFloat", 1, 2, "test", 1, 35, 32, "bla", 2, "toFloat", 1, 2, "test", 1, 35, 32, "asd", 2, -"toUUID", 1, 2, "test", 1, 35, 31, 33, 1, 11, 2, "test", 1, 35, 31, 33, 1, 12, 2, "test", 1, 35] +["_H", 1, 52, "test", 1, 0, 11, 36, 0, 2, "jsonStringify", 1, 2, "print", 1, 35, 31, 38, 53, 0, 32, +"-- test the most common expressions --", 2, "print", 1, 35, 33, 2, 33, 1, 6, 36, 0, 54, 1, 35, 33, 2, 33, 1, 7, 36, 0, +54, 1, 35, 33, 2, 33, 3, 8, 36, 0, 54, 1, 35, 33, 2, 33, 3, 9, 36, 0, 54, 1, 35, 33, 2, 33, 3, 10, 36, 0, 54, 1, 35, 33, +1, 33, 2, 3, 2, 36, 0, 54, 1, 35, 33, 1, 33, 0, 4, 2, 36, 0, 54, 1, 35, 33, 1, 33, 0, 3, 2, 36, 0, 54, 1, 35, 33, 1, 33, +0, 33, 1, 3, 2, 33, 2, 4, 3, 36, 0, 54, 1, 35, 33, 1, 33, 0, 33, 1, 3, 3, 36, 0, 54, 1, 35, 33, 1, 33, 2, 4, 2, 33, 1, +33, 2, 4, 2, 3, 2, 36, 0, 54, 1, 35, 29, 36, 0, 54, 1, 35, 29, 5, 36, 0, 54, 1, 35, 30, 36, 0, 54, 1, 35, 31, 36, 0, 54, +1, 35, 34, 3.14, 36, 0, 54, 1, 35, 33, 2, 33, 1, 11, 36, 0, 54, 1, 35, 33, 2, 33, 1, 11, 36, 0, 54, 1, 35, 33, 2, 33, 1, +12, 36, 0, 54, 1, 35, 33, 2, 33, 1, 15, 36, 0, 54, 1, 35, 33, 2, 33, 1, 16, 36, 0, 54, 1, 35, 33, 2, 33, 1, 13, 36, 0, +54, 1, 35, 33, 2, 33, 1, 14, 36, 0, 54, 1, 35, 32, "b", 32, "a", 17, 36, 0, 54, 1, 35, 32, "%a%", 32, "baa", 17, 36, 0, +54, 1, 35, 32, "%x%", 32, "baa", 17, 36, 0, 54, 1, 35, 32, "%A%", 32, "baa", 18, 36, 0, 54, 1, 35, 32, "%C%", 32, "baa", +18, 36, 0, 54, 1, 35, 32, "b", 32, "a", 18, 36, 0, 54, 1, 35, 32, "b", 32, "a", 19, 36, 0, 54, 1, 35, 32, "b", 32, "a", +20, 36, 0, 54, 1, 35, 32, "car", 32, "a", 21, 36, 0, 54, 1, 35, 32, "foo", 32, "a", 21, 36, 0, 54, 1, 35, 32, "car", 32, +"a", 22, 36, 0, 54, 1, 35, 32, "arg", 32, "another", 2, "concat", 2, 36, 0, 54, 1, 35, 33, 1, 31, 2, "concat", 2, 36, 0, +54, 1, 35, 29, 30, 2, "concat", 2, 36, 0, 54, 1, 35, 32, "test", 32, "e.*", 2, "match", 2, 36, 0, 54, 1, 35, 32, "test", +32, "^e.*", 2, "match", 2, 36, 0, 54, 1, 35, 32, "test", 32, "x.*", 2, "match", 2, 36, 0, 54, 1, 35, 32, "e.*", 32, +"test", 23, 36, 0, 54, 1, 35, 32, "e.*", 32, "test", 24, 36, 0, 54, 1, 35, 32, "^e.*", 32, "test", 23, 36, 0, 54, 1, 35, +32, "^e.*", 32, "test", 24, 36, 0, 54, 1, 35, 32, "x.*", 32, "test", 23, 36, 0, 54, 1, 35, 32, "x.*", 32, "test", 24, +36, 0, 54, 1, 35, 32, "EST", 32, "test", 25, 36, 0, 54, 1, 35, 32, "EST", 32, "test", 25, 36, 0, 54, 1, 35, 32, "EST", +32, "test", 26, 36, 0, 54, 1, 35, 33, 1, 2, "toString", 1, 36, 0, 54, 1, 35, 34, 1.5, 2, "toString", 1, 36, 0, 54, 1, +35, 29, 2, "toString", 1, 36, 0, 54, 1, 35, 31, 2, "toString", 1, 36, 0, 54, 1, 35, 32, "string", 2, "toString", 1, 36, +0, 54, 1, 35, 32, "1", 2, "toInt", 1, 36, 0, 54, 1, 35, 32, "bla", 2, "toInt", 1, 36, 0, 54, 1, 35, 32, "1.2", 2, +"toFloat", 1, 36, 0, 54, 1, 35, 32, "bla", 2, "toFloat", 1, 36, 0, 54, 1, 35, 32, "asd", 2, "toUUID", 1, 36, 0, 54, 1, +35, 31, 33, 1, 11, 36, 0, 54, 1, 35, 31, 33, 1, 12, 36, 0, 54, 1, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/recursion.hoge b/hogvm/__tests__/__snapshots__/recursion.hoge new file mode 100644 index 0000000000000..5f8d5d630584a --- /dev/null +++ b/hogvm/__tests__/__snapshots__/recursion.hoge @@ -0,0 +1,4 @@ +["_H", 1, 52, "lambda", 1, 1, 34, 33, 2, 36, 0, 15, 40, 5, 36, 0, 38, 39, 20, 33, 2, 36, 0, 7, 55, 0, 54, 1, 33, 1, 36, +0, 7, 55, 0, 54, 1, 6, 38, 31, 38, 53, 1, true, 0, 33, 6, 36, 0, 54, 1, 2, "print", 1, 35, 52, "hogonacci", 1, 1, 34, +33, 2, 36, 0, 15, 40, 5, 36, 0, 38, 39, 20, 33, 2, 36, 0, 7, 55, 0, 54, 1, 33, 1, 36, 0, 7, 55, 0, 54, 1, 6, 38, 31, 38, +53, 1, true, 1, 33, 6, 36, 1, 54, 1, 2, "print", 1, 35, 57, 57] diff --git a/hogvm/__tests__/__snapshots__/recursion.stdout b/hogvm/__tests__/__snapshots__/recursion.stdout new file mode 100644 index 0000000000000..226aaf8af79f3 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/recursion.stdout @@ -0,0 +1,2 @@ +8 +8 diff --git a/hogvm/__tests__/__snapshots__/scope.hoge b/hogvm/__tests__/__snapshots__/scope.hoge new file mode 100644 index 0000000000000..0d3dc6528af4a --- /dev/null +++ b/hogvm/__tests__/__snapshots__/scope.hoge @@ -0,0 +1,17 @@ +["_H", 1, 52, "lambda", 1, 0, 6, 33, 2, 36, 0, 8, 38, 53, 0, 36, 0, 2, "print", 1, 35, 33, 2, 36, 0, 54, 1, 2, "print", +1, 35, 33, 8, 36, 0, 54, 1, 2, "print", 1, 35, 32, "--------", 2, "print", 1, 35, 33, 5, 52, "lambda", 1, 1, 6, 55, 0, +36, 0, 8, 38, 53, 1, true, 1, 33, 2, 36, 2, 54, 1, 2, "print", 1, 35, 33, 10, 37, 1, 33, 2, 36, 2, 54, 1, 2, "print", 1, +35, 33, 8, 36, 2, 54, 1, 2, "print", 1, 35, 32, "--------", 2, "print", 1, 35, 52, "bigVar", 0, 0, 22, 33, 5, 52, +"lambda", 1, 1, 6, 55, 0, 36, 0, 8, 38, 53, 1, true, 0, 36, 1, 38, 35, 57, 53, 0, 36, 3, 54, 0, 33, 2, 36, 4, 54, 1, 2, +"print", 1, 35, 33, 8, 36, 4, 54, 1, 2, "print", 1, 35, 32, "--------", 2, "print", 1, 35, 33, 3, 52, "outerA", 0, 1, +18, 55, 0, 2, "print", 1, 35, 33, 4, 56, 0, 55, 0, 2, "print", 1, 35, 31, 38, 53, 1, true, 5, 52, "innerA", 0, 2, 19, +55, 0, 2, "print", 1, 35, 55, 1, 54, 0, 35, 55, 0, 2, "print", 1, 35, 31, 38, 53, 2, true, 5, true, 6, 36, 5, 2, +"print", 1, 35, 36, 7, 54, 0, 35, 36, 5, 2, "print", 1, 35, 32, "--------", 2, "print", 1, 35, 32, "key", 33, 3, 42, 1, +52, "outerB", 0, 1, 21, 55, 0, 2, "print", 1, 35, 55, 0, 32, "key", 33, 4, 46, 55, 0, 2, "print", 1, 35, 31, 38, 53, 1, +true, 8, 52, "innerB", 0, 2, 19, 55, 0, 2, "print", 1, 35, 55, 1, 54, 0, 35, 55, 0, 2, "print", 1, 35, 31, 38, 53, 2, +true, 8, true, 9, 36, 8, 2, "print", 1, 35, 36, 10, 54, 0, 35, 36, 8, 2, "print", 1, 35, 32, "--------", 2, "print", 1, +35, 52, "outerC", 0, 0, 28, 32, "outside", 52, "innerC", 0, 1, 8, 55, 0, 2, "print", 1, 35, 31, 38, 53, 1, true, 0, 36, +1, 54, 0, 35, 31, 38, 35, 57, 53, 0, 36, 11, 54, 0, 35, 32, "--------", 2, "print", 1, 35, 52, "myFunctionOuter", 0, 0, +41, 33, 3, 52, "myFunction", 1, 1, 6, 55, 0, 36, 0, 6, 38, 53, 1, true, 0, 33, 2, 36, 1, 54, 1, 2, "print", 1, 35, 33, +3, 36, 1, 54, 1, 2, "print", 1, 35, 31, 38, 35, 57, 53, 0, 36, 12, 54, 0, 35, 32, "--------", 2, "print", 1, 35, 35, 35, +35, 57, 57, 35, 57, 57, 35, 35, 35, 57, 35] diff --git a/hogvm/__tests__/__snapshots__/scope.stdout b/hogvm/__tests__/__snapshots__/scope.stdout new file mode 100644 index 0000000000000..72bc33d5a91da --- /dev/null +++ b/hogvm/__tests__/__snapshots__/scope.stdout @@ -0,0 +1,30 @@ +fn +4 +16 +-------- +10 +20 +80 +-------- +10 +40 +-------- +3 +3 +3 +4 +4 +4 +-------- +{'key': 3} +{'key': 3} +{'key': 3} +{'key': 4} +{'key': 4} +{'key': 4} +-------- +outside +-------- +5 +6 +-------- diff --git a/hogvm/__tests__/__snapshots__/upvalues.hoge b/hogvm/__tests__/__snapshots__/upvalues.hoge new file mode 100644 index 0000000000000..fbc0e8f1f422a --- /dev/null +++ b/hogvm/__tests__/__snapshots__/upvalues.hoge @@ -0,0 +1,4 @@ +["_H", 1, 52, "returnCallable", 1, 0, 16, 52, "lambda", 1, 1, 6, 55, 0, 36, 0, 8, 38, 53, 1, true, 0, 38, 53, 0, 33, 2, +36, 0, 54, 1, 33, 3, 36, 0, 54, 1, 33, 2, 36, 1, 54, 1, 2, "print", 1, 35, 33, 2, 36, 2, 54, 1, 2, "print", 1, 35, 32, +"----------", 2, "print", 1, 35, 52, "outer", 0, 0, 24, 32, "outside", 52, "inner", 0, 1, 8, 55, 0, 2, "print", 1, 35, +31, 38, 53, 1, true, 0, 36, 1, 38, 35, 57, 53, 0, 36, 3, 54, 0, 36, 4, 54, 0, 35, 35, 35, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/upvalues.stdout b/hogvm/__tests__/__snapshots__/upvalues.stdout new file mode 100644 index 0000000000000..00dc3fe9018a4 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/upvalues.stdout @@ -0,0 +1,4 @@ +4 +6 +---------- +outside diff --git a/hogvm/__tests__/functionVars.hog b/hogvm/__tests__/functionVars.hog new file mode 100644 index 0000000000000..a105169b1c7d4 --- /dev/null +++ b/hogvm/__tests__/functionVars.hog @@ -0,0 +1,39 @@ +fn execFunction() { + print('execFunction') +} + +fn execFunctionNested() { + fn execFunction() { + print('execFunctionNew') + } + print('execFunctionNested') + execFunction() +} + +execFunction() +execFunctionNested() +execFunction() + +print('--------') + +fn secondExecFunction() { + print('secondExecFunction') +} + +fn secondExecFunctionNested() { + print('secondExecFunctionNested') + secondExecFunction() +} + +secondExecFunction() +secondExecFunctionNested() +secondExecFunction() + +print('--------') + +let decode := () -> base64Decode +let sixtyFour := base64Encode + +print(sixtyFour('http://www.google.com')) +print((decode())(sixtyFour('http://www.google.com'))) +print(decode()(sixtyFour('http://www.google.com'))) diff --git a/hogvm/__tests__/functions.hog b/hogvm/__tests__/functions.hog index b549e9b10705a..2b74a4fae55e2 100644 --- a/hogvm/__tests__/functions.hog +++ b/hogvm/__tests__/functions.hog @@ -3,18 +3,26 @@ print('-- test functions --') fn add(a, b) { return a + b } +print(add) + fn add2(a, b) { let c := a + b return c } +print(add2) + fn mult(a, b) { return a * b } +print(mult) + fn noArgs() { let url := 'basdfasdf' let second := 2 + 3 return second } +print(noArgs) + fn empty() {} fn empty2() {} fn empty3() {} @@ -57,3 +65,22 @@ print(ifThenReturn() ?? -1) print(mult(add(3, 4) + 100 + add(2, 1), 2)) print(mult(add2(3, 4) + 100 + add2(2, 1), 10)) + +// Call functions with different argument counts + +fn printArgs(arg1, arg2, arg3, arg4, arg5, arg6, arg7) { + print(arg1, arg2, arg3, arg4, arg5, arg6, arg7) +} + +let printArgs2 := (arg1, arg2, arg3, arg4, arg5, arg6, arg7) -> { + print(arg1, arg2, arg3, arg4, arg5, arg6, arg7) +} + +printArgs(1, 2, 3, 4, 5, 6, 7) +printArgs2(1, 2, 3, 4, 5, 6, 7) +printArgs(1, 2, 3, 4, 5, 6) +printArgs2(1, 2, 3, 4, 5, 6) +printArgs(1, 2, 3, 4, 5) +printArgs2(1, 2, 3, 4, 5) +printArgs() +printArgs2() \ No newline at end of file diff --git a/hogvm/__tests__/lambdas.hog b/hogvm/__tests__/lambdas.hog new file mode 100644 index 0000000000000..4c27399e107c1 --- /dev/null +++ b/hogvm/__tests__/lambdas.hog @@ -0,0 +1,30 @@ +let b := x -> x * 2 +print(b) +print(b(2)) +print(b(8)) + +print('--------') + +let func := x -> x * 2 +let arr := [func] + +print(func(2)) +print(arr[1](2)) +print((x -> x * 2)(2)) + +print('--------') + +let withArg := x -> { + print(x) + print('moo') + print('cow') +} +withArg(2) + +print('--------') + +let noArg := () -> { + print('moo') + print('cow') +} +noArg() diff --git a/hogvm/__tests__/recursion.hog b/hogvm/__tests__/recursion.hog new file mode 100644 index 0000000000000..09938bcba3d28 --- /dev/null +++ b/hogvm/__tests__/recursion.hog @@ -0,0 +1,19 @@ +let fibonacci := (number) -> { + if (number < 2) { + return number; + } else { + return fibonacci(number - 1) + fibonacci(number - 2); + } +} + +print(fibonacci(6)) + +fn hogonacci(number) { + if (number < 2) { + return number; + } else { + return hogonacci(number - 1) + hogonacci(number - 2); + } +} + +print(hogonacci(6)) diff --git a/hogvm/__tests__/scope.hog b/hogvm/__tests__/scope.hog new file mode 100644 index 0000000000000..083282018c76d --- /dev/null +++ b/hogvm/__tests__/scope.hog @@ -0,0 +1,93 @@ +let dbl := x -> x * 2 + +print(dbl) +print(dbl(2)) +print(dbl(8)) + +print('--------') + +let var := 5 +let varify := x -> x * var + +print(varify(2)) +var := 10 +print(varify(2)) +print(varify(8)) + +print('--------') + +fn bigVar() { + let var := 5 + let varify := x -> x * var + return varify +} +let bigVarify := bigVar() + +print(bigVarify(2)) +print(bigVarify(8)) + +print('--------') + +let a := 3 + +fn outerA() { + print(a) + a := 4 + print(a) +} + +fn innerA() { + print(a) + outerA() + print(a) +} + +print(a) +innerA() +print(a) + +print('--------') + +let b := {'key': 3} + +fn outerB() { + print(b) + b.key := 4 + print(b) +} + +fn innerB() { + print(b) + outerB() + print(b) +} + +print(b) +innerB() +print(b) + +print('--------') + +fn outerC() { + let x := 'outside' + fn innerC() { + print(x) + } + innerC() +} +outerC() + +print('--------') + +fn myFunctionOuter() { + let b := 3 + fn myFunction(a) { + return a + b + } + print(myFunction(2)) + print(myFunction(3)) +} + +myFunctionOuter() + +print('--------') diff --git a/hogvm/__tests__/upvalues.hog b/hogvm/__tests__/upvalues.hog new file mode 100644 index 0000000000000..9e1de29a3e19f --- /dev/null +++ b/hogvm/__tests__/upvalues.hog @@ -0,0 +1,23 @@ +fn returnCallable(a) { + return x -> x * a +} + +let double := returnCallable(2) +let triple := returnCallable(3) + +print(double(2)) // 4 +print(triple(2)) // 6 + +print('----------') + +fn outer() { + let x := 'outside' + fn inner() { + print(x) + } + + return inner +} + +let closure := outer() +closure() diff --git a/hogvm/python/debugger.py b/hogvm/python/debugger.py index 10d4355e8c1c4..21365114e427d 100644 --- a/hogvm/python/debugger.py +++ b/hogvm/python/debugger.py @@ -2,6 +2,7 @@ from time import sleep from typing import Any +from hogvm.python.objects import CallFrame from hogvm.python.operation import Operation debug_speed = -1 @@ -62,7 +63,7 @@ def debugger( sleep(debug_speed / 1000) -def print_symbol(symbol: Operation, ip: int, bytecode: list, stack: list, call_stack: list) -> str: +def print_symbol(symbol: Operation, ip: int, bytecode: list, stack: list, call_stack: list[CallFrame]) -> str: try: match symbol: case Operation.STRING: @@ -132,15 +133,20 @@ def print_symbol(symbol: Operation, ip: int, bytecode: list, stack: list, call_s case Operation.POP: return f"POP({stack[-1]})" case Operation.RETURN: - if call_stack: - ip, stack_start, arg_len = call_stack[-1] - return f"RETURN({stack[-1]} --> {ip}/{stack_start})" + if len(call_stack) > 1: + return f"RETURN({stack[-1]}) --> {call_stack[-2].ip}" else: return "RETURN" case Operation.GET_LOCAL: return f"GET_LOCAL({bytecode[ip+1]})" case Operation.SET_LOCAL: - return f"GET_LOCAL({bytecode[ip + 1]}, {stack[-1]})" + return f"SET_LOCAL({bytecode[ip + 1]}, {stack[-1]})" + case Operation.GET_UPVALUE: + return f"GET_UPVALUE({bytecode[ip+1]})" + case Operation.SET_UPVALUE: + return f"SET_UPVALUE({bytecode[ip + 1]})" + case Operation.CLOSE_UPVALUE: + return "CLOSE_UPVALUE" case Operation.GET_PROPERTY: return f"GET_PROPERTY({stack[-2]}, {stack[-1]})" case Operation.GET_PROPERTY_NULLISH: @@ -163,8 +169,14 @@ def print_symbol(symbol: Operation, ip: int, bytecode: list, stack: list, call_s ) case Operation.DECLARE_FN: return f"DECLARE_FN({bytecode[ip+1]}, args={bytecode[ip+2]}, ops={bytecode[ip+3]})" + case Operation.CALLABLE: + return f"CALLABLE({bytecode[ip+1]}, args={bytecode[ip+2]}, upvalues={bytecode[ip+3]}, ops={bytecode[ip+4]})" + case Operation.CLOSURE: + return f"CLOSURE" case Operation.CALL_GLOBAL: - return f"CALL_GLOBAL({bytecode[ip+1]} {', '.join(str(stack[-(bytecode[ip+2] - i)]) for i in range(bytecode[ip+2]))})" + return f"CALL_GLOBAL({bytecode[ip+1]}, {', '.join(str(stack[-(bytecode[ip+2] - i)]) for i in range(bytecode[ip+2]))})" + case Operation.CALL_LOCAL: + return f"CALL_LOCAL({bytecode[ip+1]} {', '.join(str(stack[-(bytecode[ip+1] - i)]) for i in range(bytecode[ip+1]))})" case Operation.TRY: return f"TRY(+{bytecode[ip+1]})" case Operation.POP_TRY: @@ -254,6 +266,12 @@ def color_bytecode(bytecode: list) -> list: add = ["op.GET_LOCAL", f"index: {bytecode[ip+1]}"] case Operation.SET_LOCAL: add = ["op.SET_LOCAL", f"index: {bytecode[ip+1]}"] + case Operation.GET_UPVALUE: + add = ["op.GET_UPVALUE", f"index: {bytecode[ip+1]}"] + case Operation.SET_UPVALUE: + add = ["op.SET_UPVALUE", f"index: {bytecode[ip+1]}"] + case Operation.CLOSE_UPVALUE: + add = ["op.CLOSE_UPVALUE"] case Operation.GET_PROPERTY: add = ["op.GET_PROPERTY"] case Operation.GET_PROPERTY_NULLISH: @@ -274,6 +292,22 @@ def color_bytecode(bytecode: list) -> list: add = ["op.JUMP_IF_STACK_NOT_NULL", f"offset: {'+' if bytecode[ip+1] >= 0 else ''}{bytecode[ip+1]}"] case Operation.DECLARE_FN: add = ["op.DECLARE_FN", f"name: {bytecode[ip+1]}", f"args: {bytecode[ip+2]}", f"ops: {bytecode[ip+3]}"] + case Operation.CALLABLE: + add = [ + "op.CALLABLE", + f"name: {bytecode[ip+1]}", + f"args: {bytecode[ip+2]}", + f"upvalues: {bytecode[ip+3]}", + f"ops: {bytecode[ip+4]}", + ] + case Operation.CLOSURE: + upvalue_count = bytecode[ip + 1] + add = ["op.CLOSURE", f"upvalues: {upvalue_count}"] + for i in range(upvalue_count): + add.append(f"is_local({i}): {bytecode[ip + 2 + i * 2]}") + add.append(f"index({i}): {bytecode[ip + 2 + i * 2 + 1]}") + case Operation.CALL_LOCAL: + add = ["op.CALL_LOCAL", f"args: {bytecode[ip+1]}"] case Operation.CALL_GLOBAL: add = ["op.CALL_GLOBAL", f"name: {bytecode[ip+1]}", f"args: {bytecode[ip+2]}"] case Operation.TRY: diff --git a/hogvm/python/execute.py b/hogvm/python/execute.py index 1f0a47e1bd1d9..ac37a7314fc1d 100644 --- a/hogvm/python/execute.py +++ b/hogvm/python/execute.py @@ -6,7 +6,7 @@ from collections.abc import Callable from hogvm.python.debugger import debugger, color_bytecode -from hogvm.python.objects import is_hog_error +from hogvm.python.objects import is_hog_error, new_hog_closure, CallFrame, ThrowFrame, new_hog_callable, is_hog_upvalue from hogvm.python.operation import Operation, HOGQL_BYTECODE_IDENTIFIER, HOGQL_BYTECODE_IDENTIFIER_V0 from hogvm.python.stl import STL from dataclasses import dataclass @@ -24,6 +24,7 @@ from posthog.models import Team MAX_MEMORY = 64 * 1024 * 1024 # 64 MB +MAX_FUNCTION_ARGS_LENGTH = 300 @dataclass @@ -41,38 +42,65 @@ def execute_bytecode( team: Optional["Team"] = None, debug=False, ) -> BytecodeResult: - if len(bytecode) == 0 or (bytecode[0] not in (HOGQL_BYTECODE_IDENTIFIER, HOGQL_BYTECODE_IDENTIFIER_V0)): + if len(bytecode) == 0 or (bytecode[0] != HOGQL_BYTECODE_IDENTIFIER and bytecode[0] != HOGQL_BYTECODE_IDENTIFIER_V0): raise HogVMException(f"Invalid bytecode. Must start with '{HOGQL_BYTECODE_IDENTIFIER}'") version = bytecode[1] if len(bytecode) >= 2 and bytecode[0] == HOGQL_BYTECODE_IDENTIFIER else 0 result = None start_time = time.time() last_op = len(bytecode) - 1 stack: list = [] + upvalues: list[dict] = [] + upvalues_by_id: dict[int, dict] = {} mem_stack: list = [] - call_stack: list[tuple[int, int, int]] = [] # (ip, stack_start, arg_len) - throw_stack: list[tuple[int, int, int]] = [] # (call_stack_length, stack_length, catch_ip) + call_stack: list[CallFrame] = [] + throw_stack: list[ThrowFrame] = [] declared_functions: dict[str, tuple[int, int]] = {} mem_used = 0 max_mem_used = 0 - ip = 1 if version > 0 else 0 ops = 0 stdout: list[str] = [] colored_bytecode = color_bytecode(bytecode) if debug else [] if isinstance(timeout, int): timeout = timedelta(seconds=timeout) + if len(call_stack) == 0: + call_stack.append( + CallFrame( + ip=2 if bytecode[0] == HOGQL_BYTECODE_IDENTIFIER else 1, + stack_start=0, + arg_len=0, + closure=new_hog_closure( + new_hog_callable( + type="main", + arg_count=0, + upvalue_count=0, + ip=2 if bytecode[0] == HOGQL_BYTECODE_IDENTIFIER else 1, + name="", + ) + ), + ) + ) + frame = call_stack[-1] + def stack_keep_first_elements(count: int): nonlocal stack, mem_stack, mem_used + for upvalue in reversed(upvalues): + if upvalue["location"] >= count: + if not upvalue["closed"]: + upvalue["closed"] = True + upvalue["value"] = stack[upvalue["location"]] + else: + break stack = stack[0:count] mem_used -= sum(mem_stack[count:]) mem_stack = mem_stack[0:count] def next_token(): - nonlocal ip - ip += 1 - if ip > last_op: + nonlocal frame + if frame.ip >= last_op: raise HogVMException("Unexpected end of bytecode") - return bytecode[ip] + frame.ip += 1 + return bytecode[frame.ip] def pop_stack(): if not stack: @@ -91,20 +119,37 @@ def push_stack(value): if mem_used > MAX_MEMORY: raise HogVMException(f"Memory limit of {MAX_MEMORY} bytes exceeded. Tried to allocate {mem_used} bytes.") - if len(bytecode) <= 2: - return BytecodeResult(result=None, stdout=stdout, bytecode=bytecode) - def check_timeout(): if time.time() - start_time > timeout.total_seconds() and not debug: raise HogVMException(f"Execution timed out after {timeout.total_seconds()} seconds. Performed {ops} ops.") - while True: + def capture_upvalue(index) -> dict: + nonlocal upvalues + for upvalue in reversed(upvalues): + if upvalue["location"] < index: + break + if upvalue["location"] == index: + return upvalue + created_upvalue = { + "__hogUpValue__": True, + "location": index, + "closed": False, + "value": None, + "id": len(upvalues) + 1, + } + upvalues.append(created_upvalue) + upvalues_by_id[created_upvalue["id"]] = created_upvalue + upvalues.sort(key=lambda x: x["location"]) + return created_upvalue + + symbol: Any = None + while frame.ip <= last_op: ops += 1 - symbol = next_token() + symbol = bytecode[frame.ip] if (ops & 127) == 0: # every 128th operation check_timeout() elif debug: - debugger(symbol, bytecode, colored_bytecode, ip, stack, call_stack, throw_stack) + debugger(symbol, bytecode, colored_bytecode, frame.ip, stack, call_stack, throw_stack) match symbol: case None: break @@ -178,23 +223,52 @@ def check_timeout(): chain = [pop_stack() for _ in range(next_token())] if globals and chain[0] in globals: push_stack(deepcopy(get_nested_value(globals, chain))) + elif functions and chain[0] in functions: + push_stack( + new_hog_closure( + new_hog_callable( + type="stl", + name=chain[0], + arg_count=0, + upvalue_count=0, + ip=-1, + ) + ) + ) + elif chain[0] in STL and len(chain) == 1: + push_stack( + new_hog_closure( + new_hog_callable( + type="stl", + name=chain[0], + arg_count=STL[chain[0]].maxArgs or 0, + upvalue_count=0, + ip=-1, + ) + ) + ) else: raise HogVMException(f"Global variable not found: {chain[0]}") case Operation.POP: pop_stack() + case Operation.CLOSE_UPVALUE: + stack_keep_first_elements(len(stack) - 1) case Operation.RETURN: - if call_stack: - ip, stack_start, arg_len = call_stack.pop() - response = pop_stack() - stack_keep_first_elements(stack_start) - push_stack(response) - else: - return BytecodeResult(result=pop_stack(), stdout=stdout, bytecode=bytecode) + response = pop_stack() + last_call_frame = call_stack.pop() + if len(call_stack) == 0 or last_call_frame is None: + return BytecodeResult(result=response, stdout=stdout, bytecode=bytecode) + stack_start = last_call_frame.stack_start + stack_keep_first_elements(stack_start) + push_stack(response) + frame = call_stack[-1] + continue # resume the loop without incrementing frame.ip + case Operation.GET_LOCAL: - stack_start = 0 if not call_stack else call_stack[-1][1] + stack_start = 0 if not call_stack else call_stack[-1].stack_start push_stack(stack[next_token() + stack_start]) case Operation.SET_LOCAL: - stack_start = 0 if not call_stack else call_stack[-1][1] + stack_start = 0 if not call_stack else call_stack[-1].stack_start value = pop_stack() index = next_token() + stack_start stack[index] = value @@ -244,42 +318,177 @@ def check_timeout(): push_stack(()) case Operation.JUMP: count = next_token() - ip += count + frame.ip += count case Operation.JUMP_IF_FALSE: count = next_token() if not pop_stack(): - ip += count + frame.ip += count case Operation.JUMP_IF_STACK_NOT_NULL: count = next_token() if len(stack) > 0 and stack[-1] is not None: - ip += count + frame.ip += count case Operation.DECLARE_FN: + # DEPRECATED name = next_token() arg_len = next_token() body_len = next_token() - declared_functions[name] = (ip, arg_len) - ip += body_len + declared_functions[name] = (frame.ip + 1, arg_len) + frame.ip += body_len + case Operation.CALLABLE: + name = next_token() # TODO: do we need it? it could change as the variable is reassigned + arg_count = next_token() + upvalue_count = next_token() + body_length = next_token() + push_stack( + new_hog_callable( + type="local", + name=name, + arg_count=arg_count, + upvalue_count=upvalue_count, + ip=frame.ip + 1, + ) + ) + frame.ip += body_length + case Operation.CLOSURE: + closure_callable = pop_stack() + closure = new_hog_closure(closure_callable) + stack_start = frame.stack_start + upvalue_count = next_token() + if upvalue_count != closure_callable["upvalueCount"]: + raise HogVMException( + f"Invalid upvalue count. Expected {closure_callable['upvalueCount']}, got {upvalue_count}" + ) + for _ in range(closure_callable["upvalueCount"]): + is_local, index = next_token(), next_token() + if is_local: + closure["upvalues"].append(capture_upvalue(stack_start + index)["id"]) + else: + closure["upvalues"].append(frame.closure["upvalues"][index]) + push_stack(closure) + case Operation.GET_UPVALUE: + index = next_token() + closure = frame.closure + if index >= len(closure["upvalues"]): + raise HogVMException(f"Invalid upvalue index: {index}") + upvalue = upvalues_by_id[closure["upvalues"][index]] + if not is_hog_upvalue(upvalue): + raise HogVMException(f"Invalid upvalue: {upvalue}") + if upvalue["closed"]: + push_stack(upvalue["value"]) + else: + push_stack(stack[upvalue["location"]]) + case Operation.SET_UPVALUE: + index = next_token() + closure = frame.closure + if index >= len(closure["upvalues"]): + raise HogVMException(f"Invalid upvalue index: {index}") + upvalue = upvalues_by_id[closure["upvalues"][index]] + if not is_hog_upvalue(upvalue): + raise HogVMException(f"Invalid upvalue: {upvalue}") + if upvalue["closed"]: + upvalue["value"] = pop_stack() + else: + stack[upvalue["location"]] = pop_stack() case Operation.CALL_GLOBAL: check_timeout() name = next_token() + arg_count = next_token() + # This is for backwards compatibility. We use a closure on the stack with local functions now. if name in declared_functions: func_ip, arg_len = declared_functions[name] - call_stack.append((ip + 1, len(stack) - arg_len, arg_len)) - ip = func_ip + frame.ip += 1 # advance for when we return + if arg_len > arg_count: + for _ in range(arg_len - arg_count): + push_stack(None) + frame = CallFrame( + ip=func_ip, + stack_start=len(stack) - arg_len, + arg_len=arg_len, + closure=new_hog_closure( + new_hog_callable( + type="stl", + name=name, + arg_count=arg_len, + upvalue_count=0, + ip=-1, + ) + ), + ) + call_stack.append(frame) + continue # resume the loop without incrementing frame.ip else: - arg_count = next_token() - args = [pop_stack() for _ in range(arg_count)] - if version > 0: - args = list(reversed(args)) - + # Shortcut for calling STL functions (can also be done with an STL function closure) + if version == 0: + args = [pop_stack() for _ in range(arg_count)] + else: + args = list(reversed([pop_stack() for _ in range(arg_count)])) if functions is not None and name in functions: push_stack(functions[name](*args)) elif name in STL: - push_stack(STL[name](args, team, stdout, timeout.total_seconds())) + push_stack(STL[name].fn(args, team, stdout, timeout.total_seconds())) else: raise HogVMException(f"Unsupported function call: {name}") + case Operation.CALL_LOCAL: + check_timeout() + closure = pop_stack() + if not isinstance(closure, dict) or closure.get("__hogClosure__") is None: + raise HogVMException(f"Invalid closure: {closure}") + callable = closure.get("callable") + if not isinstance(callable, dict) or callable.get("__hogCallable__") is None: + raise HogVMException(f"Invalid callable: {callable}") + args_length = next_token() + if args_length > MAX_FUNCTION_ARGS_LENGTH: + raise HogVMException("Too many arguments") + + if callable.get("__hogCallable__") == "local": + if callable["argCount"] > args_length: + # TODO: specify minimum required arguments somehow + for _ in range(callable["argCount"] - args_length): + push_stack(None) + elif callable["argCount"] < args_length: + raise HogVMException( + f"Too many arguments. Passed {args_length}, expected {callable['argCount']}" + ) + frame.ip += 1 # advance for when we return + frame = CallFrame( + ip=callable["ip"], + stack_start=len(stack) - callable["argCount"], + arg_len=callable["argCount"], + closure=closure, + ) + call_stack.append(frame) + continue # resume the loop without incrementing frame.ip + + elif callable.get("__hogCallable__") == "stl": + if callable["name"] not in STL: + raise HogVMException(f"Unsupported function call: {callable['name']}") + stl_fn = STL[callable["name"]] + if stl_fn.minArgs is not None and args_length < stl_fn.minArgs: + raise HogVMException( + f"Function {callable['name']} requires at least {stl_fn.minArgs} arguments" + ) + if stl_fn.maxArgs is not None and args_length > stl_fn.maxArgs: + raise HogVMException(f"Function {callable['name']} requires at most {stl_fn.maxArgs} arguments") + if version == 0: + args = [pop_stack() for _ in range(args_length)] + else: + args = list(reversed([pop_stack() for _ in range(args_length)])) + if stl_fn.maxArgs is not None and len(args) < stl_fn.maxArgs: + args = [*args, *([None] * (stl_fn.maxArgs - len(args)))] + push_stack(stl_fn.fn(args, team, stdout, timeout.total_seconds())) + + elif callable.get("__hogCallable__") == "async": + raise HogVMException("Async functions are not supported") + + else: + raise HogVMException("Invalid callable") + case Operation.TRY: - throw_stack.append((len(call_stack), len(stack), ip + next_token())) + throw_stack.append( + ThrowFrame( + call_stack_len=len(call_stack), stack_len=len(stack), catch_ip=frame.ip + 1 + next_token() + ) + ) case Operation.POP_TRY: if throw_stack: throw_stack.pop() @@ -290,11 +499,18 @@ def check_timeout(): if not is_hog_error(exception): raise HogVMException("Can not throw: value is not of type Error") if throw_stack: - call_stack_len, stack_len, catch_ip = throw_stack.pop() + last_throw = throw_stack.pop() + call_stack_len, stack_len, catch_ip = ( + last_throw.call_stack_len, + last_throw.stack_len, + last_throw.catch_ip, + ) stack_keep_first_elements(stack_len) call_stack = call_stack[0:call_stack_len] push_stack(exception) - ip = catch_ip + frame = call_stack[-1] + frame.ip = catch_ip + continue else: raise UncaughtHogVMException( type=exception.get("type"), @@ -302,10 +518,9 @@ def check_timeout(): payload=exception.get("payload"), ) - if ip == last_op: - break + frame.ip += 1 if debug: - debugger(symbol, bytecode, colored_bytecode, ip, stack, call_stack, throw_stack) + debugger(symbol, bytecode, colored_bytecode, frame.ip, stack, call_stack, throw_stack) if len(stack) > 1: raise HogVMException("Invalid bytecode. More than one value left on stack") if len(stack) == 1: diff --git a/hogvm/python/objects.py b/hogvm/python/objects.py index a514e7a5686d9..1274a672e91ba 100644 --- a/hogvm/python/objects.py +++ b/hogvm/python/objects.py @@ -1,4 +1,20 @@ -from typing import Any +from dataclasses import dataclass +from typing import Any, Optional + + +@dataclass +class CallFrame: + ip: int + stack_start: int + arg_len: int + closure: dict + + +@dataclass +class ThrowFrame: + call_stack_len: int + stack_len: int + catch_ip: int def is_hog_date(obj: Any) -> bool: @@ -20,3 +36,46 @@ def new_hog_error(type: str, message: Any, payload: Any = None) -> dict: "message": message or "An error occurred", "payload": payload, } + + +def is_hog_callable(obj: Any) -> bool: + return ( + isinstance(obj, dict) + and "__hogCallable__" in obj + and "argCount" in obj + and "ip" in obj + and "upvalueCount" in obj + ) + + +def is_hog_closure(obj: Any) -> bool: + return isinstance(obj, dict) and "__hogClosure__" in obj and "callable" in obj and "upvalues" in obj + + +def new_hog_closure(callable: dict, upvalues: Optional[list] = None) -> dict: + return { + "__hogClosure__": True, + "callable": callable, + "upvalues": upvalues or [], + } + + +def new_hog_callable(type: str, arg_count: int, upvalue_count: int, ip: int, name: str) -> dict: + return { + "__hogCallable__": type, + "name": name, + "argCount": arg_count, + "upvalueCount": upvalue_count, + "ip": ip, + } + + +def is_hog_upvalue(obj: Any) -> bool: + return ( + isinstance(obj, dict) + and "__hogUpValue__" in obj + and "location" in obj + and "closed" in obj + and "value" in obj + and "id" in obj + ) diff --git a/hogvm/python/operation.py b/hogvm/python/operation.py index 0feb79ebfac2e..f429f0d4d206d 100644 --- a/hogvm/python/operation.py +++ b/hogvm/python/operation.py @@ -62,3 +62,9 @@ class Operation(int, Enum): THROW = 49 TRY = 50 POP_TRY = 51 + CALLABLE = 52 + CLOSURE = 53 + CALL_LOCAL = 54 + GET_UPVALUE = 55 + SET_UPVALUE = 56 + CLOSE_UPVALUE = 57 diff --git a/hogvm/python/stl/__init__.py b/hogvm/python/stl/__init__.py index 5479937d37cc1..84eb98afecebc 100644 --- a/hogvm/python/stl/__init__.py +++ b/hogvm/python/stl/__init__.py @@ -1,3 +1,4 @@ +import dataclasses import datetime import time from typing import Any, Optional, TYPE_CHECKING @@ -29,6 +30,13 @@ from posthog.models import Team +@dataclasses.dataclass +class STLFunction: + fn: Callable[[list[Any], Optional["Team"], list[str] | None, float], Any] + minArgs: Optional[int] = None + maxArgs: Optional[int] = None + + def toString(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: float): if isinstance(args[0], dict) and is_hog_datetime(args[0]): dt = datetime.datetime.fromtimestamp(args[0]["dt"], pytz.timezone(args[0]["zone"] or "UTC")) @@ -167,27 +175,39 @@ def decodeURLComponent(args: list[Any], team: Optional["Team"], stdout: Optional def trim(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: float) -> str: - if len(args) > 1 and len(args[1]) > 1: - return "" - return args[0].strip(args[1] if len(args) > 1 else None) + char = str(args[1]) if len(args) > 1 and isinstance(args[1], str) else None + if len(args) > 1: + if char is None: + char = " " + if len(char) > 1: + return "" + return args[0].strip(char) def trimLeft(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: float) -> str: - if len(args) > 1 and len(args[1]) > 1: - return "" - return args[0].lstrip(args[1] if len(args) > 1 else None) + char = str(args[1]) if len(args) > 1 and isinstance(args[1], str) else None + if len(args) > 1: + if char is None: + char = " " + if len(char) > 1: + return "" + return args[0].lstrip(char) def trimRight(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: float) -> str: - if len(args) > 1 and len(args[1]) > 1: - return "" - return args[0].rstrip(args[1] if len(args) > 1 else None) + char = str(args[1]) if len(args) > 1 and isinstance(args[1], str) else None + if len(args) > 1: + if char is None: + char = " " + if len(char) > 1: + return "" + return args[0].rstrip(char) def splitByString(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: float) -> list: separator = args[0] string = args[1] - if len(args) > 2: + if len(args) > 2 and args[2] is not None: parts = string.split(separator, args[2]) if len(parts) > args[2]: return parts[: args[2]] @@ -290,80 +310,117 @@ def _formatDateTime(args: list[Any], team: Optional["Team"], stdout: Optional[li return formatDateTime(args[0], args[1], args[2] if len(args) > 2 else None) -STL: dict[str, Callable[[list[Any], Optional["Team"], list[str] | None, float], Any]] = { - "concat": lambda args, team, stdout, timeout: "".join( - [print_hog_string_output(arg) if arg is not None else "" for arg in args] +STL: dict[str, STLFunction] = { + "concat": STLFunction( + fn=lambda args, team, stdout, timeout: "".join( + [print_hog_string_output(arg) if arg is not None else "" for arg in args] + ), + minArgs=1, + maxArgs=None, ), - "match": lambda args, team, stdout, timeout: bool(re.search(re.compile(args[1]), args[0])), - "like": lambda args, team, stdout, timeout: like(args[0], args[1]), - "ilike": lambda args, team, stdout, timeout: like(args[0], args[1], re.IGNORECASE), - "notLike": lambda args, team, stdout, timeout: not like(args[0], args[1]), - "notILike": lambda args, team, stdout, timeout: not like(args[0], args[1], re.IGNORECASE), - "toString": toString, - "toUUID": toString, - "toInt": toInt, - "toFloat": toFloat, - "ifNull": ifNull, - "length": lambda args, team, stdout, timeout: len(args[0]), - "empty": lambda args, team, stdout, timeout: not bool(args[0]), - "notEmpty": lambda args, team, stdout, timeout: bool(args[0]), - "tuple": lambda args, team, stdout, timeout: tuple(args), - "lower": lambda args, team, stdout, timeout: args[0].lower(), - "upper": lambda args, team, stdout, timeout: args[0].upper(), - "reverse": lambda args, team, stdout, timeout: args[0][::-1], - "sleep": sleep, - "print": print, - "run": run, - "jsonParse": jsonParse, - "jsonStringify": jsonStringify, - "base64Encode": base64Encode, - "base64Decode": base64Decode, - "encodeURLComponent": encodeURLComponent, - "decodeURLComponent": decodeURLComponent, - "replaceOne": lambda args, team, stdout, timeout: args[0].replace(args[1], args[2], 1), - "replaceAll": lambda args, team, stdout, timeout: args[0].replace(args[1], args[2]), - "trim": trim, - "trimLeft": trimLeft, - "trimRight": trimRight, - "splitByString": splitByString, - "generateUUIDv4": generateUUIDv4, - "sha256Hex": lambda args, team, stdout, timeout: sha256Hex(args[0]), - "md5Hex": lambda args, team, stdout, timeout: md5Hex(args[0]), - "sha256HmacChainHex": lambda args, team, stdout, timeout: sha256HmacChainHex(args[0]), - "keys": keys, - "values": values, - "arrayPushBack": arrayPushBack, - "arrayPushFront": arrayPushFront, - "arrayPopBack": arrayPopBack, - "arrayPopFront": arrayPopFront, - "arraySort": arraySort, - "arrayReverse": arrayReverse, - "arrayReverseSort": arrayReverseSort, - "arrayStringConcat": arrayStringConcat, - "has": has, - "now": lambda args, team, stdout, timeout: now(), - "toUnixTimestamp": lambda args, team, stdout, timeout: toUnixTimestamp(args[0], args[1] if len(args) > 1 else None), - "fromUnixTimestamp": lambda args, team, stdout, timeout: fromUnixTimestamp(args[0]), - "toUnixTimestampMilli": lambda args, team, stdout, timeout: toUnixTimestampMilli(args[0]), - "fromUnixTimestampMilli": lambda args, team, stdout, timeout: fromUnixTimestampMilli(args[0]), - "toTimeZone": lambda args, team, stdout, timeout: toTimeZone(args[0], args[1]), - "toDate": lambda args, team, stdout, timeout: toDate(args[0]), - "toDateTime": lambda args, team, stdout, timeout: toDateTime(args[0]), - "formatDateTime": _formatDateTime, - "HogError": lambda args, team, stdout, timeout: new_hog_error(args[0], args[1], args[2] if len(args) > 2 else None), - "Error": lambda args, team, stdout, timeout: new_hog_error("Error", args[0], args[1] if len(args) > 1 else None), - "RetryError": lambda args, team, stdout, timeout: new_hog_error( - "RetryError", args[0], args[1] if len(args) > 1 else None + "match": STLFunction( + fn=lambda args, team, stdout, timeout: bool(re.search(re.compile(args[1]), args[0])), minArgs=2, maxArgs=2 ), - "NotImplementedError": lambda args, team, stdout, timeout: new_hog_error( - "NotImplementedError", args[0], args[1] if len(args) > 1 else None + "like": STLFunction(fn=lambda args, team, stdout, timeout: like(args[0], args[1]), minArgs=2, maxArgs=2), + "ilike": STLFunction( + fn=lambda args, team, stdout, timeout: like(args[0], args[1], re.IGNORECASE), minArgs=2, maxArgs=2 ), -} - - -MIN_ARGS_INCLUDING_OPTIONAL = { - "HogError": 3, - "Error": 2, - "RetryError": 2, - "NotImplementedError": 2, + "notLike": STLFunction(fn=lambda args, team, stdout, timeout: not like(args[0], args[1]), minArgs=2, maxArgs=2), + "notILike": STLFunction( + fn=lambda args, team, stdout, timeout: not like(args[0], args[1], re.IGNORECASE), minArgs=2, maxArgs=2 + ), + "toString": STLFunction(fn=toString, minArgs=1, maxArgs=1), + "toUUID": STLFunction(fn=toString, minArgs=1, maxArgs=1), + "toInt": STLFunction(fn=toInt, minArgs=1, maxArgs=1), + "toFloat": STLFunction(fn=toFloat, minArgs=1, maxArgs=1), + "ifNull": STLFunction(fn=ifNull, minArgs=2, maxArgs=2), + "length": STLFunction(fn=lambda args, team, stdout, timeout: len(args[0]), minArgs=1, maxArgs=1), + "empty": STLFunction(fn=lambda args, team, stdout, timeout: not bool(args[0]), minArgs=1, maxArgs=1), + "notEmpty": STLFunction(fn=lambda args, team, stdout, timeout: bool(args[0]), minArgs=1, maxArgs=1), + "tuple": STLFunction(fn=lambda args, team, stdout, timeout: tuple(args), minArgs=0, maxArgs=None), + "lower": STLFunction(fn=lambda args, team, stdout, timeout: args[0].lower(), minArgs=1, maxArgs=1), + "upper": STLFunction(fn=lambda args, team, stdout, timeout: args[0].upper(), minArgs=1, maxArgs=1), + "reverse": STLFunction(fn=lambda args, team, stdout, timeout: args[0][::-1], minArgs=1, maxArgs=1), + "print": STLFunction(fn=print, minArgs=0, maxArgs=None), + "jsonParse": STLFunction(fn=jsonParse, minArgs=1, maxArgs=1), + "jsonStringify": STLFunction(fn=jsonStringify, minArgs=1, maxArgs=1), + "base64Encode": STLFunction(fn=base64Encode, minArgs=1, maxArgs=1), + "base64Decode": STLFunction(fn=base64Decode, minArgs=1, maxArgs=1), + "encodeURLComponent": STLFunction(fn=encodeURLComponent, minArgs=1, maxArgs=1), + "decodeURLComponent": STLFunction(fn=decodeURLComponent, minArgs=1, maxArgs=1), + "replaceOne": STLFunction( + fn=lambda args, team, stdout, timeout: args[0].replace(args[1], args[2], 1), minArgs=3, maxArgs=3 + ), + "replaceAll": STLFunction( + fn=lambda args, team, stdout, timeout: args[0].replace(args[1], args[2]), minArgs=3, maxArgs=3 + ), + "trim": STLFunction(fn=trim, minArgs=1, maxArgs=2), + "trimLeft": STLFunction(fn=trimLeft, minArgs=1, maxArgs=2), + "trimRight": STLFunction(fn=trimRight, minArgs=1, maxArgs=2), + "splitByString": STLFunction(fn=splitByString, minArgs=2, maxArgs=3), + "generateUUIDv4": STLFunction(fn=generateUUIDv4, minArgs=0, maxArgs=0), + "sha256Hex": STLFunction(fn=lambda args, team, stdout, timeout: sha256Hex(args[0]), minArgs=1, maxArgs=1), + "md5Hex": STLFunction(fn=lambda args, team, stdout, timeout: md5Hex(args[0]), minArgs=1, maxArgs=1), + "sha256HmacChainHex": STLFunction( + fn=lambda args, team, stdout, timeout: sha256HmacChainHex(args[0]), minArgs=1, maxArgs=1 + ), + "keys": STLFunction(fn=keys, minArgs=1, maxArgs=1), + "values": STLFunction(fn=values, minArgs=1, maxArgs=1), + "arrayPushBack": STLFunction(fn=arrayPushBack, minArgs=2, maxArgs=2), + "arrayPushFront": STLFunction(fn=arrayPushFront, minArgs=2, maxArgs=2), + "arrayPopBack": STLFunction(fn=arrayPopBack, minArgs=1, maxArgs=1), + "arrayPopFront": STLFunction(fn=arrayPopFront, minArgs=1, maxArgs=1), + "arraySort": STLFunction(fn=arraySort, minArgs=1, maxArgs=1), + "arrayReverse": STLFunction(fn=arrayReverse, minArgs=1, maxArgs=1), + "arrayReverseSort": STLFunction(fn=arrayReverseSort, minArgs=1, maxArgs=1), + "arrayStringConcat": STLFunction(fn=arrayStringConcat, minArgs=1, maxArgs=2), + "has": STLFunction(fn=has, minArgs=2, maxArgs=2), + "now": STLFunction(fn=lambda args, team, stdout, timeout: now(), minArgs=0, maxArgs=0), + "toUnixTimestamp": STLFunction( + fn=lambda args, team, stdout, timeout: toUnixTimestamp(args[0], args[1] if len(args) > 1 else None), + minArgs=1, + maxArgs=2, + ), + "fromUnixTimestamp": STLFunction( + fn=lambda args, team, stdout, timeout: fromUnixTimestamp(args[0]), minArgs=1, maxArgs=1 + ), + "toUnixTimestampMilli": STLFunction( + fn=lambda args, team, stdout, timeout: toUnixTimestampMilli(args[0]), minArgs=1, maxArgs=2 + ), + "fromUnixTimestampMilli": STLFunction( + fn=lambda args, team, stdout, timeout: fromUnixTimestampMilli(args[0]), minArgs=1, maxArgs=1 + ), + "toTimeZone": STLFunction( + fn=lambda args, team, stdout, timeout: toTimeZone(args[0], args[1]), minArgs=2, maxArgs=2 + ), + "toDate": STLFunction(fn=lambda args, team, stdout, timeout: toDate(args[0]), minArgs=1, maxArgs=1), + "toDateTime": STLFunction(fn=lambda args, team, stdout, timeout: toDateTime(args[0]), minArgs=1, maxArgs=2), + "formatDateTime": STLFunction(fn=_formatDateTime, minArgs=2, maxArgs=3), + "HogError": STLFunction( + fn=lambda args, team, stdout, timeout: new_hog_error(args[0], args[1], args[2] if len(args) > 2 else None), + minArgs=1, + maxArgs=3, + ), + "Error": STLFunction( + fn=lambda args, team, stdout, timeout: new_hog_error( + "Error", args[0] if len(args) > 0 else None, args[1] if len(args) > 1 else None + ), + minArgs=0, + maxArgs=2, + ), + "RetryError": STLFunction( + fn=lambda args, team, stdout, timeout: new_hog_error("RetryError", args[0], args[1] if len(args) > 1 else None), + minArgs=0, + maxArgs=2, + ), + "NotImplementedError": STLFunction( + fn=lambda args, team, stdout, timeout: new_hog_error( + "NotImplementedError", args[0], args[1] if len(args) > 1 else None + ), + minArgs=0, + maxArgs=2, + ), + # only in python, async function in nodejs + "sleep": STLFunction(fn=sleep, minArgs=1, maxArgs=1), + "run": STLFunction(fn=run, minArgs=1, maxArgs=1), } diff --git a/hogvm/python/stl/print.py b/hogvm/python/stl/print.py index 6c9ed0a93df20..bb6d4685086fd 100644 --- a/hogvm/python/stl/print.py +++ b/hogvm/python/stl/print.py @@ -1,6 +1,6 @@ import re -from hogvm.python.objects import is_hog_datetime, is_hog_date, is_hog_error +from hogvm.python.objects import is_hog_datetime, is_hog_date, is_hog_error, is_hog_closure, is_hog_callable # Copied from clickhouse_driver.util.escape, adapted only from single quotes to backquotes. escape_chars_map = { @@ -46,7 +46,10 @@ def print_hog_value(obj, marked: set | None = None): + (f", {print_hog_value(obj['payload'])}" if "payload" in obj and obj["payload"] is not None else "") + ")" ) - + if isinstance(obj, dict) and is_hog_closure(obj): + return print_hog_value(obj["callable"], marked) + if isinstance(obj, dict) and is_hog_callable(obj): + return f"fn<{escape_identifier(obj.get('name', 'lambda'))}({print_hog_value(obj['argCount'])})>" if isinstance(obj, list) or isinstance(obj, dict) or isinstance(obj, tuple): if id(obj) in marked: return "null" diff --git a/hogvm/python/test/test_execute.py b/hogvm/python/test/test_execute.py index 7289a1b585eff..627896cde38d0 100644 --- a/hogvm/python/test/test_execute.py +++ b/hogvm/python/test/test_execute.py @@ -4,7 +4,11 @@ from hogvm.python.execute import execute_bytecode, get_nested_value -from hogvm.python.operation import Operation as op, HOGQL_BYTECODE_IDENTIFIER as _H, HOGQL_BYTECODE_VERSION as VERSION +from hogvm.python.operation import ( + Operation as op, + HOGQL_BYTECODE_IDENTIFIER as _H, + HOGQL_BYTECODE_VERSION as VERSION, +) from hogvm.python.utils import UncaughtHogVMException from posthog.hogql.bytecode import create_bytecode from posthog.hogql.parser import parse_expr, parse_program @@ -497,10 +501,11 @@ def test_bytecode_functions(self): bytecode = create_bytecode(program) assert bytecode == [ "_H", - 1, - op.DECLARE_FN, + VERSION, + op.CALLABLE, "add", 2, + 0, 6, op.GET_LOCAL, 1, @@ -508,14 +513,18 @@ def test_bytecode_functions(self): 0, op.PLUS, op.RETURN, + op.CLOSURE, + 0, op.INTEGER, 3, op.INTEGER, 4, - op.CALL_GLOBAL, - "add", + op.GET_LOCAL, + 0, + op.CALL_LOCAL, 2, op.RETURN, + op.POP, ] response = execute_bytecode(bytecode).result diff --git a/hogvm/typescript/package.json b/hogvm/typescript/package.json index 90291d4afa369..84432be8075bb 100644 --- a/hogvm/typescript/package.json +++ b/hogvm/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@posthog/hogvm", - "version": "1.0.36", + "version": "1.0.38", "description": "PostHog Hog Virtual Machine", "types": "dist/index.d.ts", "main": "dist/index.js", diff --git a/hogvm/typescript/src/__tests__/execute.test.ts b/hogvm/typescript/src/__tests__/execute.test.ts index 9efe09b2dd9b7..7085dc529ae6e 100644 --- a/hogvm/typescript/src/__tests__/execute.test.ts +++ b/hogvm/typescript/src/__tests__/execute.test.ts @@ -544,13 +544,30 @@ describe('hogvm execute', () => { state: { bytecode, asyncSteps: 1, - callStack: [], + callStack: [ + { + ip: 8, + stackStart: 0, + argCount: 0, + closure: { + __hogClosure__: true, + callable: { + __hogCallable__: 'main', + name: '', + argCount: 0, + upvalueCount: 0, + ip: 1, + }, + upvalues: [], + }, + }, + ], throwStack: [], declaredFunctions: {}, - ip: 8, maxMemUsed: 16, ops: 3, stack: [4.2], + upvalues: [], syncDuration: expect.any(Number), }, }) @@ -572,10 +589,10 @@ describe('hogvm execute', () => { bytecode: [], callStack: [], declaredFunctions: {}, - ip: -1, maxMemUsed: 13, ops: 2, stack: [], + upvalues: [], throwStack: [], syncDuration: expect.any(Number), }, @@ -599,10 +616,10 @@ describe('hogvm execute', () => { bytecode: [], callStack: [], declaredFunctions: {}, - ip: -1, maxMemUsed: 13, ops: 3, stack: [], + upvalues: [], throwStack: [], syncDuration: expect.any(Number), }, @@ -1899,7 +1916,8 @@ describe('hogvm execute', () => { 'fetch', 1, ] - expect(exec(bytecode, { asyncFunctions: { fetch: async () => null } })).toEqual({ + const result = exec(bytecode, { asyncFunctions: { fetch: async () => null } }) + expect(result).toEqual({ asyncFunctionArgs: [{ key: 'value' }], // not a Map asyncFunctionName: 'fetch', finished: false, @@ -1907,14 +1925,493 @@ describe('hogvm execute', () => { state: { asyncSteps: 1, bytecode: bytecode, - callStack: [], + callStack: [ + { + ip: 12, + stackStart: 0, + argCount: 0, + closure: { + __hogClosure__: true, + callable: { + __hogCallable__: 'main', + name: '', + argCount: 0, + upvalueCount: 0, + ip: 1, + }, + upvalues: [], + }, + }, + ], declaredFunctions: {}, - ip: 12, maxMemUsed: 64, ops: 5, stack: [{ key: 'value' }], // is not a Map syncDuration: 0, throwStack: [], + upvalues: [], + }, + }) + }) + + test('can serialize/unserialize lambdas', () => { + // let x := 2 + // let l := (a, b) -> a + b + x + // sleep(2) + // x := 10 + // return l(4, 3) + const bytecode = [ + '_H', + 1, + 33, + 2, + 52, + 'lambda', + 2, + 1, + 9, + 55, + 0, + 36, + 1, + 36, + 0, + 6, + 6, + 38, + 53, + 1, + true, + 0, + 33, + 2, + 2, + 'sleep', + 1, + 35, + 33, + 10, + 37, + 0, + 33, + 4, + 33, + 3, + 36, + 1, + 54, + 2, + 38, + 35, + 57, + ] + const options = { + asyncFunctions: { + sleep: async (seconds: number) => new Promise((resolve) => setTimeout(resolve, seconds)), + }, + } + const result = exec(bytecode, options) + + expect(result).toEqual({ + result: undefined, + finished: false, + asyncFunctionName: 'sleep', + asyncFunctionArgs: [2], + state: { + bytecode, + stack: [ + 2, + { + __hogClosure__: true, + callable: { + __hogCallable__: 'local', + name: 'lambda', + argCount: 2, + upvalueCount: 1, + ip: 9, + }, + upvalues: [1], + }, + ], + upvalues: [ + { + __hogUpValue__: true, + location: 0, + id: 1, + closed: false, + value: null, + }, + ], + callStack: [ + { + ip: 27, + stackStart: 0, + argCount: 0, + closure: { + __hogClosure__: true, + callable: { + __hogCallable__: 'main', + name: '', + argCount: 0, + upvalueCount: 0, + ip: 1, + }, + upvalues: [], + }, + }, + ], + throwStack: [], + declaredFunctions: {}, + ops: 5, + asyncSteps: 1, + syncDuration: expect.any(Number), + maxMemUsed: 242, + }, + }) + result.state!.stack.push(null) + const result2 = exec(result.state!, options) + expect(result2).toEqual({ + result: 17, + finished: true, + state: { + bytecode: [], + stack: [], + upvalues: [], + callStack: [], + throwStack: [], + declaredFunctions: {}, + ops: 19, + asyncSteps: 1, + syncDuration: expect.any(Number), + maxMemUsed: 476, + }, + }) + }) + + test('can serialize/unserialize upvalues', () => { + // fn outer() { + // let x := 'outside' + // fn inner() { + // print(x) + // } + // + // return inner + // } + // + // let closure := outer() + // sleep(2) + // return closure() + const bytecode = [ + '_H', + 1, + 52, + 'outer', + 0, + 0, + 19, + 32, + 'outside', + 52, + 'inner', + 0, + 1, + 3, + 55, + 0, + 38, + 53, + 1, + true, + 0, + 36, + 1, + 38, + 35, + 57, + 53, + 0, + 36, + 0, + 54, + 0, + 33, + 2, + 2, + 'sleep', + 1, + 35, + 36, + 1, + 54, + 0, + 38, + 35, + 35, + ] + + const options = { + asyncFunctions: { + sleep: async (seconds: number) => new Promise((resolve) => setTimeout(resolve, seconds)), + }, + } + const result = exec(bytecode, options) + + expect(result).toEqual({ + finished: false, + asyncFunctionName: 'sleep', + asyncFunctionArgs: [2], + state: { + bytecode, + stack: [ + { + __hogClosure__: true, + callable: { + __hogCallable__: 'local', + name: 'outer', + argCount: 0, + upvalueCount: 0, + ip: 7, + }, + upvalues: [], + }, + { + __hogClosure__: true, + callable: { + __hogCallable__: 'local', + name: 'inner', + argCount: 0, + upvalueCount: 1, + ip: 14, + }, + upvalues: [1], + }, + ], + upvalues: [ + { + __hogUpValue__: true, + id: 1, + location: 1, + closed: true, + value: 'outside', + }, + ], + callStack: [ + { + ip: 37, + stackStart: 0, + argCount: 0, + closure: { + __hogClosure__: true, + callable: { + __hogCallable__: 'main', + name: '', + argCount: 0, + upvalueCount: 0, + ip: 1, + }, + upvalues: [], + }, + }, + ], + throwStack: [], + declaredFunctions: {}, + ops: 11, + asyncSteps: 1, + syncDuration: expect.any(Number), + maxMemUsed: 682, + }, + }) + result.state!.stack.push(null) + const result2 = exec(result.state!, options) + expect(result2).toEqual({ + result: 'outside', + finished: true, + state: { + bytecode: [], + stack: [], + upvalues: [], + callStack: [], + throwStack: [], + declaredFunctions: {}, + ops: 17, + asyncSteps: 1, + syncDuration: expect.any(Number), + maxMemUsed: 682, + }, + }) + }) + + test('can serialize/unserialize upvalues v2', () => { + // fn outer() { + // let x := 'outside' + // fn inner() { + // print(x) + // sleep(2) + // return x + // } + // return inner + // } + // + // let closure := outer() + // return closure() + const bytecode = [ + '_H', + 1, + 52, + 'outer', + 0, + 0, + 31, + 32, + 'outside', + 52, + 'inner', + 0, + 1, + 15, + 55, + 0, + 2, + 'print', + 1, + 35, + 33, + 2, + 2, + 'sleep', + 1, + 35, + 55, + 0, + 38, + 53, + 1, + true, + 0, + 36, + 1, + 38, + 35, + 57, + 53, + 0, + 36, + 0, + 54, + 0, + 36, + 1, + 54, + 0, + 38, + 35, + 35, + ] + + const options = { + asyncFunctions: { + sleep: async (seconds: number) => new Promise((resolve) => setTimeout(resolve, seconds)), + }, + } + const result = exec(bytecode, options) + + expect(result).toEqual({ + finished: false, + asyncFunctionName: 'sleep', + asyncFunctionArgs: [2], + state: { + bytecode: bytecode, + stack: [ + { + __hogClosure__: true, + callable: { + __hogCallable__: 'local', + name: 'outer', + argCount: 0, + upvalueCount: 0, + ip: 7, + }, + upvalues: [], + }, + { + __hogClosure__: true, + callable: { + __hogCallable__: 'local', + name: 'inner', + argCount: 0, + upvalueCount: 1, + ip: 14, + }, + upvalues: [1], + }, + ], + upvalues: [ + { + __hogUpValue__: true, + id: 1, + location: 1, + closed: true, + value: 'outside', + }, + ], + callStack: [ + { + ip: 48, + stackStart: 0, + argCount: 0, + closure: { + __hogClosure__: true, + callable: { + __hogCallable__: 'main', + name: '', + argCount: 0, + upvalueCount: 0, + ip: 1, + }, + upvalues: [], + }, + }, + { + ip: 25, + stackStart: 2, + argCount: 0, + closure: { + __hogClosure__: true, + callable: { + __hogCallable__: 'local', + name: 'inner', + argCount: 0, + upvalueCount: 1, + ip: 14, + }, + upvalues: [1], + }, + }, + ], + throwStack: [], + declaredFunctions: {}, + ops: 16, + asyncSteps: 1, + syncDuration: expect.any(Number), + maxMemUsed: 682, + }, + }) + result.state!.stack.push(null) + const result2 = exec(result.state!, options) + expect(result2).toEqual({ + result: 'outside', + finished: true, + state: { + bytecode: [], + stack: [], + upvalues: [], + callStack: [], + throwStack: [], + declaredFunctions: {}, + ops: 20, + asyncSteps: 1, + syncDuration: expect.any(Number), + maxMemUsed: 682, }, }) }) diff --git a/hogvm/typescript/src/execute.ts b/hogvm/typescript/src/execute.ts index 9f9bb5af6a422..cae8ebc4b9396 100644 --- a/hogvm/typescript/src/execute.ts +++ b/hogvm/typescript/src/execute.ts @@ -1,5 +1,16 @@ import RE2 from 're2' +import { + CallFrame, + HogUpValue, + isHogCallable, + isHogClosure, + isHogError, + isHogUpValue, + newHogCallable, + newHogClosure, + ThrowFrame, +} from './objects' import { Operation } from './operation' import { ASYNC_STL, STL } from './stl/stl' import { @@ -12,7 +23,6 @@ import { setNestedValue, UncaughtHogVMException, } from './utils' -import { isHogError } from './objects' const DEFAULT_MAX_ASYNC_STEPS = 100 const DEFAULT_MAX_MEMORY = 64 * 1024 * 1024 // 64 MB @@ -24,14 +34,14 @@ export interface VMState { bytecode: any[] /** Stack of the VM */ stack: any[] + /** Values hoisted from the stack */ + upvalues: HogUpValue[] /** Call stack of the VM */ - callStack: [number, number, number][] + callStack: CallFrame[] // [number, number, number][] /** Throw stack of the VM */ - throwStack: [number, number, number][] - /** Declared functions of the VM */ + throwStack: ThrowFrame[] + /** Declared functions of the VM (deprecated) */ declaredFunctions: Record - /** Instruction pointer of the VM */ - ip: number /** How many sync ops have been performed */ ops: number /** How many async steps have been taken */ @@ -81,13 +91,11 @@ export async function execAsync(bytecode: any[], options?: ExecOptions): Promise if (response.state && response.asyncFunctionName && response.asyncFunctionArgs) { vmState = response.state if (options?.asyncFunctions && response.asyncFunctionName in options.asyncFunctions) { - const result = await options?.asyncFunctions[response.asyncFunctionName]( - ...response.asyncFunctionArgs.map(convertHogToJS) - ) + const result = await options?.asyncFunctions[response.asyncFunctionName](...response.asyncFunctionArgs) vmState.stack.push(result) } else if (response.asyncFunctionName in ASYNC_STL) { - const result = await ASYNC_STL[response.asyncFunctionName]( - response.asyncFunctionArgs.map(convertHogToJS), + const result = await ASYNC_STL[response.asyncFunctionName].fn( + response.asyncFunctionArgs, response.asyncFunctionName, options?.timeout ?? DEFAULT_TIMEOUT_MS ) @@ -123,19 +131,44 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { const asyncSteps = vmState ? vmState.asyncSteps : 0 const syncDuration = vmState ? vmState.syncDuration : 0 + const sortedUpValues: HogUpValue[] = vmState + ? vmState.upvalues.map((v) => ({ ...v, value: convertJSToHog(v.value) })) + : [] + const upvaluesById: Record = {} + for (const upvalue of sortedUpValues) { + upvaluesById[upvalue.id] = upvalue + } const stack: any[] = vmState ? vmState.stack.map(convertJSToHog) : [] const memStack: number[] = stack.map((s) => calculateCost(s)) - const callStack: [number, number, number][] = vmState ? vmState.callStack : [] - const throwStack: [number, number, number][] = vmState ? vmState.throwStack : [] + const callStack: CallFrame[] = vmState + ? vmState.callStack.map((v) => ({ ...v, closure: convertJSToHog(v.closure) })) + : [] + const throwStack: ThrowFrame[] = vmState ? vmState.throwStack : [] const declaredFunctions: Record = vmState ? vmState.declaredFunctions : {} let memUsed = memStack.reduce((acc, val) => acc + val, 0) let maxMemUsed = Math.max(vmState ? vmState.maxMemUsed : 0, memUsed) const memLimit = options?.memoryLimit ?? DEFAULT_MAX_MEMORY - let ip = vmState ? vmState.ip : bytecode[0] === '_H' ? 2 : 1 let ops = vmState ? vmState.ops : 0 const timeout = options?.timeout ?? DEFAULT_TIMEOUT_MS const maxAsyncSteps = options?.maxAsyncSteps ?? DEFAULT_MAX_ASYNC_STEPS + if (callStack.length === 0) { + callStack.push({ + ip: bytecode[0] === '_H' ? 2 : 1, + stackStart: 0, + argCount: 0, + closure: newHogClosure( + newHogCallable('main', { + name: '', + argCount: 0, + upvalueCount: 0, + ip: 1, + }) + ), + } satisfies CallFrame) + } + let frame: CallFrame = callStack[callStack.length - 1] + function popStack(): any { if (stack.length === 0) { throw new HogVMException('Invalid HogQL bytecode, stack is empty') @@ -158,18 +191,27 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { memUsed -= memStack.splice(start, deleteCount).reduce((acc, val) => acc + val, 0) return stack.splice(start, deleteCount) } - - /** Keep start elements, return those removed */ - function stackKeepFirstElements(start: number): any[] { - memUsed -= memStack.splice(start).reduce((acc, val) => acc + val, 0) - return stack.splice(start) + function stackKeepFirstElements(count: number): any[] { + for (let i = sortedUpValues.length - 1; i >= 0; i--) { + if (sortedUpValues[i].location >= count) { + if (!sortedUpValues[i].closed) { + sortedUpValues[i].closed = true + sortedUpValues[i].value = stack[sortedUpValues[i].location] + } + } else { + // upvalues are sorted by location, so we can break early + break + } + } + memUsed -= memStack.splice(count).reduce((acc, val) => acc + val, 0) + return stack.splice(count) } function next(): any { - if (ip >= bytecode!.length - 1) { + if (frame.ip >= bytecode!.length - 1) { throw new HogVMException('Unexpected end of bytecode') } - return bytecode![++ip] + return bytecode![++frame.ip] } function checkTimeout(): void { @@ -181,23 +223,44 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { return { bytecode: [], stack: [], + upvalues: [], callStack: [], throwStack: [], declaredFunctions: {}, - ip: -1, ops, asyncSteps, syncDuration: syncDuration + (Date.now() - startTime), maxMemUsed, } } + function captureUpValue(index: number): HogUpValue { + for (let i = sortedUpValues.length - 1; i >= 0; i--) { + if (sortedUpValues[i].location < index) { + break + } + if (sortedUpValues[i].location === index) { + return sortedUpValues[i] + } + } + const createdUpValue = { + __hogUpValue__: true, + id: sortedUpValues.length + 1, // used to deduplicate post deserialization + location: index, + closed: false, + value: null, + } satisfies HogUpValue + upvaluesById[createdUpValue.id] = createdUpValue + sortedUpValues.push(createdUpValue) + sortedUpValues.sort((a, b) => a.location - b.location) + return createdUpValue + } - for (; ip < bytecode.length; ip++) { + while (frame.ip < bytecode.length) { ops += 1 if ((ops & 127) === 0) { checkTimeout() } - switch (bytecode[ip]) { + switch (bytecode[frame.ip]) { case null: break case Operation.STRING: @@ -312,8 +375,47 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { for (let i = 0; i < count; i++) { chain.push(popStack()) } + if (options?.globals && chain[0] in options.globals && Object.hasOwn(options.globals, chain[0])) { pushStack(convertJSToHog(getNestedValue(options.globals, chain))) + } else if ( + options?.asyncFunctions && + chain.length == 1 && + Object.hasOwn(options.asyncFunctions, chain[0]) && + options.asyncFunctions[chain[0]] + ) { + pushStack( + newHogClosure( + newHogCallable('async', { + name: chain[0], + argCount: 0, // TODO + upvalueCount: 0, + ip: -1, + }) + ) + ) + } else if (chain.length == 1 && chain[0] in ASYNC_STL && Object.hasOwn(ASYNC_STL, chain[0])) { + pushStack( + newHogClosure( + newHogCallable('async', { + name: chain[0], + argCount: ASYNC_STL[chain[0]].maxArgs ?? 0, + upvalueCount: 0, + ip: -1, + }) + ) + ) + } else if (chain.length == 1 && chain[0] in STL && Object.hasOwn(STL, chain[0])) { + pushStack( + newHogClosure( + newHogCallable('stl', { + name: chain[0], + argCount: STL[chain[0]].maxArgs ?? 0, + upvalueCount: 0, + ip: -1, + }) + ) + ) } else { throw new HogVMException(`Global variable not found: ${chain.join('.')}`) } @@ -322,27 +424,27 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { case Operation.POP: popStack() break - case Operation.RETURN: - if (callStack.length > 0) { - const [newIp, stackStart, _] = callStack.pop()! - const response = popStack() - stackKeepFirstElements(stackStart) - pushStack(response) - ip = newIp - break - } else { - return { - result: popStack(), - finished: true, - state: getFinishedState(), - } satisfies ExecResult + case Operation.CLOSE_UPVALUE: + stackKeepFirstElements(stack.length - 1) + break + case Operation.RETURN: { + const result = popStack() + const lastCallFrame = callStack.pop() + if (callStack.length === 0 || !lastCallFrame) { + return { result, finished: true, state: getFinishedState() } satisfies ExecResult } + const stackStart = lastCallFrame.stackStart + stackKeepFirstElements(stackStart) + pushStack(result) + frame = callStack[callStack.length - 1] + continue // resume the loop without incrementing frame.ip + } case Operation.GET_LOCAL: - temp = callStack.length > 0 ? callStack[callStack.length - 1][1] : 0 + temp = callStack.length > 0 ? callStack[callStack.length - 1].stackStart : 0 pushStack(stack[next() + temp]) break case Operation.SET_LOCAL: - temp = (callStack.length > 0 ? callStack[callStack.length - 1][1] : 0) + next() + temp = (callStack.length > 0 ? callStack[callStack.length - 1].stackStart : 0) + next() stack[temp] = popStack() temp2 = memStack[temp] memStack[temp] = calculateCost(stack[temp]) @@ -384,38 +486,129 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { break case Operation.JUMP: temp = next() - ip += temp + frame.ip += temp break case Operation.JUMP_IF_FALSE: temp = next() if (!popStack()) { - ip += temp + frame.ip += temp } break case Operation.JUMP_IF_STACK_NOT_NULL: temp = next() if (stack.length > 0 && stack[stack.length - 1] !== null) { - ip += temp + frame.ip += temp } break case Operation.DECLARE_FN: { + // DEPRECATED + const name = next() + const argCount = next() + const bodyLength = next() + declaredFunctions[name] = [frame.ip + 1, argCount] + frame.ip += bodyLength + break + } + case Operation.CALLABLE: { const name = next() const argCount = next() + const upvalueCount = next() const bodyLength = next() - declaredFunctions[name] = [ip, argCount] - ip += bodyLength + const callable = newHogCallable('local', { + name, + argCount, + upvalueCount, + ip: frame.ip + 1, + }) + pushStack(callable) + frame.ip += bodyLength + break + } + case Operation.CLOSURE: { + const callable = popStack() + if (!isHogCallable(callable)) { + throw new HogVMException(`Invalid callable: ${JSON.stringify(callable)}`) + } + const upvalueCount = next() + const closureUpValues: number[] = [] + if (upvalueCount !== callable.upvalueCount) { + throw new HogVMException( + `Invalid upvalue count. Expected ${callable.upvalueCount}, got ${upvalueCount}` + ) + } + const stackStart = frame.stackStart + for (let i = 0; i < callable.upvalueCount; i++) { + const [isLocal, index] = [next(), next()] + if (isLocal) { + closureUpValues.push(captureUpValue(stackStart + index).id) + } else { + closureUpValues.push(frame.closure.upvalues[index]) + } + } + pushStack(newHogClosure(callable, closureUpValues)) + break + } + case Operation.GET_UPVALUE: { + const index = next() + if (index >= frame.closure.upvalues.length) { + throw new HogVMException(`Invalid upvalue index: ${index}`) + } + const upvalue = upvaluesById[frame.closure.upvalues[index]] + if (!isHogUpValue(upvalue)) { + throw new HogVMException(`Invalid upvalue: ${upvalue}`) + } + if (upvalue.closed) { + pushStack(upvalue.value) + } else { + pushStack(stack[upvalue.location]) + } + break + } + case Operation.SET_UPVALUE: { + const index = next() + if (index >= frame.closure.upvalues.length) { + throw new HogVMException(`Invalid upvalue index: ${index}`) + } + const upvalue = upvaluesById[frame.closure.upvalues[index]] + if (!isHogUpValue(upvalue)) { + throw new HogVMException(`Invalid upvalue: ${upvalue}`) + } + if (upvalue.closed) { + upvalue.value = popStack() + } else { + stack[upvalue.location] = popStack() + } break } case Operation.CALL_GLOBAL: { checkTimeout() const name = next() - // excluding "toString" only because of JavaScript --> no, it's not declared, it's omnipresent! o_O + temp = next() // args.length if (name in declaredFunctions && name !== 'toString') { + // This is for backwards compatibility. We use a closure on the stack with local functions now. const [funcIp, argLen] = declaredFunctions[name] - callStack.push([ip + 1, stack.length - argLen, argLen]) - ip = funcIp + frame.ip += 1 // advance for when we return + if (argLen > temp) { + for (let i = temp; i < argLen; i++) { + pushStack(null) + } + } + frame = { + ip: funcIp, + stackStart: stack.length - argLen, + argCount: argLen, + closure: newHogClosure( + newHogCallable('stl', { + name: name, + argCount: argLen, + upvalueCount: 0, + ip: -1, + }) + ), + } satisfies CallFrame + callStack.push(frame) + continue // resume the loop without incrementing frame.ip } else { - temp = next() // args.length if (temp > stack.length) { throw new HogVMException('Not enough arguments on the stack') } @@ -443,6 +636,8 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { throw new HogVMException(`Exceeded maximum number of async steps: ${maxAsyncSteps}`) } + frame.ip += 1 // resume at the next address after async returns + return { result: undefined, finished: false, @@ -451,10 +646,13 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { state: { bytecode, stack: stack.map(convertHogToJS), - callStack, + upvalues: sortedUpValues, + callStack: callStack.map((v) => ({ + ...v, + closure: convertHogToJS(v.closure), + })), throwStack, declaredFunctions, - ip: ip + 1, ops, asyncSteps: asyncSteps + 1, syncDuration: syncDuration + (Date.now() - startTime), @@ -462,15 +660,114 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { }, } satisfies ExecResult } else if (name in STL) { - pushStack(STL[name](args, name, timeout)) + pushStack(STL[name].fn(args, name, timeout)) } else { throw new HogVMException(`Unsupported function call: ${name}`) } } break } + case Operation.CALL_LOCAL: { + checkTimeout() + const closure = popStack() + if (!isHogClosure(closure)) { + throw new HogVMException(`Invalid closure: ${JSON.stringify(closure)}`) + } + if (!isHogCallable(closure.callable)) { + throw new HogVMException(`Invalid callable: ${JSON.stringify(closure.callable)}`) + } + temp = next() // args.length + if (temp > stack.length) { + throw new HogVMException('Not enough arguments on the stack') + } + if (temp > MAX_FUNCTION_ARGS_LENGTH) { + throw new HogVMException('Too many arguments') + } + if (closure.callable.__hogCallable__ === 'local') { + if (closure.callable.argCount > temp) { + for (let i = temp; i < closure.callable.argCount; i++) { + pushStack(null) + } + } else if (closure.callable.argCount < temp) { + throw new HogVMException( + `Too many arguments. Passed ${temp}, expected ${closure.callable.argCount}` + ) + } + frame.ip += 1 // advance for when we return + frame = { + ip: closure.callable.ip, + stackStart: stack.length - closure.callable.argCount, + argCount: closure.callable.argCount, + closure, + } satisfies CallFrame + callStack.push(frame) + continue // resume the loop without incrementing frame.ip + } else if (closure.callable.__hogCallable__ === 'stl') { + if (!closure.callable.name || !(closure.callable.name in STL)) { + throw new HogVMException(`Unsupported function call: ${closure.callable.name}`) + } + const stlFn = STL[closure.callable.name] + if (stlFn.minArgs !== undefined && temp < stlFn.minArgs) { + throw new HogVMException( + `Function ${closure.callable.name} requires at least ${stlFn.minArgs} arguments` + ) + } + if (stlFn.maxArgs !== undefined && temp > stlFn.maxArgs) { + throw new HogVMException( + `Function ${closure.callable.name} requires at most ${stlFn.maxArgs} arguments` + ) + } + const args = Array(temp) + .fill(null) + .map(() => popStack()) + if (version > 0) { + args.reverse() + } + if (stlFn.maxArgs !== undefined && args.length < stlFn.maxArgs) { + for (let i = args.length; i < stlFn.maxArgs; i++) { + args.push(null) + } + } + pushStack(stlFn.fn(args, closure.callable.name, timeout)) + } else if (closure.callable.__hogCallable__ === 'async') { + if (asyncSteps >= maxAsyncSteps) { + throw new HogVMException(`Exceeded maximum number of async steps: ${maxAsyncSteps}`) + } + const args = Array(temp) + .fill(null) + .map(() => popStack()) + return { + result: undefined, + finished: false, + asyncFunctionName: closure.callable.name, + asyncFunctionArgs: args.map(convertHogToJS), + state: { + bytecode, + stack: stack.map(convertHogToJS), + upvalues: sortedUpValues, + callStack: callStack.map((v) => ({ + ...v, + closure: convertHogToJS(v.closure), + })), + throwStack, + declaredFunctions, + ops, + asyncSteps: asyncSteps + 1, + syncDuration: syncDuration + (Date.now() - startTime), + maxMemUsed, + }, + } satisfies ExecResult + } else { + throw new HogVMException(`Unsupported function call: ${closure.callable.name}`) + } + break + } case Operation.TRY: - throwStack.push([callStack.length, stack.length, ip + next()]) + throwStack.push({ + callStackLen: callStack.length, + stackLen: stack.length, + catchIp: frame.ip + 1 + next(), + }) break case Operation.POP_TRY: if (throwStack.length > 0) { @@ -485,20 +782,24 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { throw new HogVMException('Can not throw: value is not of type Error') } if (throwStack.length > 0) { - const [callStackLen, stackLen, catchIp] = throwStack.pop()! + const { callStackLen, stackLen, catchIp } = throwStack.pop()! stackKeepFirstElements(stackLen) memUsed -= memStack.splice(stackLen).reduce((acc, val) => acc + val, 0) callStack.splice(callStackLen) pushStack(exception) - ip = catchIp + frame = callStack[callStack.length - 1] + frame.ip = catchIp + continue // resume the loop without incrementing frame.ip } else { throw new UncaughtHogVMException(exception.type, exception.message, exception.payload) } - break } default: - throw new HogVMException(`Unexpected node while running bytecode: ${bytecode[ip]}`) + throw new HogVMException(`Unexpected node while running bytecode: ${bytecode[frame.ip]}`) } + + // use "continue" to skip incrementing frame.ip each iteration + frame.ip++ } if (stack.length > 1) { diff --git a/hogvm/typescript/src/objects.ts b/hogvm/typescript/src/objects.ts index e9d4df023bfd9..3efadc93d6f95 100644 --- a/hogvm/typescript/src/objects.ts +++ b/hogvm/typescript/src/objects.ts @@ -1,3 +1,16 @@ +export interface CallFrame { + closure: HogClosure + ip: number + stackStart: number + argCount: number +} + +export interface ThrowFrame { + callStackLen: number + stackLen: number + catchIp: number +} + export interface HogDate { __hogDate__: true year: number @@ -19,6 +32,28 @@ export interface HogError { payload?: Record } +export interface HogCallable { + __hogCallable__: 'local' | 'stl' | 'async' | 'main' + name?: string + argCount: number + upvalueCount: number + ip: number +} + +export interface HogUpValue { + __hogUpValue__: true + id: number + location: number + closed: boolean + value: any +} + +export interface HogClosure { + __hogClosure__: true + callable: HogCallable + upvalues: number[] +} + export function isHogDate(obj: any): obj is HogDate { return obj && typeof obj === 'object' && '__hogDate__' in obj && 'year' in obj && 'month' in obj && 'day' in obj } @@ -39,3 +74,60 @@ export function newHogError(type: string, message: string, payload?: Record = { '\b': '\\b', @@ -72,6 +72,12 @@ export function printHogValue(obj: any, marked: Set | undefined = undefined obj.payload ? `, ${printHogValue(obj.payload, marked)}` : '' })` } + if (isHogClosure(obj)) { + return printHogValue(obj.callable, marked) + } + if (isHogCallable(obj)) { + return `fn<${escapeIdentifier(obj.name ?? 'lambda')}(${printHogValue(obj.argCount)})>` + } if (obj instanceof Map) { return `{${Array.from(obj.entries()) .map(([key, value]) => `${printHogValue(key, marked)}: ${printHogValue(value, marked)}`) diff --git a/hogvm/typescript/src/stl/stl.ts b/hogvm/typescript/src/stl/stl.ts index e936d002498b9..4267d7be21b20 100644 --- a/hogvm/typescript/src/stl/stl.ts +++ b/hogvm/typescript/src/stl/stl.ts @@ -30,344 +30,583 @@ function STLToString(args: any[]): string { return printHogStringOutput(args[0]) } -export const STL: Record any> = { - concat: (args) => { - return args.map((arg: any) => (arg === null ? '' : STLToString([arg]))).join('') - }, - match: (args) => { - const regex = new RegExp(args[1]) - return regex.test(args[0]) - }, - like: ([str, pattern]) => like(str, pattern, false), - ilike: ([str, pattern]) => like(str, pattern, true), - notLike: ([str, pattern]) => !like(str, pattern, false), - notILike: ([str, pattern]) => !like(str, pattern, true), - toString: STLToString, - toUUID: (args) => { - return String(args[0]) - }, - toInt: (args) => { - if (isHogDateTime(args[0])) { - return Math.floor(args[0].dt) - } else if (isHogDate(args[0])) { - const day = DateTime.fromObject({ year: args[0].year, month: args[0].month, day: args[0].day }) - const epoch = DateTime.fromObject({ year: 1970, month: 1, day: 1 }) - return Math.floor(day.diff(epoch, 'days').days) - } - return !isNaN(parseInt(args[0])) ? parseInt(args[0]) : null - }, - toFloat: (args) => { - if (isHogDateTime(args[0])) { - return args[0].dt - } else if (isHogDate(args[0])) { - const day = DateTime.fromObject({ year: args[0].year, month: args[0].month, day: args[0].day }) - const epoch = DateTime.fromObject({ year: 1970, month: 1, day: 1 }) - return Math.floor(day.diff(epoch, 'days').days) - } - return !isNaN(parseFloat(args[0])) ? parseFloat(args[0]) : null +export interface STLFunction { + fn: (args: any[], name: string, timeout: number) => any + minArgs?: number + maxArgs?: number +} + +export interface AsyncSTLFunction { + fn: (args: any[], name: string, timeout: number) => Promise + minArgs?: number + maxArgs?: number +} + +export const STL: Record = { + concat: { + fn: (args) => { + return args.map((arg: any) => (arg === null ? '' : STLToString([arg]))).join('') + }, + minArgs: 1, + maxArgs: undefined, + }, + match: { + fn: (args) => { + const regex = new RegExp(args[1]) + return regex.test(args[0]) + }, + minArgs: 2, + maxArgs: 2, + }, + like: { fn: ([str, pattern]) => like(str, pattern, false), minArgs: 2, maxArgs: 2 }, + ilike: { fn: ([str, pattern]) => like(str, pattern, true), minArgs: 2, maxArgs: 2 }, + notLike: { fn: ([str, pattern]) => !like(str, pattern, false), minArgs: 2, maxArgs: 2 }, + notILike: { fn: ([str, pattern]) => !like(str, pattern, true), minArgs: 2, maxArgs: 2 }, + toString: { fn: STLToString, minArgs: 1, maxArgs: 1 }, + toUUID: { + fn: (args) => { + return String(args[0]) + }, + minArgs: 1, + maxArgs: 1, + }, + toInt: { + fn: (args) => { + if (isHogDateTime(args[0])) { + return Math.floor(args[0].dt) + } else if (isHogDate(args[0])) { + const day = DateTime.fromObject({ year: args[0].year, month: args[0].month, day: args[0].day }) + const epoch = DateTime.fromObject({ year: 1970, month: 1, day: 1 }) + return Math.floor(day.diff(epoch, 'days').days) + } + return !isNaN(parseInt(args[0])) ? parseInt(args[0]) : null + }, + minArgs: 1, + maxArgs: 1, + }, + toFloat: { + fn: (args) => { + if (isHogDateTime(args[0])) { + return args[0].dt + } else if (isHogDate(args[0])) { + const day = DateTime.fromObject({ year: args[0].year, month: args[0].month, day: args[0].day }) + const epoch = DateTime.fromObject({ year: 1970, month: 1, day: 1 }) + return Math.floor(day.diff(epoch, 'days').days) + } + return !isNaN(parseFloat(args[0])) ? parseFloat(args[0]) : null + }, + minArgs: 1, + maxArgs: 1, }, // ifNull is complied into JUMP instructions. Keeping the function here for backwards compatibility - ifNull: (args) => { - return args[0] !== null ? args[0] : args[1] - }, - length: (args) => { - return args[0].length - }, - empty: (args) => { - if (typeof args[0] === 'object') { - if (Array.isArray(args[0])) { - return args[0].length === 0 - } else if (args[0] === null) { - return true - } else if (args[0] instanceof Map) { - return args[0].size === 0 - } - return Object.keys(args[0]).length === 0 - } - return !args[0] - }, - notEmpty: (args) => { - return !STL.empty(args, 'empty', 0) - }, - tuple: (args) => { - const tuple = args.slice() - ;(tuple as any).__isHogTuple = true - return tuple - }, - lower: (args) => { - return args[0].toLowerCase() - }, - upper: (args) => { - return args[0].toUpperCase() - }, - reverse: (args) => { - return args[0].split('').reverse().join('') - }, - print: (args) => { - // eslint-disable-next-line no-console - console.log(...args.map(printHogStringOutput)) - }, - jsonParse: (args) => { - // Recursively convert objects to maps - function convert(x: any): any { - if (Array.isArray(x)) { - return x.map(convert) - } else if (typeof x === 'object' && x !== null) { - // DateTime and other objects will be sanitized and not converted to a map - if (x.__hogDateTime__) { - return toHogDateTime(x.dt, x.zone) - } else if (x.__hogDate__) { - return toHogDate(x.year, x.month, x.day) - } else if (x.__hogError__) { - return newHogError(x.type, x.message, x.payload) - } - // All other objects will - const map = new Map() - for (const key in x) { - map.set(key, convert(x[key])) + ifNull: { + fn: (args) => { + return args[0] !== null ? args[0] : args[1] + }, + minArgs: 2, + maxArgs: 2, + }, + length: { + fn: (args) => { + return args[0].length + }, + minArgs: 1, + maxArgs: 1, + }, + empty: { + fn: (args) => { + if (typeof args[0] === 'object') { + if (Array.isArray(args[0])) { + return args[0].length === 0 + } else if (args[0] === null) { + return true + } else if (args[0] instanceof Map) { + return args[0].size === 0 } - return map + return Object.keys(args[0]).length === 0 } - return x - } - return convert(JSON.parse(args[0])) - }, - jsonStringify: (args) => { - // Recursively convert maps to objects - function convert(x: any, marked?: Set): any { - if (!marked) { - marked = new Set() + return !args[0] + }, + minArgs: 1, + maxArgs: 1, + }, + notEmpty: { + fn: (args) => { + return !STL.empty.fn(args, 'empty', 0) + }, + minArgs: 1, + maxArgs: 1, + }, + tuple: { + fn: (args) => { + const tuple = args.slice() + ;(tuple as any).__isHogTuple = true + return tuple + }, + minArgs: 0, + maxArgs: undefined, + }, + lower: { + fn: (args) => { + return args[0].toLowerCase() + }, + minArgs: 1, + maxArgs: 1, + }, + upper: { + fn: (args) => { + return args[0].toUpperCase() + }, + minArgs: 1, + maxArgs: 1, + }, + reverse: { + fn: (args) => { + return args[0].split('').reverse().join('') + }, + minArgs: 1, + maxArgs: 1, + }, + print: { + fn: (args) => { + // eslint-disable-next-line no-console + console.log(...args.map(printHogStringOutput)) + }, + minArgs: 0, + maxArgs: undefined, + }, + jsonParse: { + fn: (args) => { + // Recursively convert objects to maps + function convert(x: any): any { + if (Array.isArray(x)) { + return x.map(convert) + } else if (typeof x === 'object' && x !== null) { + // DateTime and other objects will be sanitized and not converted to a map + if (x.__hogDateTime__) { + return toHogDateTime(x.dt, x.zone) + } else if (x.__hogDate__) { + return toHogDate(x.year, x.month, x.day) + } else if (x.__hogError__) { + return newHogError(x.type, x.message, x.payload) + } + // All other objects will + const map = new Map() + for (const key in x) { + map.set(key, convert(x[key])) + } + return map + } + return x } - if (typeof x === 'object' && x !== null) { - if (marked.has(x)) { - return null + return convert(JSON.parse(args[0])) + }, + minArgs: 1, + maxArgs: 1, + }, + jsonStringify: { + fn: (args) => { + // Recursively convert maps to objects + function convert(x: any, marked?: Set): any { + if (!marked) { + marked = new Set() } - marked.add(x) - try { - if (x instanceof Map) { + if (typeof x === 'object' && x !== null) { + if (marked.has(x)) { + return null + } + marked.add(x) + try { + if (x instanceof Map) { + const obj: Record = {} + x.forEach((value, key) => { + obj[convert(key, marked)] = convert(value, marked) + }) + return obj + } + if (Array.isArray(x)) { + return x.map((v) => convert(v, marked)) + } + if (isHogDateTime(x) || isHogDate(x) || isHogError(x)) { + return x + } const obj: Record = {} - x.forEach((value, key) => { - obj[convert(key, marked)] = convert(value, marked) - }) + for (const key in x) { + obj[key] = convert(x[key], marked) + } return obj + } finally { + marked.delete(x) } - if (Array.isArray(x)) { - return x.map((v) => convert(v, marked)) - } - if (isHogDateTime(x) || isHogDate(x) || isHogError(x)) { - return x - } - const obj: Record = {} - for (const key in x) { - obj[key] = convert(x[key], marked) - } - return obj - } finally { - marked.delete(x) } + return x } - return x - } - if (args[1] && typeof args[1] === 'number' && args[1] > 0) { - return JSON.stringify(convert(args[0]), null, args[1]) - } - return JSON.stringify(convert(args[0])) - }, - base64Encode: (args) => { - return Buffer.from(args[0]).toString('base64') - }, - base64Decode: (args) => { - return Buffer.from(args[0], 'base64').toString() - }, - tryBase64Decode: (args) => { - try { + if (args[1] && typeof args[1] === 'number' && args[1] > 0) { + return JSON.stringify(convert(args[0]), null, args[1]) + } + return JSON.stringify(convert(args[0])) + }, + minArgs: 1, + maxArgs: 1, + }, + base64Encode: { + fn: (args) => { + return Buffer.from(args[0]).toString('base64') + }, + minArgs: 1, + maxArgs: 1, + }, + base64Decode: { + fn: (args) => { return Buffer.from(args[0], 'base64').toString() - } catch (e) { - return '' - } - }, - encodeURLComponent(args) { - return encodeURIComponent(args[0]) - }, - decodeURLComponent(args) { - return decodeURIComponent(args[0]) - }, - replaceOne(args) { - return args[0].replace(args[1], args[2]) - }, - replaceAll(args) { - return args[0].replaceAll(args[1], args[2]) - }, - trim([str, char = ' ']) { - if (char.length !== 1) { - return '' - } - let start = 0 - while (str[start] === char) { - start++ - } - let end = str.length - while (str[end - 1] === char) { - end-- - } - if (start >= end) { - return '' - } - return str.slice(start, end) - }, - trimLeft([str, char = ' ']) { - if (char.length !== 1) { - return '' - } - let start = 0 - while (str[start] === char) { - start++ - } - return str.slice(start) - }, - trimRight([str, char = ' ']) { - if (char.length !== 1) { - return '' - } - let end = str.length - while (str[end - 1] === char) { - end-- - } - return str.slice(0, end) - }, - splitByString([separator, str, maxSplits = undefined]) { - if (maxSplits === undefined) { - return str.split(separator) - } - return str.split(separator, maxSplits) - }, - generateUUIDv4() { - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { - const r = (Math.random() * 16) | 0 - const v = c === 'x' ? r : (r & 0x3) | 0x8 - return v.toString(16) - }) - }, - sha256Hex([str]) { - return sha256Hex(str) - }, - md5Hex([str]) { - return md5Hex(str) - }, - sha256HmacChainHex([data]) { - return sha256HmacChainHex(data) - }, - keys([obj]) { - if (typeof obj === 'object') { - if (Array.isArray(obj)) { - return Array.from(obj.keys()) - } else if (obj instanceof Map) { - return Array.from(obj.keys()) + }, + minArgs: 1, + maxArgs: 1, + }, + tryBase64Decode: { + fn: (args) => { + try { + return Buffer.from(args[0], 'base64').toString() + } catch (e) { + return '' } - return Object.keys(obj) - } - return [] - }, - values([obj]) { - if (typeof obj === 'object') { - if (Array.isArray(obj)) { - return [...obj] - } else if (obj instanceof Map) { - return Array.from(obj.values()) + }, + minArgs: 1, + maxArgs: 1, + }, + encodeURLComponent: { + fn: (args) => encodeURIComponent(args[0]), + minArgs: 1, + maxArgs: 1, + }, + decodeURLComponent: { + fn: (args) => decodeURIComponent(args[0]), + minArgs: 1, + maxArgs: 1, + }, + replaceOne: { + fn: (args) => { + return args[0].replace(args[1], args[2]) + }, + minArgs: 3, + maxArgs: 3, + }, + replaceAll: { + fn: (args) => { + return args[0].replaceAll(args[1], args[2]) + }, + minArgs: 3, + maxArgs: 3, + }, + trim: { + fn: ([str, char]) => { + if (char === null || char === undefined) { + char = ' ' + } + if (char.length !== 1) { + return '' + } + let start = 0 + while (str[start] === char) { + start++ + } + let end = str.length + while (str[end - 1] === char) { + end-- + } + if (start >= end) { + return '' + } + return str.slice(start, end) + }, + minArgs: 1, + maxArgs: 2, + }, + trimLeft: { + fn: ([str, char]) => { + if (char === null || char === undefined) { + char = ' ' + } + if (char.length !== 1) { + return '' + } + let start = 0 + while (str[start] === char) { + start++ + } + return str.slice(start) + }, + minArgs: 1, + maxArgs: 2, + }, + trimRight: { + fn: ([str, char]) => { + if (char === null || char === undefined) { + char = ' ' + } + if (char.length !== 1) { + return '' + } + let end = str.length + while (str[end - 1] === char) { + end-- + } + return str.slice(0, end) + }, + minArgs: 1, + maxArgs: 2, + }, + splitByString: { + fn: ([separator, str, maxSplits = undefined]) => { + if (maxSplits === undefined || maxSplits === null) { + return str.split(separator) + } + return str.split(separator, maxSplits) + }, + minArgs: 2, + maxArgs: 3, + }, + generateUUIDv4: { + fn: () => { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { + const r = (Math.random() * 16) | 0 + const v = c === 'x' ? r : (r & 0x3) | 0x8 + return v.toString(16) + }) + }, + minArgs: 0, + maxArgs: 0, + }, + sha256Hex: { + fn: ([str]) => { + return sha256Hex(str) + }, + minArgs: 1, + maxArgs: 1, + }, + md5Hex: { + fn: ([str]) => { + return md5Hex(str) + }, + minArgs: 1, + maxArgs: 1, + }, + sha256HmacChainHex: { + fn: ([data]) => sha256HmacChainHex(data), + minArgs: 1, + maxArgs: 1, + }, + keys: { + fn: ([obj]) => { + if (typeof obj === 'object') { + if (Array.isArray(obj)) { + return Array.from(obj.keys()) + } else if (obj instanceof Map) { + return Array.from(obj.keys()) + } + return Object.keys(obj) } - return Object.values(obj) - } - return [] - }, - arrayPushBack([arr, item]) { - if (!Array.isArray(arr)) { - return [item] - } - return [...arr, item] - }, - arrayPushFront([arr, item]) { - if (!Array.isArray(arr)) { - return [item] - } - return [item, ...arr] - }, - arrayPopBack([arr]) { - if (!Array.isArray(arr)) { - return [] - } - return arr.slice(0, arr.length - 1) - }, - arrayPopFront([arr]) { - if (!Array.isArray(arr)) { - return [] - } - return arr.slice(1) - }, - arraySort([arr]) { - if (!Array.isArray(arr)) { - return [] - } - return [...arr].sort() - }, - arrayReverse([arr]) { - if (!Array.isArray(arr)) { return [] - } - return [...arr].reverse() - }, - arrayReverseSort([arr]) { - if (!Array.isArray(arr)) { + }, + minArgs: 1, + maxArgs: 1, + }, + values: { + fn: ([obj]) => { + if (typeof obj === 'object') { + if (Array.isArray(obj)) { + return [...obj] + } else if (obj instanceof Map) { + return Array.from(obj.values()) + } + return Object.values(obj) + } return [] - } - return [...arr].sort().reverse() - }, - arrayStringConcat([arr, separator = '']) { - if (!Array.isArray(arr)) { - return '' - } - return arr.join(separator) - }, - has([arr, elem]) { - if (!Array.isArray(arr) || arr.length === 0) { - return false - } - return arr.includes(elem) - }, - now() { - return now() - }, - toUnixTimestamp(args) { - return toUnixTimestamp(args[0], args[1]) - }, - fromUnixTimestamp(args) { - return fromUnixTimestamp(args[0]) - }, - toUnixTimestampMilli(args) { - return toUnixTimestampMilli(args[0], args[1]) - }, - fromUnixTimestampMilli(args) { - return fromUnixTimestampMilli(args[0]) - }, - toTimeZone(args) { - return toTimeZone(args[0], args[1]) - }, - toDate(args) { - return toDate(args[0]) - }, - toDateTime(args) { - return toDateTime(args[0], args[1]) - }, - formatDateTime(args) { - return formatDateTime(args[0], args[1], args[2]) + }, + minArgs: 1, + maxArgs: 1, + }, + arrayPushBack: { + fn: ([arr, item]) => { + if (!Array.isArray(arr)) { + return [item] + } + return [...arr, item] + }, + minArgs: 2, + maxArgs: 2, + }, + arrayPushFront: { + fn: ([arr, item]) => { + if (!Array.isArray(arr)) { + return [item] + } + return [item, ...arr] + }, + minArgs: 2, + maxArgs: 2, + }, + arrayPopBack: { + fn: ([arr]) => { + if (!Array.isArray(arr)) { + return [] + } + return arr.slice(0, arr.length - 1) + }, + minArgs: 1, + maxArgs: 1, + }, + arrayPopFront: { + fn: ([arr]) => { + if (!Array.isArray(arr)) { + return [] + } + return arr.slice(1) + }, + minArgs: 1, + maxArgs: 1, + }, + arraySort: { + fn: ([arr]) => { + if (!Array.isArray(arr)) { + return [] + } + return [...arr].sort() + }, + minArgs: 1, + maxArgs: 1, + }, + arrayReverse: { + fn: ([arr]) => { + if (!Array.isArray(arr)) { + return [] + } + return [...arr].reverse() + }, + minArgs: 1, + maxArgs: 1, + }, + arrayReverseSort: { + fn: ([arr]) => { + if (!Array.isArray(arr)) { + return [] + } + return [...arr].sort().reverse() + }, + minArgs: 1, + maxArgs: 1, + }, + arrayStringConcat: { + fn: ([arr, separator = '']) => { + if (!Array.isArray(arr)) { + return '' + } + return arr.join(separator) + }, + minArgs: 1, + maxArgs: 2, + }, + has: { + fn: ([arr, elem]) => { + if (!Array.isArray(arr) || arr.length === 0) { + return false + } + return arr.includes(elem) + }, + minArgs: 2, + maxArgs: 2, + }, + now: { + fn: () => { + return now() + }, + minArgs: 0, + maxArgs: 0, + }, + toUnixTimestamp: { + fn: (args) => { + return toUnixTimestamp(args[0], args[1]) + }, + minArgs: 1, + maxArgs: 2, + }, + fromUnixTimestamp: { + fn: (args) => { + return fromUnixTimestamp(args[0]) + }, + minArgs: 1, + maxArgs: 1, + }, + toUnixTimestampMilli: { + fn: (args) => { + return toUnixTimestampMilli(args[0], args[1]) + }, + minArgs: 1, + maxArgs: 2, + }, + fromUnixTimestampMilli: { + fn: (args) => { + return fromUnixTimestampMilli(args[0]) + }, + minArgs: 1, + maxArgs: 1, + }, + toTimeZone: { + fn: (args) => { + return toTimeZone(args[0], args[1]) + }, + minArgs: 2, + maxArgs: 2, + }, + toDate: { + fn: (args) => { + return toDate(args[0]) + }, + minArgs: 1, + maxArgs: 1, + }, + toDateTime: { + fn: (args) => { + return toDateTime(args[0], args[1]) + }, + minArgs: 1, + maxArgs: 2, + }, + formatDateTime: { + fn: (args) => { + return formatDateTime(args[0], args[1], args[2]) + }, + minArgs: 2, + maxArgs: 3, + }, + HogError: { + fn: (args) => newHogError(args[0], args[1], args[2]), + minArgs: 1, + maxArgs: 3, + }, + Error: { + fn: (args, name) => newHogError(name, args[0], args[1]), + minArgs: 0, + maxArgs: 2, + }, + RetryError: { + fn: (args, name) => newHogError(name, args[0], args[1]), + minArgs: 0, + maxArgs: 2, + }, + NotImplementedError: { + fn: (args, name) => newHogError(name, args[0], args[1]), + minArgs: 0, + maxArgs: 2, }, - HogError: (args) => newHogError(args[0], args[1], args[2]), - Error: (args, name) => newHogError(name, args[0], args[1]), - RetryError: (args, name) => newHogError(name, args[0], args[1]), - NotImplementedError: (args, name) => newHogError(name, args[0], args[1]), } -export const ASYNC_STL: Record Promise> = { - sleep: async (args) => { - await new Promise((resolve) => setTimeout(resolve, args[0] * 1000)) +export const ASYNC_STL: Record = { + sleep: { + fn: async (args) => { + await new Promise((resolve) => setTimeout(resolve, args[0] * 1000)) + }, + minArgs: 1, + maxArgs: 1, }, } diff --git a/hogvm/typescript/src/utils.ts b/hogvm/typescript/src/utils.ts index 13302d10c6749..c775307c83835 100644 --- a/hogvm/typescript/src/utils.ts +++ b/hogvm/typescript/src/utils.ts @@ -1,4 +1,5 @@ import { toHogDate, toHogDateTime } from './stl/date' +import { HogUpValue } from './objects' export class HogVMException extends Error { constructor(message: string) { @@ -93,6 +94,8 @@ export function convertJSToHog(x: any): any { return toHogDateTime(x.dt, x.zone) } else if (x.__hogDate__) { return toHogDate(x.year, x.month, x.day) + } else if (x.__hogClosure__ || x.__hogCallable__) { + return x } const map = new Map() for (const key in x) { @@ -113,7 +116,7 @@ export function convertHogToJS(x: any): any { } else if (typeof x === 'object' && Array.isArray(x)) { return x.map(convertHogToJS) } else if (typeof x === 'object' && x !== null) { - if (x.__hogDateTime__ || x.__hogDate__) { + if (x.__hogDateTime__ || x.__hogDate__ || x.__hogClosure__ || x.__hogCallable__) { return x } const obj: Record = {} diff --git a/plugin-server/package.json b/plugin-server/package.json index ea7467fb42993..0dc034a771828 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -52,7 +52,7 @@ "@google-cloud/storage": "^5.8.5", "@maxmind/geoip2-node": "^3.4.0", "@posthog/clickhouse": "^1.7.0", - "@posthog/hogvm": "^1.0.36", + "@posthog/hogvm": "^1.0.38", "@posthog/plugin-scaffold": "1.4.4", "@sentry/node": "^7.49.0", "@sentry/profiling-node": "^0.3.0", diff --git a/plugin-server/pnpm-lock.yaml b/plugin-server/pnpm-lock.yaml index faedff7194010..6d7d4278ec03a 100644 --- a/plugin-server/pnpm-lock.yaml +++ b/plugin-server/pnpm-lock.yaml @@ -47,8 +47,8 @@ dependencies: specifier: file:../rust/cyclotron-node version: file:../rust/cyclotron-node '@posthog/hogvm': - specifier: ^1.0.36 - version: 1.0.36(luxon@3.4.4)(re2@1.20.3) + specifier: ^1.0.38 + version: 1.0.38(luxon@3.4.4)(re2@1.20.3) '@posthog/plugin-scaffold': specifier: 1.4.4 version: 1.4.4 @@ -3116,8 +3116,8 @@ packages: engines: {node: '>=12'} dev: false - /@posthog/hogvm@1.0.36(luxon@3.4.4)(re2@1.20.3): - resolution: {integrity: sha512-O4mVlTCNYAOg3eh82r5YE0HqMf4b0em7JNHCD6tkUUMxe47rTAVflSW/vVOkV7ogfgyjxQD0bOS6w708FW/cMg==} + /@posthog/hogvm@1.0.38(luxon@3.4.4)(re2@1.20.3): + resolution: {integrity: sha512-UB0mFVUCG2CJC+bQX8rzvo/zG1Mu8oZcdrxRtasZetZfOffRVBvj6ESNPEsTNlh/hNV9u+aOvSppRomiFEvjlg==} peerDependencies: luxon: ^3.4.4 re2: ^1.21.3 diff --git a/plugin-server/tests/cdp/cdp-function-callbacks-consumer.test.ts b/plugin-server/tests/cdp/cdp-function-callbacks-consumer.test.ts index 33559d108ace9..7ef3cbe515287 100644 --- a/plugin-server/tests/cdp/cdp-function-callbacks-consumer.test.ts +++ b/plugin-server/tests/cdp/cdp-function-callbacks-consumer.test.ts @@ -169,7 +169,7 @@ describe('CDP Processed Events Consuner', () => { }, { level: 'debug', - message: "Suspending function due to async function call 'fetch'. Payload: 1331 bytes", + message: "Suspending function due to async function call 'fetch'. Payload: 1639 bytes", }, { level: 'info', @@ -217,7 +217,7 @@ describe('CDP Processed Events Consuner', () => { }, { level: 'debug', - message: "Suspending function due to async function call 'fetch'. Payload: 1331 bytes", + message: "Suspending function due to async function call 'fetch'. Payload: 1639 bytes", }, { level: 'debug', diff --git a/plugin-server/tests/cdp/cdp-processed-events-consumer.test.ts b/plugin-server/tests/cdp/cdp-processed-events-consumer.test.ts index 4de26bd05fca0..56a68bc485cd1 100644 --- a/plugin-server/tests/cdp/cdp-processed-events-consumer.test.ts +++ b/plugin-server/tests/cdp/cdp-processed-events-consumer.test.ts @@ -211,7 +211,7 @@ describe('CDP Processed Events Consuner', () => { topic: 'log_entries_test', value: { log_source: 'hog_function', - message: "Suspending function due to async function call 'fetch'. Payload: 1497 bytes", + message: "Suspending function due to async function call 'fetch'. Payload: 1805 bytes", team_id: 2, }, }) diff --git a/plugin-server/tests/cdp/hog-executor.test.ts b/plugin-server/tests/cdp/hog-executor.test.ts index e8ac3d365cd9a..3c679e5a52a5a 100644 --- a/plugin-server/tests/cdp/hog-executor.test.ts +++ b/plugin-server/tests/cdp/hog-executor.test.ts @@ -87,7 +87,7 @@ describe('Hog Executor', () => { { timestamp: expect.any(DateTime), level: 'debug', - message: "Suspending function due to async function call 'fetch'. Payload: 1456 bytes", + message: "Suspending function due to async function call 'fetch'. Payload: 1764 bytes", }, ]) }) @@ -199,7 +199,7 @@ describe('Hog Executor', () => { expect(asyncExecResult.finished).toBe(true) expect(logs.map((log) => log.message)).toEqual([ 'Executing function', - "Suspending function due to async function call 'fetch'. Payload: 1456 bytes", + "Suspending function due to async function call 'fetch'. Payload: 1764 bytes", 'Resuming function', 'Fetch response:, {"status":200,"body":"success"}', 'Function completed in 100ms. Sync: 0ms. Mem: 746 bytes. Ops: 22.', @@ -227,7 +227,7 @@ describe('Hog Executor', () => { expect(asyncExecResult.finished).toBe(true) expect(logs.map((log) => log.message)).toEqual([ 'Executing function', - "Suspending function due to async function call 'fetch'. Payload: 1456 bytes", + "Suspending function due to async function call 'fetch'. Payload: 1764 bytes", 'Resuming function', 'Fetch response:, {"status":200,"body":{"foo":"bar"}}', // The body is parsed 'Function completed in 100ms. Sync: 0ms. Mem: 746 bytes. Ops: 22.', diff --git a/posthog/hogql/bytecode.py b/posthog/hogql/bytecode.py index 1d71ec173d372..268a9dedc15c9 100644 --- a/posthog/hogql/bytecode.py +++ b/posthog/hogql/bytecode.py @@ -4,7 +4,7 @@ from collections.abc import Callable from hogvm.python.execute import execute_bytecode, BytecodeResult -from hogvm.python.stl import STL, MIN_ARGS_INCLUDING_OPTIONAL +from hogvm.python.stl import STL from posthog.hogql import ast from posthog.hogql.base import AST from posthog.hogql.context import HogQLContext @@ -62,12 +62,15 @@ def create_bytecode( supported_functions: Optional[set[str]] = None, args: Optional[list[str]] = None, context: Optional[HogQLContext] = None, + enclosing: Optional["BytecodeCompiler"] = None, ) -> list[Any]: + supported_functions = supported_functions or set() bytecode: list[Any] = [] if args is None: bytecode.append(HOGQL_BYTECODE_IDENTIFIER) bytecode.append(HOGQL_BYTECODE_VERSION) - bytecode.extend(BytecodeCompiler(supported_functions, args, context).visit(expr)) + + bytecode.extend(BytecodeCompiler(supported_functions, args, context, enclosing).visit(expr)) return bytecode @@ -75,6 +78,7 @@ def create_bytecode( class Local: name: str depth: int + is_captured: bool @dataclasses.dataclass @@ -84,17 +88,26 @@ class HogFunction: bytecode: list[Any] +class UpValue: + def __init__(self, index: int, is_local: bool): + self.index = index + self.is_local = is_local + + class BytecodeCompiler(Visitor): def __init__( self, supported_functions: Optional[set[str]] = None, args: Optional[list[str]] = None, context: Optional[HogQLContext] = None, + enclosing: Optional["BytecodeCompiler"] = None, ): super().__init__() + self.enclosing = enclosing self.supported_functions = supported_functions or set() self.locals: list[Local] = [] - self.functions: dict[str, HogFunction] = {} + self.upvalues: list[UpValue] = [] + # self.functions: dict[str, HogFunction] = {} self.scope_depth = 0 self.args = args # we're in a function definition @@ -113,7 +126,10 @@ def _end_scope(self) -> list[Any]: if local.depth <= self.scope_depth: break self.locals.pop() - response.append(Operation.POP) + if local.is_captured: + response.append(Operation.CLOSE_UPVALUE) + else: + response.append(Operation.POP) return response def _declare_local(self, name: str) -> int: @@ -123,7 +139,7 @@ def _declare_local(self, name: str) -> int: if local.name == name: raise QueryError(f"Variable `{name}` already declared in this scope") - self.locals.append(Local(name, self.scope_depth)) + self.locals.append(Local(name=name, depth=self.scope_depth, is_captured=False)) return len(self.locals) - 1 def visit_and(self, node: ast.And): @@ -158,19 +174,51 @@ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation): ARITHMETIC_OPERATIONS[node.op], ] + def _add_upvalue(self, index: int, is_local: bool) -> int: + for i, upvalue in enumerate(self.upvalues): + if upvalue.index == index and upvalue.is_local == is_local: + return i + self.upvalues.append(UpValue(index, is_local)) + return len(self.upvalues) - 1 + + def _resolve_upvalue(self, name: str) -> int: + if not self.enclosing: + return -1 + + for index, local in reversed(list(enumerate(self.enclosing.locals))): + if local.name == name: + local.is_captured = True + return self._add_upvalue(index, True) + + upvalue = self.enclosing._resolve_upvalue(name) + if upvalue != -1: + return self._add_upvalue(upvalue, False) + + return -1 + def visit_field(self, node: ast.Field): + ops: list[str | int] = [] for index, local in reversed(list(enumerate(self.locals))): if local.name == node.chain[0]: - if len(node.chain) == 1: - return [Operation.GET_LOCAL, index] - else: - ops: list[str | int] = [Operation.GET_LOCAL, index] - for element in node.chain[1:]: - if isinstance(element, int): - ops.extend([Operation.INTEGER, element, Operation.GET_PROPERTY]) - else: - ops.extend([Operation.STRING, str(element), Operation.GET_PROPERTY]) - return ops + ops = [Operation.GET_LOCAL, index] + break + + if len(ops) == 0: + arg = self._resolve_upvalue(str(node.chain[0])) + if arg != -1: + ops = [Operation.GET_UPVALUE, arg] + + if len(ops) > 0: + if len(node.chain) > 1: + for element in node.chain[1:]: + if isinstance(element, int): + ops.extend([Operation.INTEGER, element, Operation.GET_PROPERTY]) + else: + ops.extend([Operation.STRING, str(element), Operation.GET_PROPERTY]) + return ops + + # Did not find a local nor an upvalue, must be a global. + chain = [] for element in reversed(node.chain): chain.extend([Operation.STRING, element]) @@ -272,30 +320,57 @@ def visit_call(self, node: ast.Call): response.extend(if_null) return response - if node.name not in STL and node.name not in self.functions and node.name not in self.supported_functions: - raise QueryError(f"Hog function `{node.name}` is not implemented") - if node.name in self.functions and len(node.args) != len(self.functions[node.name].params): - raise QueryError( - f"Function `{node.name}` expects {len(self.functions[node.name].params)} arguments, got {len(node.args)}" - ) - response = [] + # HogQL functions can have two sets of parameters: asd(args) or asd(params)(args) + # If params exist, take them as the first set + args = node.params if node.params is not None else node.args - for expr in node.args: + response = [] + for expr in args: response.extend(self.visit(expr)) - if node.name in MIN_ARGS_INCLUDING_OPTIONAL and len(node.args) < MIN_ARGS_INCLUDING_OPTIONAL[node.name]: - for _ in range(len(node.args), MIN_ARGS_INCLUDING_OPTIONAL[node.name]): - response.append(Operation.NULL) + found_local_with_name = False + for local in reversed(self.locals): + if local.name == node.name: + found_local_with_name = True + + if found_local_with_name: + field = self.visit(ast.Field(chain=[node.name])) + response.extend([*field, Operation.CALL_LOCAL, len(args)]) + else: + upvalue = self._resolve_upvalue(node.name) + if upvalue != -1: + response.extend([Operation.GET_UPVALUE, upvalue, Operation.CALL_LOCAL, len(args)]) + else: + if self.context.globals and node.name in self.context.globals: + self.context.notices.append( + HogQLNotice(start=node.start, end=node.end, message="Global variable: " + str(node.name)) + ) + elif node.name in self.supported_functions or node.name in STL: + pass + else: + self.context.errors.append( + HogQLNotice( + start=node.start, end=node.end, message=f"Hog function `{node.name}` is not implemented" + ) + ) + + response.extend([Operation.CALL_GLOBAL, node.name, len(args)]) + + # If the node has two sets of params, process the second set now + if node.params is not None: + next_response = [] + for expr in node.args: + next_response.extend(self.visit(expr)) + response = [*next_response, *response, Operation.CALL_LOCAL, len(node.args)] - response.extend( - [ - Operation.CALL_GLOBAL, - node.name, - len(node.args) - if node.name not in MIN_ARGS_INCLUDING_OPTIONAL - else MIN_ARGS_INCLUDING_OPTIONAL[node.name], - ] - ) + return response + + def visit_expr_call(self, node: ast.ExprCall): + response = [] + for expr in node.args: + response.extend(self.visit(expr)) + response.extend(self.visit(node.expr)) + response.extend([Operation.CALL_LOCAL, len(node.args)]) return response def visit_program(self, node: ast.Program): @@ -497,7 +572,15 @@ def visit_for_in_statement(self, node: ast.ForInStatement): response.extend([Operation.INTEGER, 1]) loop_limit_local = self._declare_local("__H_limit_H__") # length of keys - response.extend([Operation.GET_LOCAL, expr_values_local, Operation.CALL_GLOBAL, "length", 1]) + response.extend( + [ + Operation.GET_LOCAL, + expr_values_local, + Operation.CALL_GLOBAL, + "length", + 1, + ] + ) if key_var is not None: key_var_local = self._declare_local(key_var) # loop key @@ -586,6 +669,7 @@ def visit_variable_assignment(self, node: ast.VariableAssignment): ] if isinstance(node.left, ast.Field) and len(node.left.chain) >= 1: + ops: list chain = node.left.chain name = chain[0] for index, local in reversed(list(enumerate(self.locals))): @@ -595,7 +679,7 @@ def visit_variable_assignment(self, node: ast.VariableAssignment): return [*self.visit(cast(AST, node.right)), Operation.SET_LOCAL, index] # else set a property on a local object - ops: list = [Operation.GET_LOCAL, index] + ops = [Operation.GET_LOCAL, index] for element in chain[1:-1]: if isinstance(element, int): ops.extend([Operation.INTEGER, element, Operation.GET_PROPERTY]) @@ -608,16 +692,30 @@ def visit_variable_assignment(self, node: ast.VariableAssignment): return ops + upvalue_index = self._resolve_upvalue(str(chain[0])) + if upvalue_index != -1: + # Set an upvalue + if len(node.left.chain) == 1: + return [*self.visit(cast(AST, node.right)), Operation.SET_UPVALUE, upvalue_index] + + # else set a property on an upvalue object + ops = [Operation.GET_UPVALUE, upvalue_index] + for element in chain[1:-1]: + if isinstance(element, int): + ops.extend([Operation.INTEGER, element, Operation.GET_PROPERTY]) + else: + ops.extend([Operation.STRING, str(element), Operation.GET_PROPERTY]) + if isinstance(chain[-1], int): + ops.extend([Operation.INTEGER, chain[-1], *self.visit(node.right), Operation.SET_PROPERTY]) + else: + ops.extend([Operation.STRING, str(chain[-1]), *self.visit(node.right), Operation.SET_PROPERTY]) + + return ops raise QueryError(f'Variable "{name}" not declared in this scope. Can not assign to globals.') raise QueryError(f"Can not assign to this type of expression") def visit_function(self, node: ast.Function): - if node.name in self.functions: - raise QueryError(f"Function `{node.name}` already declared") - all_known_functions = self.supported_functions.union(set(self.functions.keys())) - all_known_functions.add(node.name) - # add an implicit return if none at the end of the function body = node.body if isinstance(node.body, ast.Block): @@ -626,9 +724,51 @@ def visit_function(self, node: ast.Function): elif not isinstance(node.body, ast.ReturnStatement): body = ast.Block(declarations=[node.body, ast.ReturnStatement(expr=None)]) - bytecode = create_bytecode(body, all_known_functions, node.params, self.context) - self.functions[node.name] = HogFunction(node.name, node.params, bytecode) - return [Operation.DECLARE_FN, node.name, len(node.params), len(bytecode), *bytecode] + self._declare_local(node.name) + compiler = BytecodeCompiler(self.supported_functions, node.params, self.context, self) + bytecode = compiler.visit(body) + + ops = [ + Operation.CALLABLE, + node.name, + len(node.params), + len(compiler.upvalues), + len(bytecode), + *bytecode, + Operation.CLOSURE, + len(compiler.upvalues), + ] + for upvalue in compiler.upvalues: + ops.extend([upvalue.is_local, upvalue.index]) + return ops + + def visit_lambda(self, node: ast.Lambda): + # add an implicit return if none at the end of the function + expr: ast.Expr | ast.Statement = node.expr + if isinstance(expr, ast.Block): + if len(expr.declarations) == 0 or not isinstance(expr.declarations[-1], ast.ReturnStatement): + expr = ast.Block(declarations=[*expr.declarations, ast.ReturnStatement(expr=None)]) + elif not isinstance(expr, ast.ReturnStatement): + if isinstance(expr, ast.Statement): + expr = ast.Block(declarations=[expr, ast.ReturnStatement(expr=None)]) + else: + expr = ast.ReturnStatement(expr=expr) + + compiler = BytecodeCompiler(self.supported_functions, node.args, self.context, self) + bytecode = compiler.visit(expr) + ops = [ + Operation.CALLABLE, + "lambda", + len(node.args), + len(compiler.upvalues), + len(bytecode), + *bytecode, + Operation.CLOSURE, + len(compiler.upvalues), + ] + for upvalue in compiler.upvalues: + ops.extend([upvalue.is_local, upvalue.index]) + return ops def visit_dict(self, node: ast.Dict): response = []