diff --git a/.dockerignore b/.dockerignore index 2c26fc9cb00411..74aba7a4bdbff2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,6 +23,7 @@ !frontend/public !frontend/*.* !ee +!hogql_parser !.devcontainer !plugin-server/package.json !plugin-server/pnpm-lock.yaml @@ -33,4 +34,4 @@ !plugin-server/.prettierrc !share/GeoLite2-City.mmdb !hogvm/python -!unit.json \ No newline at end of file +!unit.json diff --git a/hogql_parser/parser.cpp b/hogql_parser/parser.cpp index b1721bab829a59..28cd6812c06d05 100644 --- a/hogql_parser/parser.cpp +++ b/hogql_parser/parser.cpp @@ -1,7 +1,6 @@ #define PY_SSIZE_T_CLEAN #include #include -#include #include #include "HogQLLexer.h" @@ -271,7 +270,7 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor { if (array_join_clause_ctx) { if (Py_IsNone(PyObject_GetAttrString(select_query, "select_from"))) { Py_DECREF(select_query); - throw HogQLSyntaxError("Using ARRAY JOIN without a FROM clause is not permitted", 0, 0); // FIXME: start=end= + throw HogQLSyntaxError("Using ARRAY JOIN without a FROM clause is not permitted", 0, 0); // FIXME: start=end= } PyObject_SetAttrString( select_query, "array_join_op", @@ -288,7 +287,7 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor { if (!is_ast_node_instance(expr, "Alias")) { Py_DECREF(array_join_list); Py_DECREF(select_query); - throw HogQLSyntaxError("ARRAY JOIN arrays must have an alias", 0, 0); // FIXME: start=end= + throw HogQLSyntaxError("ARRAY JOIN arrays must have an alias", 0, 0); // FIXME: start=end= } } PyObject_SetAttrString(select_query, "array_join_list", array_join_list); @@ -573,7 +572,7 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor { if (find(RESERVED_KEYWORDS.begin(), RESERVED_KEYWORDS.end(), alias) != RESERVED_KEYWORDS.end()) { Py_DECREF(expr); - throw HogQLSyntaxError("ALIAS is a reserved keyword", 0, 0); // FIXME: start=end= + throw HogQLSyntaxError("ALIAS is a reserved keyword", 0, 0); // FIXME: start=end= } return build_ast_node("Alias", "{s:N,s:s#}", "expr", expr, "alias", alias.c_str(), alias.size()); @@ -768,7 +767,7 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor { PyObject_RichCompareBool(PyObject_GetAttrString(property, "value"), PyLong_FromLong(0), Py_EQ)) { Py_DECREF(property); Py_DECREF(object); - throw HogQLSyntaxError("SQL indexes start from one, not from zero. E.g: array[1]", 0, 0); // FIXME: start=end= + throw HogQLSyntaxError("SQL indexes start from one, not from zero. E.g: array[1]", 0, 0); // FIXME: start=end= } return build_ast_node("ArrayAccess", "{s:N,s:N}", "array", object, "property", property); } @@ -836,7 +835,7 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor { if (PyObject_RichCompareBool(index, PyLong_FromLong(0), Py_EQ)) { Py_DECREF(index); Py_DECREF(tuple); - throw HogQLSyntaxError("SQL indexes start from one, not from zero. E.g: array[1]", 0, 0); // FIXME: start=end= + throw HogQLSyntaxError("SQL indexes start from one, not from zero. E.g: array[1]", 0, 0); // FIXME: start=end= } return build_ast_node("TupleAccess", "{s:N,s:N}", "tuple", tuple, "index", index); } @@ -923,7 +922,9 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor { VISIT(ColumnLambdaExpr) { vector args = visitAsVectorOfStrings(ctx->identifier()); - return build_ast_node("Lambda", "{s:N,s:N}", "args", X_PyList_FromStrings(args), "expr", visitAsPyObject(ctx->columnExpr())); + return build_ast_node( + "Lambda", "{s:N,s:N}", "args", X_PyList_FromStrings(args), "expr", visitAsPyObject(ctx->columnExpr()) + ); } VISIT(WithExprList) { @@ -1001,7 +1002,7 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor { auto alias_ctx = ctx->alias(); string alias = any_cast(alias_ctx ? visit(alias_ctx) : visit(ctx->identifier())); if (find(RESERVED_KEYWORDS.begin(), RESERVED_KEYWORDS.end(), alias) != RESERVED_KEYWORDS.end()) { - throw HogQLSyntaxError("ALIAS is a reserved keyword", 0, 0); // FIXME: start=end= + throw HogQLSyntaxError("ALIAS is a reserved keyword", 0, 0); // FIXME: start=end= } PyObject* table = visitAsPyObject(ctx->tableExpr()); PyObject* py_alias = PyUnicode_FromStringAndSize(alias.c_str(), alias.size()); @@ -1298,8 +1299,14 @@ static int parser_clear(PyObject* module) { } static struct PyModuleDef parser = { - PyModuleDef_HEAD_INIT, .m_name = "parser", .m_doc = "HogQL parsing", .m_size = sizeof(parser_state), - .m_methods = parser_methods, .m_slots = parser_slots, .m_traverse = parser_traverse, .m_clear = parser_clear, + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "parser", + .m_doc = "HogQL parsing", + .m_size = sizeof(parser_state), + .m_methods = parser_methods, + .m_slots = parser_slots, + .m_traverse = parser_traverse, + .m_clear = parser_clear, }; PyMODINIT_FUNC PyInit_hogql_parser(void) { diff --git a/hogql_parser/setup.py b/hogql_parser/setup.py index 9975e79a6653d6..65650b100e1d6c 100644 --- a/hogql_parser/setup.py +++ b/hogql_parser/setup.py @@ -1,6 +1,7 @@ from distutils.core import setup, Extension import platform +arch = "aarch64" if platform.machine() == "arm64" else "x86_64" module = Extension( "hogql_parser", sources=[ @@ -13,10 +14,15 @@ "parser.cpp", ], include_dirs=[ - "/opt/homebrew/include/antlr4-runtime/" if platform.system() == "Darwin" else "/usr/include/antlr4-runtime/", + "/opt/homebrew/include/antlr4-runtime/" + if platform.system() == "Darwin" + else "/usr/local/include/antlr4-runtime/", "/opt/homebrew/include/" if platform.system() == "Darwin" else "/usr/include/", ], - library_dirs=["/opt/homebrew/Cellar/antlr4-cpp-runtime/4.13.1/lib/", "/opt/homebrew/Cellar/boost/1.82.0_1/lib/"], + library_dirs=[ + "/opt/homebrew/Cellar/antlr4-cpp-runtime/4.13.1/lib/" if platform.system() == "Darwin" else "/usr/local/lib/", + "/opt/homebrew/Cellar/boost/1.82.0_1/lib/" if platform.system() == "Darwin" else f"/usr/lib/{arch}-linux-gnu/", + ], libraries=["antlr4-runtime"], extra_compile_args=["-std=c++20"], ) diff --git a/production.Dockerfile b/production.Dockerfile index 5075ef0671b136..b93322de9df89e 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -88,19 +88,27 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Compile and install Python dependencies. # We install those dependencies on a custom folder that we will # then copy to the last image. -COPY requirements.txt ./ RUN apt-get update && \ apt-get install -y --no-install-recommends \ - "build-essential" \ - "git" \ - "libpq-dev" \ - "libxmlsec1" \ - "libxmlsec1-dev" \ - "libffi-dev" \ - "pkg-config" \ - && \ - rm -rf /var/lib/apt/lists/* && \ - pip install -r requirements.txt --compile --no-cache-dir --target=/python-runtime + "build-essential" "git" "libpq-dev" "libxmlsec1" "libxmlsec1-dev" "libboost-all-dev" "libffi-dev" "curl" "unzip" \ + "cmake" "uuid-dev" "pkg-config" && \ + rm -rf /var/lib/apt/lists/* +RUN curl https://www.antlr.org/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip && \ + unzip antlr4-source.zip -d antlr4-source && \ + cd antlr4-source && \ + mkdir build && \ + mkdir run && \ + cd build && \ + cmake .. && \ + DESTDIR=../run make install && \ + cd ../run/usr/local/include && \ + cp -r antlr4-runtime /usr/local/include && \ + cd ../lib && \ + cp libantlr4-runtime.so libantlr4-runtime.so.* /usr/local/lib && \ + ldconfig +COPY requirements.txt ./ +COPY hogql_parser hogql_parser/ +RUN pip install -r requirements.txt --compile --no-cache-dir --target=/python-runtime ENV PATH=/python-runtime/bin:$PATH \ PYTHONPATH=/python-runtime diff --git a/requirements.txt b/requirements.txt index 6ec7905f80d3fa..475f3354a43da1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -248,7 +248,7 @@ gunicorn==20.1.0 # via -r requirements.in h11==0.13.0 # via wsproto -hogql-parser @ file:///Users/twixes/Developer/posthog/hogql_parser +./hogql_parser # via -r requirements.in idna==2.8 # via