Make dev/generate-kernel-signatures.py part of the build process and …

…add Python ctypes signatures as well. (#966)

* Inserted dev/generate-kernel-signatures.py into the build process.

* Refactored generate-kernel-signatures.py, deleted kernels.h.

* Now generating _cpu_kernels_signatures.py.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Merge with main.

* Renamed to _kernel_signatures.py and everything should be working now.

* Quick fix.

* The 'raw setup.py' method of including generated files.

* Remove CUDA test-build and deploy.

* Applying @henryiii's build_py-based correction.

* Implemented nplike kernels and Content.handle_error.

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.ci/azure-buildtest-awkward.yml

@@ -328,17 +328,6 @@ jobs:
           python -m pytest -vv -rs tests
         displayName: "Test"
 
-      - script: |
-          python dev/generate-cuda.py
-        displayName: "Generate CUDA kernels"
-        condition: "and(succeeded(), and(ne(variables['python.version'], '3.8'), ne(variables['python.version'], '3.9')))"
-
-      - script: |
-          python -m pip install wheel
-          bash cuda-build.sh
-        displayName: "Build the cuda-kernels"
-        condition: "and(succeeded(), and(ne(variables['python.version'], '2.7'), ne(variables['python.version'], '3.9')))"
-
       - script: |
           echo '===== include directory ============================='
           ls -R `python -m awkward.config --incdir`

.gitignore

@@ -5,14 +5,17 @@ docs/demos/test-program.cpp
 .ci/logs/
 .clangd/
 
+# Generated includes
+include/awkward/kernels.h
+src/awkward/_kernel_signatures.py
+
 # Generated CUDA kernels
 src/cuda-kernels/awkward_*.cu
 
 # Kernel tests
-tests-cpu-kernels/test*
-tests-cuda-kernels/test*
-tests-spec/kernels.py
-tests-spec/test*
+tests-cpu-kernels
+tests-cuda-kernels
+tests-spec
 
 dependent-project/CMakeFiles
 dependent-project/CMakeCache.txt

MANIFEST.in

@@ -1,4 +1,5 @@
 include requirements*.txt VERSION_INFO CMakeLists.txt LICENSE README*.md CONTRIBUTING.md runtime.txt
+include dev/generate-kernel-signatures.py
 recursive-include include *.h
 recursive-include src *.cpp *.py *.cu
 recursive-include tests *.cpp *.py samples/*

VERSION_INFO

@@ -1 +1 @@
-1.4.0rc1
+1.4.0rc2

dev/clean-generated.py

@@ -1,28 +1,27 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
+
+from __future__ import absolute_import
+
 import argparse
 import glob
 import os
+import shutil
 
 CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
 
 
 def clean_tests():
-    if os.path.exists(os.path.join(CURRENT_DIR, "..", "tests-spec", "kernels.py")):
-        os.remove(os.path.join(CURRENT_DIR, "..", "tests-spec", "kernels.py"))
-    cpu_kernel_tests = glob.glob(
-        os.path.join(CURRENT_DIR, "..", "tests-cpu-kernels", "test") + "*"
-    )
-    for testfile in cpu_kernel_tests:
-        os.remove(testfile)
-    cuda_kernel_tests = glob.glob(
-        os.path.join(CURRENT_DIR, "..", "tests-cuda-kernels", "test") + "*"
-    )
-    for testfile in cuda_kernel_tests:
-        os.remove(testfile)
-    kernel_spec_tests = glob.glob(
-        os.path.join(CURRENT_DIR, "..", "tests-spec", "test") + "*"
-    )
-    for testfile in kernel_spec_tests:
-        os.remove(testfile)
+    tests_spec = os.path.join(CURRENT_DIR, "..", "tests-spec")
+    if os.path.exists(tests_spec):
+        shutil.rmtree(tests_spec)
+
+    tests_cpu_kernels = os.path.join(CURRENT_DIR, "..", "tests-cpu-kernels")
+    if os.path.exists(tests_cpu_kernels):
+        shutil.rmtree(tests_cpu_kernels)
+
+    tests_cuda_kernels = os.path.join(CURRENT_DIR, "..", "tests-cuda-kernels")
+    if os.path.exists(tests_cuda_kernels):
+        shutil.rmtree(tests_cuda_kernels)
 
 
 def clean_cuda_kernels():

dev/generate-cuda.py

@@ -1,5 +1,7 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
 
+from __future__ import absolute_import
+
 import argparse
 import ast
 import copy

dev/generate-kernel-signatures.py

@@ -0,0 +1,206 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
+
+from __future__ import absolute_import
+
+import os
+import datetime
+
+import yaml
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def type_to_ctype(typename):
+    is_const = False
+    if "Const[" in typename:
+        is_const = True
+        typename = typename[len("Const[") : -1]
+    count = 0
+    while "List[" in typename:
+        count += 1
+        typename = typename[len("List[") : -1]
+    typename = typename + "*" * count
+    if is_const:
+        typename = "const " + typename
+    return typename
+
+
+def include_kernels_h(specification):
+    print("Generating include/awkward/kernels.h...")
+
+    with open(
+        os.path.join(CURRENT_DIR, "..", "include", "awkward", "kernels.h"), "w"
+    ) as header:
+        header.write(
+            """// AUTO GENERATED ON {0}
+// DO NOT EDIT BY HAND!
+//
+// To regenerate file, run
+//
+//     python dev/generate-kernel-signatures.py
+//
+// (It is usually run as part of pip install . or localbuild.py.)
+
+#ifndef AWKWARD_KERNELS_H_
+#define AWKWARD_KERNELS_H_
+
+#include "awkward/common.h"
+
+extern "C" {{
+
+""".format(
+                datetime.datetime.now().isoformat().replace("T", " AT ")[:22]
+            )
+        )
+        for spec in specification["kernels"]:
+            for childfunc in spec["specializations"]:
+                header.write(" " * 2 + "EXPORT_SYMBOL ERROR\n")
+                header.write(" " * 2 + childfunc["name"] + "(\n")
+                for i, arg in enumerate(childfunc["args"]):
+                    header.write(
+                        " " * 4 + type_to_ctype(arg["type"]) + " " + arg["name"]
+                    )
+                    if i == (len(childfunc["args"]) - 1):
+                        header.write(");\n")
+                    else:
+                        header.write(",\n")
+            header.write("\n")
+        header.write(
+            """}
+
+#endif // AWKWARD_KERNELS_H_
+"""
+        )
+
+    print("Done with  include/awkward/kernels.h.")
+
+
+type_to_dtype = {
+    "bool": "bool_",
+    "int8": "int8",
+    "uint8": "uint8",
+    "int16": "int16",
+    "uint16": "uint16",
+    "int32": "int32",
+    "uint32": "uint32",
+    "int64": "int64",
+    "uint64": "uint64",
+    "float": "float32",
+    "double": "float64",
+}
+
+
+def type_to_pytype(typename, special):
+    if "Const[" in typename:
+        typename = typename[len("Const[") : -1]
+    count = 0
+    while "List[" in typename:
+        count += 1
+        typename = typename[len("List[") : -1]
+    if typename.endswith("_t"):
+        typename = typename[:-2]
+    if count != 0:
+        special.append(type_to_dtype[typename])
+    return ("POINTER(" * count) + ("c_" + typename) + (")" * count)
+
+
+def kernel_signatures_py(specification):
+    print("Generating src/awkward/_kernel_signatures.py...")
+
+    with open(
+        os.path.join(CURRENT_DIR, "..", "src", "awkward", "_kernel_signatures.py"),
+        "w",
+    ) as file:
+        file.write(
+            """# AUTO GENERATED ON {0}
+# DO NOT EDIT BY HAND!
+#
+# To regenerate file, run
+#
+#     python dev/generate-kernel-signatures.py
+#
+# (It is usually run as part of pip install . or localbuild.py.)
+
+# fmt: off
+
+from ctypes import (
+    POINTER,
+    Structure,
+    c_bool,
+    c_int8,
+    c_uint8,
+    c_int16,
+    c_uint16,
+    c_int32,
+    c_uint32,
+    c_int64,
+    c_uint64,
+    c_float,
+    c_double,
+    c_char_p,
+)
+
+import numpy as np
+
+from numpy import (
+    bool_,
+    int8,
+    uint8,
+    int16,
+    uint16,
+    int32,
+    uint32,
+    int64,
+    uint64,
+    float32,
+    float64,
+)
+
+class ERROR(Structure):
+    _fields_ = [
+        ("str", c_char_p),
+        ("filename", c_char_p),
+        ("id", c_int64),
+        ("attempt", c_int64),
+        ("pass_through", c_bool),
+    ]
+
+
+def by_signature(lib):
+    out = {{}}
+""".format(
+                datetime.datetime.now().isoformat().replace("T", " AT ")[:22]
+            )
+        )
+
+        for spec in specification["kernels"]:
+            for childfunc in spec["specializations"]:
+                special = [repr(spec["name"])]
+                arglist = [
+                    type_to_pytype(x["type"], special) for x in childfunc["args"]
+                ]
+                file.write(
+                    """
+    f = lib.{0}
+    f.argtypes = [{1}]
+    f.restype = ERROR
+    out[{2}] = f
+""".format(
+                        childfunc["name"], ", ".join(arglist), ", ".join(special)
+                    )
+                )
+
+        file.write(
+            """
+    return out
+"""
+        )
+
+    print("Done with  src/awkward/_kernel_signatures.py...")
+
+
+if __name__ == "__main__":
+    with open(os.path.join(CURRENT_DIR, "..", "kernel-specification.yml")) as specfile:
+        specification = yaml.safe_load(specfile)
+        include_kernels_h(specification)
+        kernel_signatures_py(specification)

dev/generate-kerneldocs.py

@@ -1,5 +1,7 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
 
+from __future__ import absolute_import
+
 import os
 
 import yaml