union array support wip

Morglod · Apr 5, 2024 · b52909a · b52909a
1 parent aca3e0c
commit b52909a
Show file tree

Hide file tree

Showing 9 changed files with 376 additions and 58 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,8 @@
 generate-wgpu-bindings_cache_*.json
 generate-bindings_tmp_exec
 example.ts
+lib
+bindings_cache_*
 
 # Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
 

diff --git a/.npmignore b/.npmignore
@@ -2,3 +2,4 @@ node_modules
 generate-wgpu-bindings_cache_*.json
 generate-bindings_tmp_exec
 example.ts
+src
diff --git a/README.md b/README.md
@@ -25,11 +25,15 @@ bun install bun-ffi-gen
 Example of generating bindings for wgpu library.
 
 ```ts
-import { ClangTypeInfoCache, clangGetAstJson, CodeGen, parseClangAst } from "bun-ffi-gen";
+import { ClangTypeInfoCache, clangGetAstJson, CodeGen, parseClangAst, clangClean, addIncludeDir } from "bun-ffi-gen";
+import path from "path";
 
 const HEADER_PATH = "./wgpu/wgpu.h";
 const TYPE_CACHE_PATH = "./generate-wgpu-bindings_cache";
 
+// add include dirs for clang
+addIncludeDir(path.resolve("my_include_dir"));
+
 // get header ast from clang
 const wgpuAst = clangGetAstJson(HEADER_PATH);
 

diff --git a/package.json b/package.json
@@ -6,6 +6,9 @@
   "type": "module",
   "main": "lib/index.ts",
   "types": "lib/index.ts",
+  "scripts": {
+    "build": "tsc"
+  },
   "author": {
     "name": "morglod",
     "url": "https://github.com/Morglod"

diff --git a/src/clang.ts b/src/clang.ts
@@ -1,20 +1,56 @@
-import { execSync } from "child_process";
+import { execSync as nodeExecSync, exec as nodeExec } from "child_process";
 import { existsSync, rmSync, writeFileSync } from "fs";
+import { logInfo, logVerbose } from "./log";
 
-export function clangGetAstJson(headerPath: string): any[] {
-    const cmd = "clang -Xclang -ast-dump=json -fsyntax-only " + headerPath;
-    const result = execSync(cmd).toString();
-    return JSON.parse(result).inner;
+const includeDirArgs: string[] = [];
+
+export function addIncludeDir(dir: string) {
+    includeDirArgs.push(`-I${dir}`);
+}
+
+export const execSync: typeof nodeExecSync = (command: string, ...args: any): any => {
+    logVerbose("execSync", command);
+    return nodeExecSync(command, ...args);
+};
+
+export async function execLargeJSON(command: string): Promise<any> {
+    logVerbose("execLargeJSON", command);
+    const tmpFileName = `./exec_tmp_${Date.now()}_${Math.floor(Math.random() * 9999)}`;
+
+    nodeExecSync(command + " > " + tmpFileName);
+    const json = await Bun.file(tmpFileName).json();
+    rmSync(tmpFileName);
+
+    return json;
+}
+
+export async function clangGetAstJson(headerPath: string): Promise<any[]> {
+    const cmd = `clang -Xclang -ast-dump=json -fsyntax-only ${includeDirArgs.join(" ")} ` + headerPath;
+
+    logInfo(`clangGetAstJson headerPath="${headerPath}" command="${cmd}"`);
+
+    const result = await execLargeJSON(cmd);
+    return result.inner;
 }
 
 export function clangCompileAndRunReadOut(code: string) {
-    execSync("clang -x c - -o ./generate-bindings_tmp_exec", {
+    const command = `clang ${includeDirArgs.join(" ")} -x c++ - -o ./generate-bindings_tmp_exec`;
+    if (command === "clang -I/Users/work_vk/Desktop/Dev/personal/bun-ffi-gen/include -x c++ - -o ./generate-bindings_tmp_exec") {
+        execSync("clang -I/Users/work_vk/Desktop/Dev/personal/bun-ffi-gen/include -x c++ - -E > aaa", {
+            input: code,
+            stdio: "pipe",
+        });
+    }
+    execSync(command, {
         input: code,
+        stdio: "pipe",
     });
     return execSync("./generate-bindings_tmp_exec").toString();
 }
 
 export async function clangClean() {
+    logInfo("clangClean");
+
     if (existsSync("./generate-bindings_tmp_exec")) {
         rmSync("./generate-bindings_tmp_exec");
     }
@@ -27,11 +63,14 @@ export class ClangTypeInfoCache {
     offsetOf: Record<string, number> = {};
 
     async save() {
+        logInfo("ClangTypeInfoCache.save");
         writeFileSync(this.cacheFilePath + "_sizeof.json", JSON.stringify(this.sizeOf));
         writeFileSync(this.cacheFilePath + "_offsetof.json", JSON.stringify(this.offsetOf));
     }
 
     static async create(cacheFilePath: string) {
+        logInfo(`ClangTypeInfoCache.create cacheFilePath="${cacheFilePath}"`);
+
         let sizeOf = {};
         if (await Bun.file(cacheFilePath + "_sizeof.json").exists()) {
             sizeOf = await Bun.file(cacheFilePath + "_sizeof.json").json();
@@ -84,7 +123,9 @@ export function clangGetOffsetOf(headerPath: string, cTypeName: string, fieldNam
         #include <stdio.h>
 
         int main() {
-            printf("[ %lu ]", offsetof(${cTypeName}, ${fieldName}));
+            printf("[ %lu ]",
+                ((size_t)&(reinterpret_cast<${cTypeName}*>(0)->${fieldName}))
+            );
             return 0;
         }
     `;

diff --git a/src/gen.ts b/src/gen.ts
@@ -1,3 +1,4 @@
+import { logVerbose } from "./log";
 import type {
     ParsedClangAstItem,
     ParsedClangAstItem_Alias,
@@ -155,6 +156,7 @@ export class CodeGen {
                 type Pointer as BunPointer,
                 read as bunRead,
                 suffix as bunSuffix,
+                toBuffer as bunToBuffer,
             } from "bun:ffi";`
         );
     }
@@ -170,15 +172,16 @@ type _PartialStructArg<T> = {
     T[P] extends PtrT<any> ? T[P] :
     T[P] extends ConstPtrT<any> ? T[P] :
     T[P] extends CString ? T[P] :
+    T[P] extends Buffer ? T[P] :
     T[P] extends object ? _PartialStructArg<T[P]> :
     T[P];
 };
 
 const Pointer = BunFFIType.ptr;
 type Pointer = BunPointer | null;
-type ConstPtrT<T> = (Pointer | TypedArray | Buffer) & { __type: T, __const_ptr: true };
-type PtrT<T> = (Pointer | TypedArray | Buffer) & { __type: T, __const_ptr: false };
-type TypedArrayPtr<T> = (TypedArray | Buffer) & { __type: T, __const_ptr: any };
+type ConstPtrT<T> = (Pointer | NodeJS.TypedArray | Buffer) & { __type: T, __const_ptr: true };
+type PtrT<T> = (Pointer | NodeJS.TypedArray | Buffer) & { __type: T, __const_ptr: false };
+type TypedArrayPtr<T> = (NodeJS.TypedArray | Buffer) & { __type: T, __const_ptr: any };
 
 export const NULL = null as any as Pointer & { __type: any, __const_ptr: any };
 
@@ -194,7 +197,7 @@ export function bunReadArray<T>(from: Pointer | TypedArrayPtr<T>, offset: number
 }
 
 export function alloc_CString(str: string) {
-    return new BunCString(bunPtr(Buffer.from(str + "\\0")));
+    return new BunCString(bunPtr(Buffer.from(str + "\\0")) as any);
 }
 
 export function alloc_opaque_pointer(x: Pointer, buffer?: Buffer): TypedArrayPtr<any> {
@@ -237,10 +240,10 @@ export function alloc_opaque_pointer(x: Pointer, buffer?: Buffer): TypedArrayPtr
         if (this.opts.readers) {
             this.writeLn(out, `export function read_${name}(from: BunPointer, offset: number): ${tsType} {`);
             if (d.ffiType === "BunFFIType.cstring") {
-                this.writeLn(out, `return new BunCString(bunRead.ptr(from, offset));`, 1);
+                this.writeLn(out, `return new BunCString(bunRead.ptr(from, offset) as any);`, 1);
             } else {
                 const readFn = this.mapFFITypeToReadFn(d.ffiType);
-                this.writeLn(out, `return bunRead.${readFn}(from, offset);`, 1);
+                this.writeLn(out, `return bunRead.${readFn}(from, offset) as any;`, 1);
             }
             this.writeLn(out, `}`);
         }
@@ -307,6 +310,9 @@ export function alloc_opaque_pointer(x: Pointer, buffer?: Buffer): TypedArrayPtr
             this.writeLn(out, `${fieldName} = ${f.value},`, 1);
         }
         this.writeLn(out, `}`);
+        for (const [fieldName, f] of d.fields) {
+            this.writeLn(out, `export const ${fieldName} = ${name}.${fieldName};`, 0);
+        }
 
         if (this.opts.readers) {
             this.writeLn(out, `export function read_${name}(from: BunPointer, offset: number): ${name} {`);
@@ -321,6 +327,11 @@ export function alloc_opaque_pointer(x: Pointer, buffer?: Buffer): TypedArrayPtr
     }
 
     generateAlias(out: string[], name: string, d: ParsedClangAstItem_Alias) {
+        if (d.noEmit) {
+            logVerbose("skip noEmit alias", d);
+            return;
+        }
+
         this.writeLn(out, `export type ${name} = ${d.aliasTo.name};`);
 
         if (this.opts.readers) {
@@ -352,7 +363,7 @@ export function alloc_opaque_pointer(x: Pointer, buffer?: Buffer): TypedArrayPtr
             if (funcDeclCode instanceof Error) {
             } else {
                 this.writeLn(out, `export function read_${name}(from: BunPointer, offset: number): ${name} {`);
-                this.writeLn(out, `const ptr = bunRead.ptr(from, offset);`, 1);
+                this.writeLn(out, `const ptr = bunRead.ptr(from, offset) as any as BunPointer;`, 1);
                 this.writeLn(out, `return BunCFunction({`, 1);
                 this.writeLn(out, `ptr,`, 2);
                 this.writeLn(out, `...${funcDeclCode},`, 2);
@@ -396,7 +407,10 @@ export function alloc_opaque_pointer(x: Pointer, buffer?: Buffer): TypedArrayPtr
     }
 
     inlineTsType(d: ParsedClangAstItem): string {
-        if (d.name) return d.name;
+        if (d.type === "static_array" || d.type === "union") {
+            return `Buffer`;
+        }
+        if (!(d as any).noEmit && d.name) return d.name;
         switch (d.type) {
             case "alias":
                 return this.inlineTsType(d.aliasTo);
@@ -444,6 +458,11 @@ export function alloc_opaque_pointer(x: Pointer, buffer?: Buffer): TypedArrayPtr
                 if (f.valueType.type === "pointer") {
                     this.writeLn(out, `${f.name}: read_opaque_pointer(from, offset + ${f.offset}) as any,`, 2);
                 } else {
+                    if (f.valueType.type === "union" || f.valueType.type === "static_array") {
+                        this.writeLn(out, `${f.name}: bunToBuffer(from, offset + ${f.offset}, ${f.valueType.size}),`, 2);
+                        continue;
+                    }
+
                     if (!f.valueType.name) {
                         throw new Error("bad value type");
                     }
@@ -460,6 +479,10 @@ export function alloc_opaque_pointer(x: Pointer, buffer?: Buffer): TypedArrayPtr
                 if (f.valueType.type === "pointer") {
                     this.writeLn(out, `x.${f.name} !== undefined && write_opaque_pointer(x.${f.name}, buffer, offset + ${f.offset});`, 2);
                 } else {
+                    if (f.valueType.type === "union" || f.valueType.type === "static_array") {
+                        this.writeLn(out, `x.${f.name} !== undefined && buffer.copy(new Uint8Array(x.${f.name}), offset + ${f.offset}, 0, ${f.valueType.size});`, 1);
+                        continue;
+                    }
                     if (!f.valueType.name) {
                         throw new Error("bad value type");
                     }

diff --git a/src/log.ts b/src/log.ts
@@ -0,0 +1,15 @@
+export enum LogLevel {
+    none = 0,
+    info = 1,
+    verbose = 2,
+}
+
+export let logLevel = LogLevel.info;
+
+export function logVerbose(...args: any[]) {
+    if (logLevel === LogLevel.verbose) console.log("[verbose]", ...args);
+}
+
+export function logInfo(...args: any[]) {
+    if (logLevel >= LogLevel.info) console.log("[info]", ...args);
+}