reorganize build process

.github/workflows/test.yml

@@ -184,7 +184,6 @@ jobs:
                     "tests/end_to_end/candid_rpc/class_syntax/motoko_examples/superheroes",
                     "tests/end_to_end/candid_rpc/class_syntax/motoko_examples/threshold_ecdsa",
                     "tests/end_to_end/candid_rpc/class_syntax/motoko_examples/whoami",
-                    "tests/end_to_end/candid_rpc/class_syntax/new",
                     "tests/end_to_end/candid_rpc/class_syntax/new_candid_rpc",
                     "tests/end_to_end/candid_rpc/class_syntax/notify_raw",
                     "tests/end_to_end/candid_rpc/class_syntax/null_example",
@@ -284,6 +283,7 @@ jobs:
                     "tests/end_to_end/http_server/internet_identity",
                     "tests/end_to_end/http_server/large_files",
                     "tests/end_to_end/http_server/nest",
+                    "tests/end_to_end/http_server/new",
                     "tests/end_to_end/http_server/open_value_sharing",
                     "tests/end_to_end/http_server/sqlite",
                     "tests/end_to_end/http_server/sqlite_drizzle",

Cargo.toml

@@ -1,2 +1,2 @@
 [workspace]
-members = ["src/compiler/rust/canister", "src/compiler/rust/open_value_sharing"]
+members = ["src/build/rust/canister", "src/build/rust/open_value_sharing"]

dfx_extension/extension.json

@@ -9,7 +9,7 @@
     "canister_type": {
         "defaults": {
             "candid": ".azle/{{canister_name}}/{{canister_name}}.did",
-            "build": "npx azle {{canister_name}}",
+            "build": "npx azle compile {{canister_name}}",
             "post_install": "npx azle upload-assets {{canister_name}}",
             "wasm": ".azle/{{canister_name}}/{{canister_name}}.wasm",
             "gzip": true,

examples/basic_bitcoin/dfx.json

@@ -3,9 +3,10 @@
         "basic_bitcoin": {
             "type": "azle",
             "main": "src/index.ts",
-            "env": ["BITCOIN_NETWORK"],
             "custom": {
-                "experimental": true
+                "experimental": true,
+                "candid_gen": "http",
+                "env": ["BITCOIN_NETWORK"]
             }
         }
     },

examples/bitcoin_psbt/dfx.json

@@ -3,9 +3,10 @@
         "bitcoin_psbt": {
             "type": "azle",
             "main": "src/index.ts",
-            "env": ["BITCOIN_NETWORK"],
             "custom": {
-                "experimental": true
+                "experimental": true,
+                "candid_gen": "http",
+                "env": ["BITCOIN_NETWORK"]
             }
         }
     },

examples/ckbtc/dfx.json

@@ -24,12 +24,15 @@
             "type": "azle",
             "main": "wallet/backend/index.ts",
             "candid": "wallet/backend/index.did",
-            "candid_gen": "custom",
-            "env": ["CK_BTC_PRINCIPAL", "MINTER_PRINCIPAL"],
-            "assets": [["minter/minter.did", "minter/minter.did"]],
             "declarations": {
                 "node_compatibility": true,
                 "output": "wallet/frontend/dfx_generated/wallet_backend"
+            },
+            "custom": {
+                "experimental": true,
+                "assets": [["minter/minter.did", "minter/minter.did"]],
+                "candid_gen": "custom",
+                "env": ["CK_BTC_PRINCIPAL", "MINTER_PRINCIPAL"]
             }
         },
         "wallet_frontend": {

examples/hello_world/dfx.json

@@ -3,10 +3,10 @@
         "backend": {
             "type": "azle",
             "main": "src/backend/index.ts",
-            "assets": [["src/frontend/dist", "dist"]],
-            "build_assets": "npm run build",
             "custom": {
-                "experimental": true
+                "experimental": true,
+                "assets": [["src/frontend/dist", "dist"]],
+                "build_assets": "npm run build"
             }
         }
     }

examples/hello_world_candid_rpc/dfx.json

@@ -3,7 +3,6 @@
         "hello_world": {
             "type": "azle",
             "main": "src/index.ts",
-            "candid_gen": "automatic",
             "declarations": {
                 "output": "test/dfx_generated/hello_world",
                 "node_compatibility": true

package.json

@@ -11,7 +11,7 @@
     },
     "type": "module",
     "bin": {
-        "azle": "./src/compiler/index.ts"
+        "azle": "./src/build/index.ts"
     },
     "main": "./src/lib/stable/index.ts",
     "repository": {

src/build/experimental/commands/compile/candid_and_method_meta.ts

@@ -0,0 +1,102 @@
+import { IOType } from 'child_process';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+
+import { execute } from '../../../stable/commands/compile/candid_and_method_meta/execute';
+import { AZLE_PACKAGE_PATH } from '../../../stable/utils/global_paths';
+import {
+    CandidAndMethodMeta,
+    CandidGen,
+    MethodMeta
+} from '../../../stable/utils/types';
+import { WasmData } from '../../utils/types';
+import { getWasmBinary } from './wasm_binary';
+
+export async function getCandidAndMethodMeta(
+    canisterName: string,
+    candidGen: CandidGen | undefined,
+    canisterPath: string,
+    candidPath: string,
+    js: string,
+    ioType: IOType,
+    wasmData: WasmData
+): Promise<CandidAndMethodMeta> {
+    if (
+        candidGen === undefined ||
+        candidGen === 'automatic' ||
+        candidGen === 'custom'
+    ) {
+        return await handleAutomaticAndCustom(
+            candidGen,
+            candidPath,
+            canisterName,
+            ioType,
+            js,
+            wasmData,
+            canisterPath
+        );
+    }
+
+    if (candidGen === 'http') {
+        return await handleHttp();
+    }
+
+    throw new Error(`dfx.json: "candid_gen": "${candidGen}" is not supported`);
+}
+
+async function handleAutomaticAndCustom(
+    candidGen: CandidGen | undefined,
+    candidPath: string,
+    canisterName: string,
+    ioType: IOType,
+    js: string,
+    wasmData: WasmData,
+    canisterPath: string
+): Promise<CandidAndMethodMeta> {
+    const wasmBinary = await getWasmBinary(
+        canisterName,
+        ioType,
+        js,
+        wasmData,
+        canisterPath
+    );
+
+    const { candid, methodMeta } = await execute(wasmBinary);
+
+    return {
+        candid:
+            candidGen === 'custom'
+                ? (await readFile(candidPath)).toString()
+                : candid,
+        methodMeta
+    };
+}
+
+async function handleHttp(): Promise<CandidAndMethodMeta> {
+    const candid = (
+        await readFile(join(AZLE_PACKAGE_PATH, 'server.did'))
+    ).toString();
+
+    const methodMeta: MethodMeta = {
+        queries: [
+            {
+                name: 'http_request',
+                index: 0,
+                composite: true
+            }
+        ],
+        updates: [
+            {
+                name: 'http_request_update',
+                index: 1
+            }
+        ],
+        init: { name: 'init', index: 2 },
+        post_upgrade: { name: 'postUpgrade', index: 3 }
+    };
+
+    return {
+        candid,
+        methodMeta
+    };
+}

src/compiler/file_watcher/file_watcher.ts → src/build/experimental/commands/compile/file_watcher/file_watcher.ts

@@ -2,10 +2,9 @@ import { Actor, ActorMethod, ActorSubclass } from '@dfinity/agent';
 import { watch } from 'chokidar';
 import { writeFile } from 'fs/promises';
 
-import { createAuthenticatedAgent } from '../../../dfx';
-import { getCanisterJavaScript } from '../get_canister_javascript';
-import { generateUploaderIdentity } from '../uploader_identity';
-import { ok } from '../utils/result';
+import { createAuthenticatedAgent } from '../../../../../../dfx';
+import { generateUploaderIdentity } from '../../upload_assets/uploader_identity';
+import { compile as compileJavaScript } from '../javascript';
 
 type ActorReloadJs = ActorSubclass<_SERVICE>;
 interface _SERVICE {
@@ -27,7 +26,6 @@ const esmAliases = JSON.parse(process.argv[6]);
 const esmExternals = JSON.parse(process.argv[7]);
 const canisterName = process.argv[8];
 const postUpgradeIndex = Number(process.argv[9]);
-const experimental = process.argv[10] === 'true' ? true : false;
 
 // TODO https://github.com/demergent-labs/azle/issues/1664
 const watcher = watch([`**/*.ts`, `**/*.js`], {
@@ -50,8 +48,7 @@ watcher.on('all', async (event, path) => {
                 actor,
                 reloadedJsPath,
                 mainPath,
-                wasmedgeQuickJsPath,
-                experimental
+                wasmedgeQuickJsPath
             );
         } catch (error) {
             console.error(error);
@@ -63,28 +60,16 @@ async function reloadJs(
     actor: ActorReloadJs,
     reloadedJsPath: string,
     mainPath: string,
-    wasmedgeQuickJsPath: string,
-    experimental: boolean
+    wasmedgeQuickJsPath: string
 ): Promise<void> {
-    const canisterJavaScriptResult = await getCanisterJavaScript(
+    const javaScript = await compileJavaScript(
         mainPath,
         wasmedgeQuickJsPath,
         esmAliases,
-        esmExternals,
-        experimental
+        esmExternals
     );
 
-    if (!ok(canisterJavaScriptResult)) {
-        if (process.env.AZLE_VERBOSE === 'true') {
-            console.error(canisterJavaScriptResult.err!.error);
-            console.error(canisterJavaScriptResult.err!.suggestion);
-            console.error(canisterJavaScriptResult.err!.exitCode);
-        }
-
-        throw new Error(`TypeScript compilation failed`);
-    }
-
-    const reloadedJs = Buffer.from(canisterJavaScriptResult.ok);
+    const reloadedJs = Buffer.from(javaScript);
 
     const chunkSize = 2_000_000; // The current message limit is about 2 MiB
     const timestamp = process.hrtime.bigint();

src/compiler/file_watcher/setup_file_watcher.ts → src/build/experimental/commands/compile/file_watcher/setup_file_watcher.ts

@@ -1,8 +1,8 @@
 import { spawn } from 'child_process';
 import { join } from 'path';
 
-import { execSyncPretty } from '../utils/exec_sync_pretty';
-import { AZLE_PACKAGE_PATH } from '../utils/global_paths';
+import { execSyncPretty } from '../../../../stable/utils/exec_sync_pretty';
+import { AZLE_PACKAGE_PATH } from '../../../../stable/utils/global_paths';
 
 export function setupFileWatcher(
     reloadedJsPath: string,
@@ -12,8 +12,7 @@ export function setupFileWatcher(
     esmAliases: Record<string, string>,
     esmExternals: string[],
     canisterName: string,
-    postUpgradeIndex: number,
-    experimental: boolean
+    postUpgradeIndex: number
 ): void {
     try {
         // TODO should we check that this was successful in killing
@@ -36,7 +35,10 @@ export function setupFileWatcher(
             join(
                 AZLE_PACKAGE_PATH,
                 'src',
-                'compiler',
+                'build',
+                'experimental',
+                'commands',
+                'compile',
                 'file_watcher',
                 'file_watcher_loader.js'
             ),
@@ -47,8 +49,7 @@ export function setupFileWatcher(
             JSON.stringify(esmAliases),
             JSON.stringify(esmExternals),
             canisterName,
-            postUpgradeIndex.toString(),
-            experimental.toString()
+            postUpgradeIndex.toString()
         ],
         {
             detached: true,

src/build/experimental/commands/compile/get_context.ts

@@ -0,0 +1,55 @@
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+
+import { getCanisterId } from '../../../../../dfx';
+import { version } from '../../../../../package.json';
+import { getContext as getStableContext } from '../../../stable/commands/compile/get_context';
+import {
+    AZLE_PACKAGE_PATH,
+    GLOBAL_AZLE_CONFIG_DIR
+} from '../../../stable/utils/global_paths';
+import { CanisterConfig } from '../../../stable/utils/types';
+import { Context, WasmData } from '../../utils/types';
+import { getConsumer } from './open_value_sharing/consumer';
+
+export async function getContext(
+    canisterName: string,
+    canisterConfig: CanisterConfig
+): Promise<Context> {
+    const stableContext = getStableContext(canisterName, canisterConfig);
+
+    const canisterId = getCanisterId(canisterName);
+
+    const esmAliases = canisterConfig.custom?.esm_aliases ?? {};
+    const esmExternals = canisterConfig.custom?.esm_externals ?? [];
+
+    const reloadedJsPath = join('.azle', canisterName, 'main_reloaded.js');
+
+    const consumer = await getConsumer(canisterConfig);
+    const managementDid = (
+        await readFile(
+            join(AZLE_PACKAGE_PATH, 'canisters', 'management', 'ic.did')
+        )
+    ).toString();
+    const wasmData: WasmData = {
+        ...stableContext.wasmData,
+        consumer,
+        managementDid
+    };
+
+    const wasmedgeQuickJsName = `wasmedge-quickjs_${version}`;
+    const wasmedgeQuickJsPath = join(
+        GLOBAL_AZLE_CONFIG_DIR,
+        wasmedgeQuickJsName
+    );
+
+    return {
+        ...stableContext,
+        canisterId,
+        esmAliases,
+        esmExternals,
+        reloadedJsPath,
+        wasmData,
+        wasmedgeQuickJsPath
+    };
+}