attempt to fix autoreload file watching

src/build/experimental/commands/compile/file_watcher/file_watcher.ts

@@ -0,0 +1,138 @@
+import { Actor, ActorMethod, ActorSubclass } from '@dfinity/agent';
+import { watch } from 'chokidar';
+import { writeFile } from 'fs/promises';
+
+import { createAuthenticatedAgent } from '../../../../../../dfx';
+import { generateUploaderIdentity } from '../../upload_assets/uploader_identity';
+import { compile as compileJavaScript } from '../javascript';
+
+type ActorReloadJs = ActorSubclass<_SERVICE>;
+interface _SERVICE {
+    _azle_reload_js: ActorMethod<
+        [bigint, bigint, Uint8Array, bigint, number],
+        void
+    >;
+}
+
+// We have made this mutable to help with speed
+// We don't want to have to create the agent on each file change
+let actor: ActorReloadJs | undefined;
+
+const reloadedJsPath = process.argv[2];
+const canisterId = process.argv[3];
+const mainPath = process.argv[4];
+const wasmedgeQuickJsPath = process.argv[5];
+const esmAliases = JSON.parse(process.argv[6]);
+const esmExternals = JSON.parse(process.argv[7]);
+const canisterName = process.argv[8];
+const postUpgradeIndex = Number(process.argv[9]);
+
+// TODO https://github.com/demergent-labs/azle/issues/1664
+const watcher = watch([`**/*.ts`, `**/*.js`], {
+    ignored: ['**/.dfx/**', '**/.azle/**', '**/node_modules/**', '**/target/**']
+});
+
+watcher.on('all', async (event, path) => {
+    if (actor === undefined) {
+        actor = await createActorReloadJs(canisterName);
+    }
+
+    if (process.env.AZLE_VERBOSE === 'true') {
+        console.info('event', event);
+        console.info('path', path);
+    }
+
+    if (event === 'change') {
+        try {
+            await reloadJs(
+                actor,
+                reloadedJsPath,
+                mainPath,
+                wasmedgeQuickJsPath
+            );
+        } catch (error) {
+            console.error(error);
+        }
+    }
+});
+
+async function reloadJs(
+    actor: ActorReloadJs,
+    reloadedJsPath: string,
+    mainPath: string,
+    wasmedgeQuickJsPath: string
+): Promise<void> {
+    const javaScript = await compileJavaScript(
+        mainPath,
+        wasmedgeQuickJsPath,
+        esmAliases,
+        esmExternals
+    );
+
+    const reloadedJs = Buffer.from(javaScript);
+
+    const chunkSize = 2_000_000; // The current message limit is about 2 MiB
+    const timestamp = process.hrtime.bigint();
+    let chunkNumber = 0n;
+
+    for (let i = 0; i < reloadedJs.length; i += chunkSize) {
+        const chunk = reloadedJs.slice(i, i + chunkSize);
+
+        if (process.env.AZLE_VERBOSE === 'true') {
+            console.info(
+                `Uploading chunk: ${timestamp}, ${chunkNumber}, ${chunk.length}, ${reloadedJs.length}`
+            );
+        }
+
+        actor
+            ._azle_reload_js(
+                timestamp,
+                chunkNumber,
+                chunk,
+                BigInt(reloadedJs.length),
+                postUpgradeIndex
+            )
+            .catch((error) => {
+                if (process.env.AZLE_VERBOSE === 'true') {
+                    console.error(error);
+                }
+            });
+
+        chunkNumber += 1n;
+    }
+
+    if (process.env.AZLE_VERBOSE === 'true') {
+        console.info(`Finished uploading chunks`);
+    }
+
+    await writeFile(reloadedJsPath, reloadedJs);
+}
+
+async function createActorReloadJs(
+    canisterName: string
+): Promise<ActorReloadJs> {
+    const identityName = generateUploaderIdentity(canisterName);
+    const agent = await createAuthenticatedAgent(identityName);
+
+    return Actor.createActor(
+        ({ IDL }) => {
+            return IDL.Service({
+                _azle_reload_js: IDL.Func(
+                    [
+                        IDL.Nat64,
+                        IDL.Nat64,
+                        IDL.Vec(IDL.Nat8),
+                        IDL.Nat64,
+                        IDL.Int32
+                    ],
+                    [],
+                    []
+                )
+            });
+        },
+        {
+            agent,
+            canisterId
+        }
+    );
+}

src/build/experimental/commands/compile/file_watcher/file_watcher_loader.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+import 'tsx';
+import('./file_watcher.ts');

src/build/experimental/commands/compile/file_watcher/setup_file_watcher.ts

@@ -0,0 +1,61 @@
+import { spawn } from 'child_process';
+import { join } from 'path';
+
+import { execSyncPretty } from '../../../../stable/utils/exec_sync_pretty';
+import { AZLE_PACKAGE_PATH } from '../../../../stable/utils/global_paths';
+
+export function setupFileWatcher(
+    reloadedJsPath: string,
+    canisterId: string,
+    mainPath: string,
+    wasmedgeQuickJsPath: string,
+    esmAliases: Record<string, string>,
+    esmExternals: string[],
+    canisterName: string,
+    postUpgradeIndex: number
+): void {
+    try {
+        // TODO should we check that this was successful in killing
+        // TODO the process and then warn the user if not?
+        // TODO should we figure out why the || true
+        // TODO does not result in a 0 exit code
+        // TODO and look into removing the try catch?
+        execSyncPretty(`pkill -f ./file_watcher_loader.js || true`);
+    } catch (error) {
+        // For some reason pkill throws even with || true
+    }
+
+    if (process.env.AZLE_AUTORELOAD !== 'true') {
+        return;
+    }
+
+    const watcherProcess = spawn(
+        'node',
+        [
+            join(
+                AZLE_PACKAGE_PATH,
+                'src',
+                'build',
+                'experimental',
+                'commands',
+                'compile',
+                'file_watcher',
+                'file_watcher_loader.js'
+            ),
+            reloadedJsPath,
+            canisterId,
+            mainPath,
+            wasmedgeQuickJsPath,
+            JSON.stringify(esmAliases),
+            JSON.stringify(esmExternals),
+            canisterName,
+            postUpgradeIndex.toString()
+        ],
+        {
+            detached: true,
+            stdio: 'inherit'
+        }
+    );
+
+    watcherProcess.unref();
+}

src/build/experimental/commands/compile/get_context.ts

@@ -1,6 +1,7 @@
 import { readFile } from 'fs/promises';
 import { join } from 'path';
 
+import { getCanisterId } from '../../../../../dfx';
 import { version } from '../../../../../package.json';
 import { getContext as getStableContext } from '../../../stable/commands/compile/get_context';
 import {
@@ -17,9 +18,13 @@ export async function getContext(
 ): Promise<Context> {
     const stableContext = getStableContext(canisterName, canisterConfig);
 
+    const canisterId = getCanisterId(canisterName);
+
     const esmAliases = canisterConfig.custom?.esm_aliases ?? {};
     const esmExternals = canisterConfig.custom?.esm_externals ?? [];
 
+    const reloadedJsPath = join('.azle', canisterName, 'main_reloaded.js');
+
     const consumer = await getConsumer(canisterConfig);
     const managementDid = (
         await readFile(
@@ -40,8 +45,10 @@ export async function getContext(
 
     return {
         ...stableContext,
+        canisterId,
         esmAliases,
         esmExternals,
+        reloadedJsPath,
         wasmData,
         wasmedgeQuickJsPath
     };

src/build/experimental/commands/compile/index.ts

@@ -7,6 +7,7 @@ import {
 import { execSyncPretty } from '../../../stable/utils/exec_sync_pretty';
 import { CanisterConfig } from '../../../stable/utils/types';
 import { getCandidAndMethodMeta } from './candid_and_method_meta';
+import { setupFileWatcher } from './file_watcher/setup_file_watcher';
 import { getContext } from './get_context';
 import { compile as compileJavaScript } from './javascript';
 import { getWasmBinary } from './wasm_binary';
@@ -17,11 +18,13 @@ export async function runCommand(
     ioType: IOType
 ): Promise<void> {
     const {
+        canisterId,
         canisterPath,
         candidPath,
         esmAliases,
         esmExternals,
         main,
+        reloadedJsPath,
         wasmBinaryPath,
         wasmData,
         wasmedgeQuickJsPath
@@ -65,6 +68,17 @@ export async function runCommand(
     );
 
     buildAssets(canisterConfig, ioType);
+
+    setupFileWatcher(
+        reloadedJsPath,
+        canisterId,
+        main,
+        wasmedgeQuickJsPath,
+        esmAliases,
+        esmExternals,
+        canisterName,
+        methodMeta.post_upgrade?.index ?? -1
+    );
 }
 
 function buildAssets(canisterConfig: CanisterConfig, ioType: IOType): void {

src/build/experimental/utils/types.ts

@@ -5,8 +5,10 @@ import {
 import { Consumer } from '../commands/compile/open_value_sharing/consumer';
 
 export type Context = {
+    canisterId: string;
     esmAliases: Record<string, string>;
     esmExternals: string[];
+    reloadedJsPath: string;
     wasmData: WasmData;
     wasmedgeQuickJsPath: string;
 } & StableContext;