Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Wasm binary deploy process #1866

Merged
merged 40 commits into from
Jul 12, 2024
Merged
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
b094de1
first successful manual wasm binary manipulation!
lastmjs Jul 5, 2024
675b4c2
fix linters and prettiers
lastmjs Jul 5, 2024
3dd4d22
fix linters and prettiers
lastmjs Jul 5, 2024
fd30a6f
start working towards cleaning up unnecessary compilation code
lastmjs Jul 8, 2024
6a022d0
major cleanup of podman/docker
lastmjs Jul 8, 2024
21c12dd
manual binary manipulation working very well for all canister method …
lastmjs Jul 8, 2024
64aa180
add environment vars, get rid of regular assets option
lastmjs Jul 8, 2024
df12971
Merge branch 'main' of github.com:demergent-labs/azle into wasm_binar…
lastmjs Jul 9, 2024
6837c91
Merge branch 'main' of github.com:demergent-labs/azle into wasm_binar…
lastmjs Jul 9, 2024
b70f439
let us see if this fixes a lot of things
lastmjs Jul 10, 2024
259c06b
add return types
lastmjs Jul 10, 2024
b44b8e4
replace all occurrences of ts-node with tsx
lastmjs Jul 10, 2024
6c376b1
remove --skip-project
lastmjs Jul 10, 2024
667cdbe
add http canister optimization back in
lastmjs Jul 10, 2024
a2741e3
make many things asynchronous
lastmjs Jul 10, 2024
e3b8969
fix large files
lastmjs Jul 10, 2024
add4317
fix whoami example
lastmjs Jul 10, 2024
f209000
fix tfjs
lastmjs Jul 10, 2024
52bd8cb
implement indexes for class-based syntax
lastmjs Jul 10, 2024
11aa9c9
simplify the test.yml
lastmjs Jul 10, 2024
9fbd164
fix autoreload, compile reload_js into the binary
lastmjs Jul 10, 2024
bd29296
introduce concept of WasmData, get open value sharing to work
lastmjs Jul 11, 2024
e632060
add management canister did automatically
lastmjs Jul 11, 2024
3e555d8
manually upload the icrc files
lastmjs Jul 11, 2024
5415d8f
add better error handling to heartbeat, inspect_message, and pre_upgr…
lastmjs Jul 11, 2024
5bc965e
add panic hooks to init and post upgrade
lastmjs Jul 11, 2024
8c2a908
ckbtc fixed
lastmjs Jul 11, 2024
d7966e0
rip out all guard function stuff
lastmjs Jul 11, 2024
dae659e
fix the robust imports example
lastmjs Jul 12, 2024
98040d0
update binary
lastmjs Jul 12, 2024
d5916b3
get rid of trailing comma
lastmjs Jul 12, 2024
fb190d1
get rid of canister_methods
lastmjs Jul 12, 2024
39563c2
lots of cleanup
lastmjs Jul 12, 2024
c7ef94d
clean up fix candid
lastmjs Jul 12, 2024
6bb02ca
fix path issues
lastmjs Jul 12, 2024
4cc5def
add .npmignore
lastmjs Jul 12, 2024
efbc0f2
get rid of thirdparty.yml
lastmjs Jul 12, 2024
f53d603
misc cleanup
lastmjs Jul 12, 2024
3e41d42
add init/post_upgrade comment
lastmjs Jul 12, 2024
df7002a
cleanup
lastmjs Jul 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions src/compiler/file_watcher/file_watcher.ts
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ import { ok } from '../utils/result';

type ActorReloadJs = ActorSubclass<_SERVICE>;
interface _SERVICE {
reload_js: ActorMethod<[bigint, bigint, Uint8Array, bigint], void>;
reload_js: ActorMethod<[bigint, bigint, Uint8Array, bigint, number], void>;
}

// We have made this mutable to help with speed
@@ -23,11 +23,14 @@ const wasmedgeQuickJsPath = process.argv[5];
const esmAliases = JSON.parse(process.argv[6]);
const esmExternals = JSON.parse(process.argv[7]);
const canisterName = process.argv[8];
const postUpgradeIndex = Number(process.argv[9]);

// TODO https://github.com/demergent-labs/azle/issues/1664
watch(process.cwd(), {
ignored: ['**/.dfx/**', '**/.azle/**', '**/node_modules/**']
}).on('all', async (event, path) => {
const watcher = watch([`**/*.ts`, `**/*.js`], {
ignored: ['**/.dfx/**', '**/.azle/**', '**/node_modules/**', '**/target/**']
});

watcher.on('all', async (event, path) => {
if (actor === undefined) {
actor = await createActorReloadJs(canisterName);
}
@@ -37,7 +40,7 @@ watch(process.cwd(), {
console.info('path', path);
}

if (event === 'change' && (path.endsWith('.ts') || path.endsWith('.js'))) {
if (event === 'change') {
try {
await reloadJs(
actor,
@@ -90,7 +93,13 @@ async function reloadJs(
}

actor
.reload_js(timestamp, chunkNumber, chunk, BigInt(reloadedJs.length))
.reload_js(
timestamp,
chunkNumber,
chunk,
BigInt(reloadedJs.length),
postUpgradeIndex
)
.catch((error) => {
if (process.env.AZLE_VERBOSE === 'true') {
console.error(error);
@@ -117,7 +126,13 @@ async function createActorReloadJs(
({ IDL }) => {
return IDL.Service({
reload_js: IDL.Func(
[IDL.Nat64, IDL.Nat64, IDL.Vec(IDL.Nat8), IDL.Nat64],
[
IDL.Nat64,
IDL.Nat64,
IDL.Vec(IDL.Nat8),
IDL.Nat64,
IDL.Int32
],
[],
[]
)
16 changes: 6 additions & 10 deletions src/compiler/file_watcher/file_watcher_loader.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('ts-node').register({
transpileOnly: true,
ignore: [`node_modules/(?!azle)`],
compilerOptions: {
module: 'commonjs',
allowJs: true
}
});
require('./file_watcher.ts');
#!/usr/bin/env node

console.log('spawing the file watcher');

import 'tsx';
import('./file_watcher.ts');
6 changes: 4 additions & 2 deletions src/compiler/file_watcher/setup_file_watcher.ts
Original file line number Diff line number Diff line change
@@ -11,7 +11,8 @@ export function setupFileWatcher(
wasmedgeQuickJsPath: string,
esmAliases: Record<string, string>,
esmExternals: string[],
canisterName: string
canisterName: string,
postUpgradeIndex: number
): void {
try {
// TODO should we check that this was successful in killing
@@ -44,7 +45,8 @@ export function setupFileWatcher(
wasmedgeQuickJsPath,
JSON.stringify(esmAliases),
JSON.stringify(esmExternals),
canisterName
canisterName,
postUpgradeIndex.toString()
],
{
detached: true,
21 changes: 11 additions & 10 deletions src/compiler/index.ts
Original file line number Diff line number Diff line change
@@ -57,16 +57,6 @@ async function azle(): Promise<void> {
esmExternals
} = await getNamesAfterCli();

setupFileWatcher(
reloadedJsPath,
canisterId,
canisterConfig.main,
wasmedgeQuickJsPath,
esmAliases,
esmExternals,
canisterName
);

await time(
`\nBuilding canister ${green(canisterName)}`,
'default',
@@ -103,6 +93,17 @@ async function azle(): Promise<void> {
canisterJavaScript
);

setupFileWatcher(
reloadedJsPath,
canisterId,
canisterConfig.main,
wasmedgeQuickJsPath,
esmAliases,
esmExternals,
canisterName,
canisterMethods.post_upgrade?.index ?? -1
);

// This is for the dfx.json candid property
await writeFile(candidPath, candid);

34 changes: 9 additions & 25 deletions src/compiler/rust/canister_methods/src/lib.rs
Original file line number Diff line number Diff line change
@@ -72,16 +72,14 @@ pub fn canister_methods(_: TokenStream) -> TokenStream {
MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
ic_wasi_polyfill::init_with_memory(&[], &env_vars, polyfill_memory);

ASSETS_DIR.extract("/").unwrap();

let js = get_js_code();

initialize_js(std::str::from_utf8(&js).unwrap(), true, function_index, pass_arg_data);

ic_cdk::spawn(async {
let consumer = get_consumer("consumer.json").unwrap();
open_value_sharing::init(&consumer).await;
});
// ic_cdk::spawn(async {
// let consumer = get_consumer("consumer.json").unwrap();
// open_value_sharing::init(&consumer).await;
// });
}
};

@@ -101,16 +99,14 @@ pub fn canister_methods(_: TokenStream) -> TokenStream {
MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
ic_wasi_polyfill::init_with_memory(&[], &env_vars, polyfill_memory);

ASSETS_DIR.extract("/").unwrap();

let js = get_js_code();

initialize_js(std::str::from_utf8(&js).unwrap(), false, function_index, pass_arg_data);

ic_cdk::spawn(async {
let consumer = get_consumer("consumer.json").unwrap();
open_value_sharing::init(&consumer).await;
});
// ic_cdk::spawn(async {
// let consumer = get_consumer("consumer.json").unwrap();
// open_value_sharing::init(&consumer).await;
// });
}
};

@@ -211,27 +207,15 @@ pub fn canister_methods(_: TokenStream) -> TokenStream {
}
});

let reload_js = reload_js::get_reload_js(&compiler_info.env_vars);
let reload_js = reload_js::get_reload_js();

let upload_file_chunk = upload_file_chunk::get_upload_file_chunk();

quote! {
static ASSETS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/src/assets");

#init_method

#post_upgrade_method

// #pre_upgrade_method

// #inspect_message_method

// #heartbeat_method

// #(#query_methods)*

// #(#update_methods)*

#[ic_cdk_macros::query]
fn __get_candid_interface_tmp_hack() -> String {
std::str::from_utf8(CANDID)
54 changes: 23 additions & 31 deletions src/compiler/rust/canister_methods/src/reload_js.rs
Original file line number Diff line number Diff line change
@@ -1,42 +1,34 @@
use quote::quote;

pub fn get_reload_js(env_vars: &Vec<(String, String)>) -> proc_macro2::TokenStream {
let azle_autoreload_env_var = env_vars.iter().find(|(key, _)| key == "AZLE_AUTORELOAD");
pub fn get_reload_js() -> proc_macro2::TokenStream {
return quote! {
#[ic_cdk_macros::update(guard = guard_against_non_controllers)]
bdemann marked this conversation as resolved.
Show resolved Hide resolved
fn reload_js(timestamp: u64, chunk_number: u64, js_bytes: Vec<u8>, total_len: u64, function_index: i32) {
RELOADED_JS_TIMESTAMP.with(|reloaded_js_timestamp| {
let mut reloaded_js_timestamp_mut = reloaded_js_timestamp.borrow_mut();

if let Some((_, value)) = azle_autoreload_env_var {
if value == "true" {
return quote! {
#[ic_cdk_macros::update(guard = guard_against_non_controllers)]
fn reload_js(timestamp: u64, chunk_number: u64, js_bytes: Vec<u8>, total_len: u64) {
RELOADED_JS_TIMESTAMP.with(|reloaded_js_timestamp| {
let mut reloaded_js_timestamp_mut = reloaded_js_timestamp.borrow_mut();

if timestamp > *reloaded_js_timestamp_mut {
*reloaded_js_timestamp_mut = timestamp;

RELOADED_JS.with(|reloaded_js| {
let mut reloaded_js_mut = reloaded_js.borrow_mut();
reloaded_js_mut.clear();
});
}
});
if timestamp > *reloaded_js_timestamp_mut {
*reloaded_js_timestamp_mut = timestamp;

RELOADED_JS.with(|reloaded_js| {
let mut reloaded_js_mut = reloaded_js.borrow_mut();
reloaded_js_mut.insert(chunk_number, js_bytes);
reloaded_js_mut.clear();
});
}
});

let reloaded_js_complete_bytes: Vec<u8> = reloaded_js_mut.values().flat_map(|v| v.clone()).collect();
RELOADED_JS.with(|reloaded_js| {
let mut reloaded_js_mut = reloaded_js.borrow_mut();
reloaded_js_mut.insert(chunk_number, js_bytes);

if reloaded_js_complete_bytes.len() as u64 == total_len {
let js_string = String::from_utf8_lossy(&reloaded_js_complete_bytes);
initialize_js(&js_string, false);
ic_cdk::println!("Azle: Reloaded canister JavaScript");
}
});
let reloaded_js_complete_bytes: Vec<u8> = reloaded_js_mut.values().flat_map(|v| v.clone()).collect();

if reloaded_js_complete_bytes.len() as u64 == total_len {
let js_string = String::from_utf8_lossy(&reloaded_js_complete_bytes);
initialize_js(&js_string, false, function_index, 1); // TODO should the last arg be 0?
ic_cdk::println!("Azle: Reloaded canister JavaScript");
}
};
});
}
}

quote! {}
};
}
Binary file modified static_canister_template.wasm
Binary file not shown.