diff --git a/Cargo.lock b/Cargo.lock index 156d8b4192..9ae71dcb4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -750,8 +750,8 @@ dependencies = [ [[package]] name = "ic-wasi-polyfill" -version = "0.3.14" -source = "git+https://github.com/wasm-forge/ic-wasi-polyfill?rev=2d2edb382816e12da9bc81b786b7cd1a00d36735#2d2edb382816e12da9bc81b786b7cd1a00d36735" +version = "0.4.0" +source = "git+https://github.com/wasm-forge/ic-wasi-polyfill?rev=88bddc8190caf93a1e052f0513b5d6bc074929c3#88bddc8190caf93a1e052f0513b5d6bc074929c3" dependencies = [ "function_name", "ic-cdk", @@ -1486,9 +1486,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "stable-fs" -version = "0.1.11" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3e74a9be516e35bacc8f64a686829670ef196bf99e1fba6b5b6b3125ee6132" +checksum = "3b5c2803cdd539ff47bb6265fab36e26949996c22ce4121fc338d3990b734ab4" dependencies = [ "bitflags 2.4.0", "ciborium", diff --git a/examples/large_files/test/tests.ts b/examples/large_files/test/tests.ts index c82cbbc4bd..f7bed96820 100644 --- a/examples/large_files/test/tests.ts +++ b/examples/large_files/test/tests.ts @@ -106,46 +106,19 @@ export function getTests(canisterId: string): Test[] { return { Ok: false }; } }, - // Permanent Assets - generateTest( - origin, - 'photos/people/george-washington.tif', - 'permanent' - ), - generateTest(origin, 'photos/places/dinosaurNM.jpg', 'permanent'), - generateTest(origin, 'photos/places/slc.jpg', 'permanent'), - generateTest(origin, 'photos/things/book.jpg', 'permanent'), - generateTest(origin, 'photos/things/utah-teapot.jpg', 'permanent'), - generateTest( - origin, - 'text/subfolder/deep-sub-folder/deep.txt', - 'permanent' - ), - generateTest( - origin, - 'text/subfolder/sibling-deep-sub-folder/deep.txt', - 'permanent' - ), - generateTest(origin, 'text/subfolder/other-thing.txt', 'permanent'), - generateTest(origin, 'text/thing.txt', 'permanent'), - generateTest(origin, 'text/thing.txt', 'permanent'), - generateTest(origin, 'text/single.txt', undefined, 'single_asset.txt'), - - // Auto Generated Assets - // Edge Cases - generateTest(origin, 'test0B', 'auto'), - generateTest(origin, 'test1B', 'auto'), - generateTest(origin, `test${120 * 1024 * 1024 + 1}B`, 'auto'), - generateTest(origin, 'test2000001B', 'auto'), - // General Cases - generateTest(origin, 'test1KiB', 'auto'), - generateTest(origin, 'test10KiB', 'auto'), - generateTest(origin, 'test100KiB', 'auto'), - generateTest(origin, 'test1MiB', 'auto'), - generateTest(origin, 'test10MiB', 'auto'), - generateTest(origin, 'test100MiB', 'auto'), - generateTest(origin, 'test250MiB', 'auto'), - generateTest(origin, 'test1GiB', 'auto'), + ...generateStandardFileTests('Upload', origin), + { + name: 'redeploy', + prep: async () => { + await generateTestFileOfSize(1, 'KiB'); + await generateTestFileOfSize(10, 'KiB'); + await generateTestFileOfSize(100, 'KiB'); + execSync(`dfx deploy --upgrade-unchanged`, { + stdio: 'inherit' + }); + } + }, + ...generateStandardFileTests('Stable check', origin), // Manual Upload { name: 'test manual upload', @@ -164,7 +137,7 @@ export function getTests(canisterId: string): Test[] { return { Ok: (await response.json()) === true }; } }, - generateTest(origin, 'test150MiB', 'manual'), + generateTest('manual test', origin, 'test150MiB', 'manual'), // TODO CI CD isn't working with the 2GiB tests so we're just going to have this one for local tests. { name: 'deploy', @@ -180,7 +153,79 @@ export function getTests(canisterId: string): Test[] { }, skip: true }, - { ...generateTest(origin, 'test2GiB', 'auto'), skip: true } + { + ...generateTest('large file', origin, 'test2GiB', 'auto'), + skip: true + } + ]; +} + +function generateStandardFileTests(label: string, origin: string): Test[] { + return [ + // Permanent Assets + generateTest( + label, + origin, + 'photos/people/george-washington.tif', + 'permanent' + ), + generateTest( + label, + origin, + 'photos/places/dinosaurNM.jpg', + 'permanent' + ), + generateTest(label, origin, 'photos/places/slc.jpg', 'permanent'), + generateTest(label, origin, 'photos/things/book.jpg', 'permanent'), + generateTest( + label, + origin, + 'photos/things/utah-teapot.jpg', + 'permanent' + ), + generateTest( + label, + origin, + 'text/subfolder/deep-sub-folder/deep.txt', + 'permanent' + ), + generateTest( + label, + origin, + 'text/subfolder/sibling-deep-sub-folder/deep.txt', + 'permanent' + ), + generateTest( + label, + origin, + 'text/subfolder/other-thing.txt', + 'permanent' + ), + generateTest(label, origin, 'text/thing.txt', 'permanent'), + generateTest(label, origin, 'text/thing.txt', 'permanent'), + generateTest( + label, + origin, + 'text/single.txt', + undefined, + 'single_asset.txt' + ), + + // Auto Generated Assets + // Edge Cases + generateTest(label, origin, 'test0B', 'auto'), + generateTest(label, origin, 'test1B', 'auto'), + generateTest(label, origin, `test${120 * 1024 * 1024 + 1}B`, 'auto'), + generateTest(label, origin, 'test2000001B', 'auto'), + // General Cases + generateTest(label, origin, 'test1KiB', 'auto'), + generateTest(label, origin, 'test10KiB', 'auto'), + generateTest(label, origin, 'test100KiB', 'auto'), + generateTest(label, origin, 'test1MiB', 'auto'), + generateTest(label, origin, 'test10MiB', 'auto'), + generateTest(label, origin, 'test100MiB', 'auto'), + generateTest(label, origin, 'test250MiB', 'auto'), + generateTest(label, origin, 'test1GiB', 'auto') ]; } @@ -199,13 +244,14 @@ export function getTests(canisterId: string): Test[] { * @returns */ function generateTest( + label: string, origin: string, canisterPath: string, localDir?: string, localPath?: string ): Test { return { - name: `upload: ${canisterPath}`, + name: `${label}: ${canisterPath}`, test: async () => { const canisterFilePath = join('assets', canisterPath); const localFilePath = join( diff --git a/examples/stable_memory/test/tests.ts b/examples/stable_memory/test/tests.ts index 5b14c511db..79fc86810e 100644 --- a/examples/stable_memory/test/tests.ts +++ b/examples/stable_memory/test/tests.ts @@ -17,7 +17,7 @@ export function getTests( const result = await stableMemoryCanister.stableSize(); return { - Ok: result === 0 + Ok: result === 513 }; } }, @@ -27,7 +27,7 @@ export function getTests( const result = await stableMemoryCanister.stable64Size(); return { - Ok: result === 0n + Ok: result === 513n }; } }, diff --git a/scripts/hash_file/index.ts b/scripts/hash_file/index.ts index e7b10aba00..6f26130bc0 100644 --- a/scripts/hash_file/index.ts +++ b/scripts/hash_file/index.ts @@ -34,7 +34,7 @@ async function getBytesToHash( // Read the bytes // TODO it would be great to get the size of the chunks from the canister, then we wouldn't have to every update this - const limit = 120 * 1024 * 1024; // Must be the same as on the canister end or hashes will not match + const limit = 75 * 1024 * 1024; // Must be the same as on the canister end or hashes will not match let buffer = Buffer.alloc(limit); // Allocate a Buffer for reading const fileReadResult = await file.read(buffer, 0, limit, position); diff --git a/src/compiler/file_uploader/upload_file.ts b/src/compiler/file_uploader/upload_file.ts index d69ba88aa2..33577be8b9 100644 --- a/src/compiler/file_uploader/upload_file.ts +++ b/src/compiler/file_uploader/upload_file.ts @@ -2,6 +2,7 @@ import { stat, open } from 'fs/promises'; import { Dest, Src } from '.'; import { bytesToHumanReadable } from './bytes_to_human_readable'; import { UploaderActor } from './uploader_actor'; +import { hashFile } from '../../../scripts/hash_file'; export async function uploadFile( srcPath: Src, @@ -9,6 +10,9 @@ export async function uploadFile( chunkSize: number, actor: UploaderActor ) { + if (!(await shouldBeUploaded(srcPath, destPath, actor))) { + return; + } const uploadStartTime = process.hrtime.bigint(); const fileSize = (await stat(srcPath)).size; const file = await open(srcPath, 'r'); @@ -69,3 +73,16 @@ function calculatePercentComplete( } return (bytesComplete / Math.max(fileSize, 1)) * 100; } + +async function shouldBeUploaded( + srcPath: string, + destPath: string, + actor: UploaderActor +): Promise { + const localHash = (await hashFile(srcPath)).toString('hex'); + const canisterHashOption = await actor.get_file_hash(destPath); + if (canisterHashOption.length === 0) { + return true; + } + return localHash !== canisterHashOption[0]; +} diff --git a/src/compiler/rust/canister/Cargo.toml b/src/compiler/rust/canister/Cargo.toml index fd8beb697c..a28955eb23 100644 --- a/src/compiler/rust/canister/Cargo.toml +++ b/src/compiler/rust/canister/Cargo.toml @@ -21,8 +21,9 @@ wasmi = "0.31.2" sha2 = "0.10.8" serde_json = "1.0.107" -# TODO transient feature can be removed once https://github.com/wasm-forge/stable-fs/issues/2 is resolved -ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "2d2edb382816e12da9bc81b786b7cd1a00d36735" , features = [ +# TODO transient feature can be removed once https://github.com/demergent-labs/azle/issues/1731 is resolved +# ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "88bddc8190caf93a1e052f0513b5d6bc074929c3" } +ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "88bddc8190caf93a1e052f0513b5d6bc074929c3" , features = [ "transient", ] } diff --git a/src/compiler/rust/canister_methods/src/hash_file.rs b/src/compiler/rust/canister_methods/src/hash_file.rs index ffed18ed21..e441abe640 100644 --- a/src/compiler/rust/canister_methods/src/hash_file.rs +++ b/src/compiler/rust/canister_methods/src/hash_file.rs @@ -39,7 +39,7 @@ pub fn get_hash_file() -> proc_macro2::TokenStream { std::io::Seek::seek(&mut file, std::io::SeekFrom::Start(position)).unwrap(); // Read the bytes - let limit = 120 * 1024 * 1024; // This limit will be determine by how much hashing an update method can do without running out of cycles. It runs out somewhere between 120 and 135 + let limit = 75 * 1024 * 1024; // This limit will be determine by how much hashing an update method can do without running out of cycles. It runs out somewhere between 75 and 80 // This limit must be the same as on the node side or else the hashes will not match let mut buffer = vec![0; limit]; let bytes_read = std::io::Read::read(&mut file, &mut buffer).unwrap(); diff --git a/src/compiler/rust/canister_methods/src/lib.rs b/src/compiler/rust/canister_methods/src/lib.rs index cd831c4025..c87c260f81 100644 --- a/src/compiler/rust/canister_methods/src/lib.rs +++ b/src/compiler/rust/canister_methods/src/lib.rs @@ -65,11 +65,9 @@ pub fn canister_methods(_: TokenStream) -> TokenStream { let init_method = quote! { #[ic_cdk_macros::init] fn init() { - // let polyfill_memory = - // MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254))); - // ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory); - // TODO replace the line below with the lines above after https://github.com/wasm-forge/stable-fs/issues/2 is resolved - ic_wasi_polyfill::init(&[], &[#(#env_vars),*]); + let polyfill_memory = + MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254))); + ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory); ASSETS_DIR.extract("/").unwrap(); @@ -90,11 +88,9 @@ pub fn canister_methods(_: TokenStream) -> TokenStream { let post_upgrade_method = quote! { #[ic_cdk_macros::post_upgrade] fn post_upgrade() { - // let polyfill_memory = - // MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254))); - // ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory); - // TODO replace the line below with the lines above after https://github.com/wasm-forge/stable-fs/issues/2 is resolved - ic_wasi_polyfill::init(&[], &[#(#env_vars),*]); + let polyfill_memory = + MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254))); + ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory); ASSETS_DIR.extract("/").unwrap();