Skip to content

Commit

Permalink
Merge pull request #1732 from demergent-labs/stable_file_fix
Browse files Browse the repository at this point in the history 
Stable file fix
  • Loading branch information
@lastmjs
lastmjs authored Apr 10, 2024
2 parents 2e68d48 + d1ce8f8 commit dde03c3
Show file tree
Hide file tree
Showing 8 changed files with 123 additions and 63 deletions.
8 changes: 4 additions & 4 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

132 changes: 89 additions & 43 deletions examples/large_files/test/tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,46 +106,19 @@ export function getTests(canisterId: string): Test[] {
return { Ok: false };
}
},
// Permanent Assets
generateTest(
origin,
'photos/people/george-washington.tif',
'permanent'
),
generateTest(origin, 'photos/places/dinosaurNM.jpg', 'permanent'),
generateTest(origin, 'photos/places/slc.jpg', 'permanent'),
generateTest(origin, 'photos/things/book.jpg', 'permanent'),
generateTest(origin, 'photos/things/utah-teapot.jpg', 'permanent'),
generateTest(
origin,
'text/subfolder/deep-sub-folder/deep.txt',
'permanent'
),
generateTest(
origin,
'text/subfolder/sibling-deep-sub-folder/deep.txt',
'permanent'
),
generateTest(origin, 'text/subfolder/other-thing.txt', 'permanent'),
generateTest(origin, 'text/thing.txt', 'permanent'),
generateTest(origin, 'text/thing.txt', 'permanent'),
generateTest(origin, 'text/single.txt', undefined, 'single_asset.txt'),

// Auto Generated Assets
// Edge Cases
generateTest(origin, 'test0B', 'auto'),
generateTest(origin, 'test1B', 'auto'),
generateTest(origin, `test${120 * 1024 * 1024 + 1}B`, 'auto'),
generateTest(origin, 'test2000001B', 'auto'),
// General Cases
generateTest(origin, 'test1KiB', 'auto'),
generateTest(origin, 'test10KiB', 'auto'),
generateTest(origin, 'test100KiB', 'auto'),
generateTest(origin, 'test1MiB', 'auto'),
generateTest(origin, 'test10MiB', 'auto'),
generateTest(origin, 'test100MiB', 'auto'),
generateTest(origin, 'test250MiB', 'auto'),
generateTest(origin, 'test1GiB', 'auto'),
...generateStandardFileTests('Upload', origin),
{
name: 'redeploy',
prep: async () => {
await generateTestFileOfSize(1, 'KiB');
await generateTestFileOfSize(10, 'KiB');
await generateTestFileOfSize(100, 'KiB');
execSync(`dfx deploy --upgrade-unchanged`, {
stdio: 'inherit'
});
}
},
...generateStandardFileTests('Stable check', origin),
// Manual Upload
{
name: 'test manual upload',
Expand All @@ -164,7 +137,7 @@ export function getTests(canisterId: string): Test[] {
return { Ok: (await response.json()) === true };
}
},
generateTest(origin, 'test150MiB', 'manual'),
generateTest('manual test', origin, 'test150MiB', 'manual'),
// TODO CI CD isn't working with the 2GiB tests so we're just going to have this one for local tests.
{
name: 'deploy',
Expand All @@ -180,7 +153,79 @@ export function getTests(canisterId: string): Test[] {
},
skip: true
},
{ ...generateTest(origin, 'test2GiB', 'auto'), skip: true }
{
...generateTest('large file', origin, 'test2GiB', 'auto'),
skip: true
}
];
}

function generateStandardFileTests(label: string, origin: string): Test[] {
return [
// Permanent Assets
generateTest(
label,
origin,
'photos/people/george-washington.tif',
'permanent'
),
generateTest(
label,
origin,
'photos/places/dinosaurNM.jpg',
'permanent'
),
generateTest(label, origin, 'photos/places/slc.jpg', 'permanent'),
generateTest(label, origin, 'photos/things/book.jpg', 'permanent'),
generateTest(
label,
origin,
'photos/things/utah-teapot.jpg',
'permanent'
),
generateTest(
label,
origin,
'text/subfolder/deep-sub-folder/deep.txt',
'permanent'
),
generateTest(
label,
origin,
'text/subfolder/sibling-deep-sub-folder/deep.txt',
'permanent'
),
generateTest(
label,
origin,
'text/subfolder/other-thing.txt',
'permanent'
),
generateTest(label, origin, 'text/thing.txt', 'permanent'),
generateTest(label, origin, 'text/thing.txt', 'permanent'),
generateTest(
label,
origin,
'text/single.txt',
undefined,
'single_asset.txt'
),

// Auto Generated Assets
// Edge Cases
generateTest(label, origin, 'test0B', 'auto'),
generateTest(label, origin, 'test1B', 'auto'),
generateTest(label, origin, `test${120 * 1024 * 1024 + 1}B`, 'auto'),
generateTest(label, origin, 'test2000001B', 'auto'),
// General Cases
generateTest(label, origin, 'test1KiB', 'auto'),
generateTest(label, origin, 'test10KiB', 'auto'),
generateTest(label, origin, 'test100KiB', 'auto'),
generateTest(label, origin, 'test1MiB', 'auto'),
generateTest(label, origin, 'test10MiB', 'auto'),
generateTest(label, origin, 'test100MiB', 'auto'),
generateTest(label, origin, 'test250MiB', 'auto'),
generateTest(label, origin, 'test1GiB', 'auto')
];
}

Expand All @@ -199,13 +244,14 @@ export function getTests(canisterId: string): Test[] {
* @returns
*/
function generateTest(
label: string,
origin: string,
canisterPath: string,
localDir?: string,
localPath?: string
): Test {
return {
name: `upload: ${canisterPath}`,
name: `${label}: ${canisterPath}`,
test: async () => {
const canisterFilePath = join('assets', canisterPath);
const localFilePath = join(
Expand Down
4 changes: 2 additions & 2 deletions examples/stable_memory/test/tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export function getTests(
const result = await stableMemoryCanister.stableSize();

return {
Ok: result === 0
Ok: result === 513
};
}
},
Expand All @@ -27,7 +27,7 @@ export function getTests(
const result = await stableMemoryCanister.stable64Size();

return {
Ok: result === 0n
Ok: result === 513n
};
}
},
Expand Down
2 changes: 1 addition & 1 deletion scripts/hash_file/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ async function getBytesToHash(

// Read the bytes
// TODO it would be great to get the size of the chunks from the canister, then we wouldn't have to every update this
const limit = 120 * 1024 * 1024; // Must be the same as on the canister end or hashes will not match
const limit = 75 * 1024 * 1024; // Must be the same as on the canister end or hashes will not match
let buffer = Buffer.alloc(limit); // Allocate a Buffer for reading

const fileReadResult = await file.read(buffer, 0, limit, position);
Expand Down
17 changes: 17 additions & 0 deletions src/compiler/file_uploader/upload_file.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,17 @@ import { stat, open } from 'fs/promises';
import { Dest, Src } from '.';
import { bytesToHumanReadable } from './bytes_to_human_readable';
import { UploaderActor } from './uploader_actor';
import { hashFile } from '../../../scripts/hash_file';

export async function uploadFile(
srcPath: Src,
destPath: Dest,
chunkSize: number,
actor: UploaderActor
) {
if (!(await shouldBeUploaded(srcPath, destPath, actor))) {
return;
}
const uploadStartTime = process.hrtime.bigint();
const fileSize = (await stat(srcPath)).size;
const file = await open(srcPath, 'r');
Expand Down Expand Up @@ -69,3 +73,16 @@ function calculatePercentComplete(
}
return (bytesComplete / Math.max(fileSize, 1)) * 100;
}

async function shouldBeUploaded(
srcPath: string,
destPath: string,
actor: UploaderActor
): Promise<boolean> {
const localHash = (await hashFile(srcPath)).toString('hex');
const canisterHashOption = await actor.get_file_hash(destPath);
if (canisterHashOption.length === 0) {
return true;
}
return localHash !== canisterHashOption[0];
}
5 changes: 3 additions & 2 deletions src/compiler/rust/canister/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ wasmi = "0.31.2"
sha2 = "0.10.8"
serde_json = "1.0.107"

# TODO transient feature can be removed once https://github.com/wasm-forge/stable-fs/issues/2 is resolved
ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "2d2edb382816e12da9bc81b786b7cd1a00d36735" , features = [
# TODO transient feature can be removed once https://github.com/demergent-labs/azle/issues/1731 is resolved
# ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "88bddc8190caf93a1e052f0513b5d6bc074929c3" }
ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "88bddc8190caf93a1e052f0513b5d6bc074929c3" , features = [
"transient",
] }

Expand Down
2 changes: 1 addition & 1 deletion src/compiler/rust/canister_methods/src/hash_file.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ pub fn get_hash_file() -> proc_macro2::TokenStream {
std::io::Seek::seek(&mut file, std::io::SeekFrom::Start(position)).unwrap();

// Read the bytes
let limit = 120 * 1024 * 1024; // This limit will be determine by how much hashing an update method can do without running out of cycles. It runs out somewhere between 120 and 135
let limit = 75 * 1024 * 1024; // This limit will be determine by how much hashing an update method can do without running out of cycles. It runs out somewhere between 75 and 80
// This limit must be the same as on the node side or else the hashes will not match
let mut buffer = vec![0; limit];
let bytes_read = std::io::Read::read(&mut file, &mut buffer).unwrap();
Expand Down
16 changes: 6 additions & 10 deletions src/compiler/rust/canister_methods/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,9 @@ pub fn canister_methods(_: TokenStream) -> TokenStream {
let init_method = quote! {
#[ic_cdk_macros::init]
fn init() {
// let polyfill_memory =
// MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
// ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory);
// TODO replace the line below with the lines above after https://github.com/wasm-forge/stable-fs/issues/2 is resolved
ic_wasi_polyfill::init(&[], &[#(#env_vars),*]);
let polyfill_memory =
MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory);

ASSETS_DIR.extract("/").unwrap();

Expand All @@ -90,11 +88,9 @@ pub fn canister_methods(_: TokenStream) -> TokenStream {
let post_upgrade_method = quote! {
#[ic_cdk_macros::post_upgrade]
fn post_upgrade() {
// let polyfill_memory =
// MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
// ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory);
// TODO replace the line below with the lines above after https://github.com/wasm-forge/stable-fs/issues/2 is resolved
ic_wasi_polyfill::init(&[], &[#(#env_vars),*]);
let polyfill_memory =
MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory);

ASSETS_DIR.extract("/").unwrap();

Expand Down

0 comments on commit dde03c3

Please sign in to comment.