Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Stable file fix #1732

Merged
merged 5 commits into from
Apr 10, 2024
Merged

Stable file fix #1732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9843ac7
get stable files working
bdemann Mar 29, 2024
81fc120
modify files before redeploy
bdemann Apr 3, 2024
9a436cc
disable transient
bdemann Apr 3, 2024
569f284
update default size of the stable memory
bdemann Apr 3, 2024
d1ce8f8
pr fixes
bdemann Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

132 changes: 89 additions & 43 deletions examples/large_files/test/tests.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,46 +106,19 @@ export function getTests(canisterId: string): Test[] {
return { Ok: false };
}
},
// Permanent Assets
generateTest(
origin,
'photos/people/george-washington.tif',
'permanent'
),
generateTest(origin, 'photos/places/dinosaurNM.jpg', 'permanent'),
generateTest(origin, 'photos/places/slc.jpg', 'permanent'),
generateTest(origin, 'photos/things/book.jpg', 'permanent'),
generateTest(origin, 'photos/things/utah-teapot.jpg', 'permanent'),
generateTest(
origin,
'text/subfolder/deep-sub-folder/deep.txt',
'permanent'
),
generateTest(
origin,
'text/subfolder/sibling-deep-sub-folder/deep.txt',
'permanent'
),
generateTest(origin, 'text/subfolder/other-thing.txt', 'permanent'),
generateTest(origin, 'text/thing.txt', 'permanent'),
generateTest(origin, 'text/thing.txt', 'permanent'),
generateTest(origin, 'text/single.txt', undefined, 'single_asset.txt'),

// Auto Generated Assets
// Edge Cases
generateTest(origin, 'test0B', 'auto'),
generateTest(origin, 'test1B', 'auto'),
generateTest(origin, `test${120 * 1024 * 1024 + 1}B`, 'auto'),
generateTest(origin, 'test2000001B', 'auto'),
// General Cases
generateTest(origin, 'test1KiB', 'auto'),
generateTest(origin, 'test10KiB', 'auto'),
generateTest(origin, 'test100KiB', 'auto'),
generateTest(origin, 'test1MiB', 'auto'),
generateTest(origin, 'test10MiB', 'auto'),
generateTest(origin, 'test100MiB', 'auto'),
generateTest(origin, 'test250MiB', 'auto'),
generateTest(origin, 'test1GiB', 'auto'),
...generateStandardFileTests('Upload', origin),
{
name: 'redeploy',
prep: async () => {
await generateTestFileOfSize(1, 'KiB');
await generateTestFileOfSize(10, 'KiB');
await generateTestFileOfSize(100, 'KiB');
execSync(`dfx deploy --upgrade-unchanged`, {
stdio: 'inherit'
});
}
},
...generateStandardFileTests('Stable check', origin),
// Manual Upload
{
name: 'test manual upload',
Expand All @@ -164,7 +137,7 @@ export function getTests(canisterId: string): Test[] {
return { Ok: (await response.json()) === true };
}
},
generateTest(origin, 'test150MiB', 'manual'),
generateTest('manual test', origin, 'test150MiB', 'manual'),
// TODO CI CD isn't working with the 2GiB tests so we're just going to have this one for local tests.
{
name: 'deploy',
Expand All @@ -180,7 +153,79 @@ export function getTests(canisterId: string): Test[] {
},
skip: true
},
{ ...generateTest(origin, 'test2GiB', 'auto'), skip: true }
{
...generateTest('large file', origin, 'test2GiB', 'auto'),
skip: true
}
];
}

function generateStandardFileTests(label: string, origin: string): Test[] {
return [
// Permanent Assets
generateTest(
label,
origin,
'photos/people/george-washington.tif',
'permanent'
),
generateTest(
label,
origin,
'photos/places/dinosaurNM.jpg',
'permanent'
),
generateTest(label, origin, 'photos/places/slc.jpg', 'permanent'),
generateTest(label, origin, 'photos/things/book.jpg', 'permanent'),
generateTest(
label,
origin,
'photos/things/utah-teapot.jpg',
'permanent'
),
generateTest(
label,
origin,
'text/subfolder/deep-sub-folder/deep.txt',
'permanent'
),
generateTest(
label,
origin,
'text/subfolder/sibling-deep-sub-folder/deep.txt',
'permanent'
),
generateTest(
label,
origin,
'text/subfolder/other-thing.txt',
'permanent'
),
generateTest(label, origin, 'text/thing.txt', 'permanent'),
generateTest(label, origin, 'text/thing.txt', 'permanent'),
generateTest(
label,
origin,
'text/single.txt',
undefined,
'single_asset.txt'
),

// Auto Generated Assets
// Edge Cases
generateTest(label, origin, 'test0B', 'auto'),
generateTest(label, origin, 'test1B', 'auto'),
generateTest(label, origin, `test${120 * 1024 * 1024 + 1}B`, 'auto'),
generateTest(label, origin, 'test2000001B', 'auto'),
// General Cases
generateTest(label, origin, 'test1KiB', 'auto'),
generateTest(label, origin, 'test10KiB', 'auto'),
generateTest(label, origin, 'test100KiB', 'auto'),
generateTest(label, origin, 'test1MiB', 'auto'),
generateTest(label, origin, 'test10MiB', 'auto'),
generateTest(label, origin, 'test100MiB', 'auto'),
generateTest(label, origin, 'test250MiB', 'auto'),
generateTest(label, origin, 'test1GiB', 'auto')
];
}

Expand All @@ -199,13 +244,14 @@ export function getTests(canisterId: string): Test[] {
* @returns
*/
function generateTest(
label: string,
origin: string,
canisterPath: string,
localDir?: string,
localPath?: string
): Test {
return {
name: `upload: ${canisterPath}`,
name: `${label}: ${canisterPath}`,
test: async () => {
const canisterFilePath = join('assets', canisterPath);
const localFilePath = join(
Expand Down
4 changes: 2 additions & 2 deletions examples/stable_memory/test/tests.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export function getTests(
const result = await stableMemoryCanister.stableSize();

return {
Ok: result === 0
Ok: result === 513
};
}
},
Expand All @@ -27,7 +27,7 @@ export function getTests(
const result = await stableMemoryCanister.stable64Size();

return {
Ok: result === 0n
Ok: result === 513n
};
}
},
Expand Down
2 changes: 1 addition & 1 deletion scripts/hash_file/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ async function getBytesToHash(

// Read the bytes
// TODO it would be great to get the size of the chunks from the canister, then we wouldn't have to every update this
const limit = 120 * 1024 * 1024; // Must be the same as on the canister end or hashes will not match
const limit = 75 * 1024 * 1024; // Must be the same as on the canister end or hashes will not match
let buffer = Buffer.alloc(limit); // Allocate a Buffer for reading

const fileReadResult = await file.read(buffer, 0, limit, position);
Expand Down
17 changes: 17 additions & 0 deletions src/compiler/file_uploader/upload_file.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,17 @@ import { stat, open } from 'fs/promises';
import { Dest, Src } from '.';
import { bytesToHumanReadable } from './bytes_to_human_readable';
import { UploaderActor } from './uploader_actor';
import { hashFile } from '../../../scripts/hash_file';

export async function uploadFile(
srcPath: Src,
destPath: Dest,
chunkSize: number,
actor: UploaderActor
) {
if (!(await shouldBeUploaded(srcPath, destPath, actor))) {
lastmjs marked this conversation as resolved.
Show resolved Hide resolved
return;
}
const uploadStartTime = process.hrtime.bigint();
const fileSize = (await stat(srcPath)).size;
const file = await open(srcPath, 'r');
Expand Down Expand Up @@ -69,3 +73,16 @@ function calculatePercentComplete(
}
return (bytesComplete / Math.max(fileSize, 1)) * 100;
}

async function shouldBeUploaded(
srcPath: string,
destPath: string,
actor: UploaderActor
): Promise<boolean> {
const localHash = (await hashFile(srcPath)).toString('hex');
const canisterHashOption = await actor.get_file_hash(destPath);
if (canisterHashOption.length === 0) {
return true;
}
return localHash !== canisterHashOption[0];
}
5 changes: 3 additions & 2 deletions src/compiler/rust/canister/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ wasmi = "0.31.2"
sha2 = "0.10.8"
serde_json = "1.0.107"

# TODO transient feature can be removed once https://github.com/wasm-forge/stable-fs/issues/2 is resolved
ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "2d2edb382816e12da9bc81b786b7cd1a00d36735" , features = [
# TODO transient feature can be removed once https://github.com/demergent-labs/azle/issues/1731 is resolved
# ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "88bddc8190caf93a1e052f0513b5d6bc074929c3" }
ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", rev = "88bddc8190caf93a1e052f0513b5d6bc074929c3" , features = [
"transient",
] }

Expand Down
2 changes: 1 addition & 1 deletion src/compiler/rust/canister_methods/src/hash_file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ pub fn get_hash_file() -> proc_macro2::TokenStream {
std::io::Seek::seek(&mut file, std::io::SeekFrom::Start(position)).unwrap();

// Read the bytes
let limit = 120 * 1024 * 1024; // This limit will be determine by how much hashing an update method can do without running out of cycles. It runs out somewhere between 120 and 135
let limit = 75 * 1024 * 1024; // This limit will be determine by how much hashing an update method can do without running out of cycles. It runs out somewhere between 75 and 80
// This limit must be the same as on the node side or else the hashes will not match
let mut buffer = vec![0; limit];
let bytes_read = std::io::Read::read(&mut file, &mut buffer).unwrap();
Expand Down
16 changes: 6 additions & 10 deletions src/compiler/rust/canister_methods/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,9 @@ pub fn canister_methods(_: TokenStream) -> TokenStream {
let init_method = quote! {
#[ic_cdk_macros::init]
fn init() {
// let polyfill_memory =
// MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
// ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory);
// TODO replace the line below with the lines above after https://github.com/wasm-forge/stable-fs/issues/2 is resolved
ic_wasi_polyfill::init(&[], &[#(#env_vars),*]);
let polyfill_memory =
MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory);

ASSETS_DIR.extract("/").unwrap();

Expand All @@ -90,11 +88,9 @@ pub fn canister_methods(_: TokenStream) -> TokenStream {
let post_upgrade_method = quote! {
#[ic_cdk_macros::post_upgrade]
fn post_upgrade() {
// let polyfill_memory =
// MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
// ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory);
// TODO replace the line below with the lines above after https://github.com/wasm-forge/stable-fs/issues/2 is resolved
ic_wasi_polyfill::init(&[], &[#(#env_vars),*]);
let polyfill_memory =
MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
ic_wasi_polyfill::init_with_memory(&[], &[#(#env_vars),*], polyfill_memory);

ASSETS_DIR.extract("/").unwrap();

Expand Down
Loading