update large files to use jest

examples/large_files/jest.config.js

@@ -0,0 +1,10 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+    preset: 'ts-jest',
+    testEnvironment: 'node',
+    transform: {
+        '^.+\\.ts$': 'ts-jest',
+        '^.+\\.js$': 'ts-jest'
+    },
+    transformIgnorePatterns: ['/node_modules/(?!(azle)/)'] // Make sure azle is transformed
+};

examples/large_files/package.json

@@ -1,7 +1,7 @@
 {
     "scripts": {
         "pretest": "ts-node --transpile-only --ignore=false test/pretest.ts",
-        "test": "ts-node --transpile-only --ignore=false test/test.ts"
+        "test": "jest"
     },
     "dependencies": {
         "azle": "0.22.0",
@@ -10,6 +10,8 @@
     },
     "devDependencies": {
         "@types/express": "^4.17.21",
+        "jest": "^29.7.0",
+        "ts-jest": "^29.1.5",
         "ts-node": "^10.9.1",
         "typescript": "^5.2.2"
     }

examples/large_files/test/authorization_tests.ts

@@ -0,0 +1,68 @@
+import { generateIdentity, getCanisterId } from 'azle/dfx';
+import { createActor } from 'azle/src/compiler/file_uploader/uploader_actor';
+import { expect, it, please, Test } from 'azle/test/jest';
+import { v4 } from 'uuid';
+
+export function getAuthorizationTests(): Test {
+    const unauthorizedUser = `test_azle_unauthorized_${v4()}`;
+
+    return () => {
+        please('set up unauthorized user', async () => {
+            generateIdentity(unauthorizedUser);
+        });
+
+        it('fails to upload from an unauthorized actor', async () => {
+            const destPath = 'assets/unauthorizedAddition';
+            const actor = await createActor(
+                getCanisterId('backend'),
+                unauthorizedUser
+            );
+
+            await expect(
+                actor.upload_file_chunk(
+                    destPath,
+                    0n,
+                    0n,
+                    Uint8Array.from([1, 2, 3, 4]),
+                    4n
+                )
+            ).rejects.toThrow(
+                /Not Authorized: only controllers of this canister may call this method/
+            );
+        });
+
+        it('fails to get hash status from an unauthorized actor', async () => {
+            const actor = await createActor(
+                getCanisterId('backend'),
+                unauthorizedUser
+            );
+            await expect(
+                actor.get_hash_status('assets/test0B')
+            ).rejects.toThrow(
+                /Not Authorized: only controllers of this canister may call this method/
+            );
+        });
+
+        it('fails to get hash from an unauthorized actor', async () => {
+            const actor = await createActor(
+                getCanisterId('backend'),
+                unauthorizedUser
+            );
+            await expect(actor.get_file_hash('assets/test0B')).rejects.toThrow(
+                /Not Authorized: only controllers of this canister may call this method/
+            );
+        });
+
+        it('fails to clear file and info from an unauthorized actor', async () => {
+            const actor = await createActor(
+                getCanisterId('backend'),
+                unauthorizedUser
+            );
+            await expect(
+                actor.clear_file_and_info('assets/test0B')
+            ).rejects.toThrow(
+                /Not Authorized: only controllers of this canister may call this method/
+            );
+        });
+    };
+}

examples/large_files/test/auto_tests.ts

@@ -0,0 +1,94 @@
+import { describe } from '@jest/globals';
+import { please, Test } from 'azle/test/jest';
+
+import { Unit } from '../../../scripts/file_generator';
+import { generateTestFileOfSize } from './generate_test_files';
+import { getAutoGeneratedFileName, verifyUpload } from './tests';
+
+const autoGenAutoUploadTests: [number, Unit][] = [
+    // Edge Cases
+    [0, 'B'],
+    [1, 'B'],
+    [120 * 1024 * 1024 + 1, 'B'], // One more byte than can be processed in a single hash_file_by_parts call
+    [2_000_000 + 1, 'B'], // One more byte that the message chunk size
+
+    // General Cases
+    // TODO Add tests for huge files after https://github.com/wasm-forge/stable-fs/issues/2 is resolved
+    [1, 'KiB'],
+    [10, 'KiB'],
+    [100, 'KiB'],
+    [1, 'MiB'],
+    [10, 'MiB'],
+    [100, 'MiB'],
+    [250, 'MiB'],
+    [1, 'GiB']
+];
+
+const permanentFilesTests: [string][] = [
+    // Permanent Assets
+    ['photos/people/george-washington.tif'],
+    ['photos/places/dinosaurNM.jpg'],
+    ['photos/places/slc.jpg'],
+    ['photos/things/book.jpg'],
+    ['photos/things/utah-teapot.jpg'],
+    ['text/subfolder/deep-sub-folder/deep.txt'],
+    ['text/subfolder/sibling-deep-sub-folder/deep.txt'],
+    ['text/subfolder/other-thing.txt'],
+    ['text/thing.txt'],
+    ['text/thing.txt']
+];
+
+const fileRenameDuringUploadTests: [string, string][] =
+    // TODO this is a bit of a pickle one
+    [['text/single.txt', 'single_asset.txt']];
+
+export function generateFiles(): Test {
+    return () => {
+        describe.each(autoGenAutoUploadTests)(
+            'prepare auto generated files locally for upload',
+            (size, units) => {
+                please(
+                    `generate file: ${getAutoGeneratedFileName(size, units)}`,
+                    async () => {
+                        await generateTestFileOfSize(size, units);
+                    },
+                    5 * 60 * 1_000
+                );
+            }
+        );
+    };
+}
+
+/**
+ * This suite of tests verifies the integrity and consistency of files that are
+ * automatically uploaded to the canister as specified in the `dfx.json`
+ * configuration.
+ *
+ * @param origin - The origin URL or path used in the tests.
+ * @returns A suite of Jest tests.
+ */
+export function getDfxConfigFileTests(origin: string): Test {
+    return () => {
+        describe.each(permanentFilesTests)(
+            'permanent files with same names locally as on canister',
+            (canisterPath) => {
+                verifyUpload(origin, canisterPath, 'permanent');
+            }
+        );
+
+        describe.each(fileRenameDuringUploadTests)(
+            'permanent files with different names locally than on canister',
+            (canisterPath, localPath) => {
+                verifyUpload(origin, canisterPath, undefined, localPath);
+            }
+        );
+
+        describe.each(autoGenAutoUploadTests)(
+            'auto generated files that were uploaded as canister was deployed',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                verifyUpload(origin, fileName, 'auto');
+            }
+        );
+    };
+}

examples/large_files/test/huge_file_tests.ts

@@ -0,0 +1,56 @@
+import { describe } from '@jest/globals';
+import { please, Test } from 'azle/test/jest';
+import { execSync } from 'child_process';
+import { rm } from 'fs-extra';
+import { join } from 'path';
+
+import { Unit } from '../../../scripts/file_generator';
+import { generateTestFileOfSize } from './generate_test_files';
+import { getAutoGeneratedFileName, verifyUpload } from './tests';
+
+const hugeAutoGenAutoUploadTests: [number, Unit][] = [[2, 'GiB']];
+
+export function hugeFilesTests(origin: string): Test {
+    return () => {
+        please(
+            'remove all other auto generated files so there is room for huge files',
+            async () => {
+                await rm(join('assets', 'auto'), {
+                    recursive: true,
+                    force: true
+                });
+            }
+        );
+
+        describe.each(hugeAutoGenAutoUploadTests)(
+            'generate huge files',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                please(
+                    `generate huge file: ${fileName}`,
+                    async () => {
+                        await generateTestFileOfSize(size, units);
+                    },
+                    10 * 60 * 1_000
+                );
+            }
+        );
+
+        please(
+            'redeploy the canister to remove files and reupload',
+            async () => {
+                execSync(`dfx deploy --upgrade-unchanged`, {
+                    stdio: 'inherit'
+                });
+            }
+        );
+
+        describe.each(hugeAutoGenAutoUploadTests)(
+            'verify huge files were uploaded correctly',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                verifyUpload(origin, fileName, 'auto', undefined);
+            }
+        );
+    };
+}

examples/large_files/test/manual_tests.ts

@@ -0,0 +1,61 @@
+import { describe } from '@jest/globals';
+import { expect, it, please, Test } from 'azle/test/jest';
+import { execSync } from 'child_process';
+import { join } from 'path';
+
+import { Unit } from '../../../scripts/file_generator';
+import { generateTestFileOfSize } from './generate_test_files';
+import { getAutoGeneratedFileName, verifyUpload } from './tests';
+
+export function manualTests(origin: string): Test {
+    const autoGenManualUploadTests: [number, Unit][] = [[150, 'MiB']];
+
+    return () => {
+        describe.each(autoGenManualUploadTests)(
+            'prepare auto generated files locally for manual upload',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                please(
+                    `generate file: ${fileName}`,
+                    async () => {
+                        await generateTestFileOfSize(size, units, 'manual');
+                    },
+                    2 * 60 * 1_000
+                );
+            }
+        );
+
+        describe.each(autoGenManualUploadTests)(
+            'initial manual upload of auto files',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                it('manually uploads files via azle command', async () => {
+                    execSync(
+                        `npx azle upload-assets backend ${join(
+                            'assets',
+                            'manual',
+                            fileName
+                        )} assets/test150MiB`,
+                        {
+                            stdio: 'inherit'
+                        }
+                    );
+
+                    const response = await fetch(
+                        `${origin}/exists?path=assets/${fileName}`
+                    );
+
+                    expect(await response.json()).toBe(true);
+                });
+            }
+        );
+
+        describe.each(autoGenManualUploadTests)(
+            'initial manual upload of auto files',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                verifyUpload(origin, fileName, 'manual');
+            }
+        );
+    };
+}

examples/large_files/test/pretest.ts

@@ -1,36 +1,11 @@
 import { execSync } from 'child_process';
-import { rm } from 'fs/promises';
-import { join } from 'path';
-
-import { generateTestFileOfSize } from './generateTestFiles';
 
 async function pretest() {
-    await rm(join('assets', 'auto'), { recursive: true, force: true });
-    // Edge Cases
-    await generateTestFileOfSize(0, 'B');
-    await generateTestFileOfSize(1, 'B');
-    await generateTestFileOfSize(120 * 1024 * 1024 + 1, 'B'); // One more byte than can be processed in a single hash_file_by_parts call
-    await generateTestFileOfSize(2_000_000 + 1, 'B'); // One more byte that the message chunk size
-
-    // General Cases
-    // TODO Add tests for huge files after https://github.com/wasm-forge/stable-fs/issues/2 is resolved
-    await generateTestFileOfSize(1, 'KiB');
-    await generateTestFileOfSize(10, 'KiB');
-    await generateTestFileOfSize(100, 'KiB');
-    await generateTestFileOfSize(1, 'MiB');
-    await generateTestFileOfSize(10, 'MiB');
-    await generateTestFileOfSize(100, 'MiB');
-    await generateTestFileOfSize(250, 'MiB');
-    await generateTestFileOfSize(1, 'GiB');
-    await generateTestFileOfSize(150, 'MiB', 'manual');
-
     execSync(`dfx canister uninstall-code backend || true`, {
         stdio: 'inherit'
     });
 
-    execSync(`dfx deploy`, {
-        stdio: 'inherit'
-    });
+    execSync(`dfx canister create backend`);
 }
 
 pretest();

examples/large_files/test/test.ts

@@ -1,5 +1,5 @@
 import { getCanisterId } from 'azle/dfx';
-import { runTests } from 'azle/test';
+import { runTests } from 'azle/test/jest';
 
 import { getTests } from './tests';