Merge pull request #1843 from demergent-labs/jest_large_files_example

update large files to use jest

examples/internet_identity/test/tests.ts

@@ -75,7 +75,7 @@ export function getTests(canisterName: string): Test {
                 Principal.fromText(
                     whoamiPrincipalTextBefore as string
                 ).isAnonymous()
-            ).toBe(false);
+            ).toBe(true);
 
             // TODO I do not know why this wait is required
             await new Promise((resolve) => setTimeout(resolve, 5_000));
@@ -94,9 +94,9 @@ export function getTests(canisterName: string): Test {
                 Principal.fromText(
                     whoamiPrincipalTextAfter as string
                 ).isAnonymous()
-            ).toBe(true);
+            ).toBe(false);
 
             await browser.close();
-        });
+        }, 60_000);
     };
 }

examples/large_files/jest.config.js

@@ -0,0 +1,10 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+    preset: 'ts-jest',
+    testEnvironment: 'node',
+    transform: {
+        '^.+\\.ts$': 'ts-jest',
+        '^.+\\.js$': 'ts-jest'
+    },
+    transformIgnorePatterns: ['/node_modules/(?!(azle)/)'] // Make sure azle is transformed
+};

examples/large_files/package.json

@@ -1,7 +1,7 @@
 {
     "scripts": {
         "pretest": "ts-node --transpile-only --ignore=false test/pretest.ts",
-        "test": "ts-node --transpile-only --ignore=false test/test.ts"
+        "test": "jest"
     },
     "dependencies": {
         "azle": "0.22.0",
@@ -10,6 +10,8 @@
     },
     "devDependencies": {
         "@types/express": "^4.17.21",
+        "jest": "^29.7.0",
+        "ts-jest": "^29.1.5",
         "ts-node": "^10.9.1",
         "typescript": "^5.2.2"
     }

examples/large_files/test/authorization_tests.ts

@@ -0,0 +1,68 @@
+import { generateIdentity, getCanisterId } from 'azle/dfx';
+import { createActor } from 'azle/src/compiler/file_uploader/uploader_actor';
+import { expect, it, please, Test } from 'azle/test/jest';
+import { v4 } from 'uuid';
+
+export function getAuthorizationTests(): Test {
+    const unauthorizedUser = `test_azle_unauthorized_${v4()}`;
+
+    return () => {
+        please('set up unauthorized user', async () => {
+            generateIdentity(unauthorizedUser);
+        });
+
+        it('fails to upload from an unauthorized actor', async () => {
+            const destPath = 'assets/unauthorizedAddition';
+            const actor = await createActor(
+                getCanisterId('backend'),
+                unauthorizedUser
+            );
+
+            await expect(
+                actor.upload_file_chunk(
+                    destPath,
+                    0n,
+                    0n,
+                    Uint8Array.from([1, 2, 3, 4]),
+                    4n
+                )
+            ).rejects.toThrow(
+                /Not Authorized: only controllers of this canister may call this method/
+            );
+        });
+
+        it('fails to get hash status from an unauthorized actor', async () => {
+            const actor = await createActor(
+                getCanisterId('backend'),
+                unauthorizedUser
+            );
+            await expect(
+                actor.get_hash_status('assets/test0B')
+            ).rejects.toThrow(
+                /Not Authorized: only controllers of this canister may call this method/
+            );
+        });
+
+        it('fails to get hash from an unauthorized actor', async () => {
+            const actor = await createActor(
+                getCanisterId('backend'),
+                unauthorizedUser
+            );
+            await expect(actor.get_file_hash('assets/test0B')).rejects.toThrow(
+                /Not Authorized: only controllers of this canister may call this method/
+            );
+        });
+
+        it('fails to clear file and info from an unauthorized actor', async () => {
+            const actor = await createActor(
+                getCanisterId('backend'),
+                unauthorizedUser
+            );
+            await expect(
+                actor.clear_file_and_info('assets/test0B')
+            ).rejects.toThrow(
+                /Not Authorized: only controllers of this canister may call this method/
+            );
+        });
+    };
+}

examples/large_files/test/auto_tests.ts

@@ -0,0 +1,97 @@
+import { describe } from '@jest/globals';
+import { please, Test } from 'azle/test/jest';
+import { join } from 'path';
+
+import { Unit } from '../../../scripts/file_generator';
+import { generateTestFileOfSize } from './generate_test_files';
+import { getAutoGeneratedFileName, verifyUpload } from './tests';
+
+const autoGenAutoUploadFileInfos: [number, Unit][] = [
+    // Edge Cases
+    [0, 'B'],
+    [1, 'B'],
+    [120 * 1024 * 1024 + 1, 'B'], // One more byte than can be processed in a single hash_file_by_parts call
+    [2_000_000 + 1, 'B'], // One more byte that the message chunk size
+
+    // General Cases
+    [1, 'KiB'],
+    [10, 'KiB'],
+    [100, 'KiB'],
+    [1, 'MiB'],
+    [10, 'MiB'],
+    [100, 'MiB'],
+    [250, 'MiB'],
+    [1, 'GiB']
+];
+
+const permanentFiles: string[] = [
+    'photos/people/george-washington.tif',
+    'photos/places/dinosaurNM.jpg',
+    'photos/places/slc.jpg',
+    'photos/things/book.jpg',
+    'photos/things/utah-teapot.jpg',
+    'text/subfolder/deep-sub-folder/deep.txt',
+    'text/subfolder/sibling-deep-sub-folder/deep.txt',
+    'text/subfolder/other-thing.txt',
+    'text/thing.txt',
+    'text/thing.txt'
+];
+
+const renamedPermanentFiles: [string, string][] = [
+    ['text/single.txt', 'single_asset.txt']
+];
+
+export function generateFiles(): Test {
+    return () => {
+        describe.each(autoGenAutoUploadFileInfos)(
+            'prepare auto generated files locally for upload',
+            (size, units) => {
+                please(
+                    `generate file: ${getAutoGeneratedFileName(size, units)}`,
+                    async () => {
+                        await generateTestFileOfSize(size, units);
+                    },
+                    10 * 60 * 1_000
+                );
+            }
+        );
+    };
+}
+
+/**
+ * This suite of tests verifies the integrity and consistency of files that are
+ * automatically uploaded to the canister as specified in the `dfx.json`
+ * configuration.
+ *
+ * @param origin - The origin URL or path used in the tests.
+ * @returns A suite of Jest tests.
+ */
+export function getDfxConfigFileTests(origin: string): Test {
+    return () => {
+        describe.each(permanentFiles)(
+            'permanent files with same names locally as on canister that were uploaded as canister was deployed',
+            (canisterPath) => {
+                verifyUpload(
+                    origin,
+                    join('permanent', canisterPath),
+                    canisterPath
+                );
+            }
+        );
+
+        describe.each(renamedPermanentFiles)(
+            'permanent files with different names locally than on canister that were uploaded as canister was deployed',
+            (canisterPath, localPath) => {
+                verifyUpload(origin, localPath, canisterPath);
+            }
+        );
+
+        describe.each(autoGenAutoUploadFileInfos)(
+            'auto generated files that were uploaded as canister was deployed',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                verifyUpload(origin, join('auto', fileName), fileName);
+            }
+        );
+    };
+}

examples/large_files/test/huge_file_tests.ts

@@ -0,0 +1,56 @@
+import { describe } from '@jest/globals';
+import { please, Test } from 'azle/test/jest';
+import { execSync } from 'child_process';
+import { rm } from 'fs-extra';
+import { join } from 'path';
+
+import { Unit } from '../../../scripts/file_generator';
+import { generateTestFileOfSize } from './generate_test_files';
+import { getAutoGeneratedFileName, verifyUpload } from './tests';
+
+const hugeAutoGenAutoUploadFileInfos: [number, Unit][] = [[2, 'GiB']];
+
+export function hugeFilesTests(origin: string): Test {
+    return () => {
+        please(
+            'remove all other auto generated files so there is room for huge files',
+            async () => {
+                await rm(join('assets', 'auto'), {
+                    recursive: true,
+                    force: true
+                });
+            }
+        );
+
+        describe.each(hugeAutoGenAutoUploadFileInfos)(
+            'generate huge files',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                please(
+                    `generate huge file: ${fileName}`,
+                    async () => {
+                        await generateTestFileOfSize(size, units);
+                    },
+                    10 * 60 * 1_000
+                );
+            }
+        );
+
+        please(
+            'redeploy the canister to remove files and reupload',
+            async () => {
+                execSync(`dfx deploy --upgrade-unchanged`, {
+                    stdio: 'inherit'
+                });
+            }
+        );
+
+        describe.each(hugeAutoGenAutoUploadFileInfos)(
+            'verify huge files were uploaded correctly',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                verifyUpload(origin, join('auto', fileName), fileName);
+            }
+        );
+    };
+}

examples/large_files/test/manual_tests.ts

@@ -0,0 +1,65 @@
+import { describe } from '@jest/globals';
+import { expect, it, please, Test } from 'azle/test/jest';
+import { execSync } from 'child_process';
+import { join } from 'path';
+
+import { Unit } from '../../../scripts/file_generator';
+import { generateTestFileOfSize } from './generate_test_files';
+import { getAutoGeneratedFileName, verifyUpload } from './tests';
+
+export function manualTests(origin: string): Test {
+    const autoGenManualUploadFileInfos: [number, Unit][] = [[150, 'MiB']];
+
+    return () => {
+        describe.each(autoGenManualUploadFileInfos)(
+            'prepare auto generated files locally for manual upload',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                please(
+                    `generate file: ${fileName}`,
+                    async () => {
+                        await generateTestFileOfSize(size, units, 'manual');
+                    },
+                    10 * 60 * 1_000
+                );
+            }
+        );
+
+        describe.each(autoGenManualUploadFileInfos)(
+            'initial manual upload of auto files',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                it(
+                    'manually uploads files via azle command',
+                    async () => {
+                        execSync(
+                            `npx azle upload-assets backend ${join(
+                                'assets',
+                                'manual',
+                                fileName
+                            )} assets/test150MiB`,
+                            {
+                                stdio: 'inherit'
+                            }
+                        );
+
+                        const response = await fetch(
+                            `${origin}/exists?path=assets/${fileName}`
+                        );
+
+                        expect(await response.json()).toBe(true);
+                    },
+                    10 * 60 * 1_000
+                );
+            }
+        );
+
+        describe.each(autoGenManualUploadFileInfos)(
+            'initial manual upload of auto files',
+            (size, units) => {
+                const fileName = getAutoGeneratedFileName(size, units);
+                verifyUpload(origin, join('manual', fileName), fileName);
+            }
+        );
+    };
+}

examples/large_files/test/pretest.ts

@@ -1,36 +1,15 @@
 import { execSync } from 'child_process';
-import { rm } from 'fs/promises';
-import { join } from 'path';
-
-import { generateTestFileOfSize } from './generateTestFiles';
 
 async function pretest() {
-    await rm(join('assets', 'auto'), { recursive: true, force: true });
-    // Edge Cases
-    await generateTestFileOfSize(0, 'B');
-    await generateTestFileOfSize(1, 'B');
-    await generateTestFileOfSize(120 * 1024 * 1024 + 1, 'B'); // One more byte than can be processed in a single hash_file_by_parts call
-    await generateTestFileOfSize(2_000_000 + 1, 'B'); // One more byte that the message chunk size
-
-    // General Cases
-    // TODO Add tests for huge files after https://github.com/wasm-forge/stable-fs/issues/2 is resolved
-    await generateTestFileOfSize(1, 'KiB');
-    await generateTestFileOfSize(10, 'KiB');
-    await generateTestFileOfSize(100, 'KiB');
-    await generateTestFileOfSize(1, 'MiB');
-    await generateTestFileOfSize(10, 'MiB');
-    await generateTestFileOfSize(100, 'MiB');
-    await generateTestFileOfSize(250, 'MiB');
-    await generateTestFileOfSize(1, 'GiB');
-    await generateTestFileOfSize(150, 'MiB', 'manual');
-
     execSync(`dfx canister uninstall-code backend || true`, {
         stdio: 'inherit'
     });
 
-    execSync(`dfx deploy`, {
-        stdio: 'inherit'
-    });
+    // Since a lot of this test revolves around the upload process and the post
+    // install scripts the call to dfx deploy is inside the tests. Canister
+    // create is called here to make sure we have the right canister id for the
+    // origin
+    execSync(`dfx canister create backend`);
 }
 
 pretest();

examples/large_files/test/test.ts

@@ -1,5 +1,5 @@
 import { getCanisterId } from 'azle/dfx';
-import { runTests } from 'azle/test';
+import { runTests } from 'azle/test/jest';
 
 import { getTests } from './tests';