Performance counter property tests

.github/workflows/test.yml

@@ -261,6 +261,7 @@ jobs:
                     "tests/property/ic_api/id",
                     "tests/property/ic_api/instruction_counter",
                     "tests/property/ic_api/is_controller",
+                    "tests/property/ic_api/performance_counter",
                     "tests/property/ic_api/time",
                     "tests/property/ic_api/trap"
                   ]

tests/property/ic_api/performance_counter/.gitignore

@@ -0,0 +1,4 @@
+.azle
+.dfx
+dfx_generated
+node_modules

tests/property/ic_api/performance_counter/dfx.json

@@ -0,0 +1,13 @@
+{
+    "canisters": {
+        "canister": {
+            "type": "azle",
+            "main": "src/index.ts",
+            "candid_gen": "automatic",
+            "declarations": {
+                "output": "test/dfx_generated/canister",
+                "node_compatibility": true
+            }
+        }
+    }
+}

tests/property/ic_api/performance_counter/jest.config.js

@@ -0,0 +1,12 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+    preset: 'ts-jest',
+    testEnvironment: 'node',
+    bail: true,
+    testTimeout: 100_000_000,
+    transform: {
+        '^.+\\.ts$': ['ts-jest', { isolatedModules: true }],
+        '^.+\\.js$': 'ts-jest'
+    },
+    transformIgnorePatterns: ['/node_modules/(?!(azle)/)'] // Make sure azle is transformed
+};

tests/property/ic_api/performance_counter/package.json

@@ -0,0 +1,16 @@
+{
+    "scripts": {
+        "pretest": "tsx test/pretest.ts",
+        "test": "jest"
+    },
+    "dependencies": {
+        "azle": "^0.23.0"
+    },
+    "devDependencies": {
+        "@dfinity/agent": "^2.0.0",
+        "jest": "^29.7.0",
+        "ts-jest": "^29.1.4",
+        "tsx": "^4.15.7",
+        "typescript": "^5.2.2"
+    }
+}

tests/property/ic_api/performance_counter/src/index.ts

@@ -0,0 +1,43 @@
+import { chunk, IDL, performanceCounter, query, update } from 'azle';
+
+export default class {
+    @query([IDL.Nat32], IDL.Nat64)
+    async queryPerformanceCounter0(loops: number): Promise<bigint> {
+        await sum(loops, false);
+
+        return performanceCounter(0);
+    }
+
+    @update([IDL.Nat32], IDL.Nat64)
+    async updatePerformanceCounter0(loops: number): Promise<bigint> {
+        await sum(loops, false);
+
+        return performanceCounter(0);
+    }
+
+    @query([IDL.Nat32], IDL.Nat64)
+    async queryPerformanceCounter1(loops: number): Promise<bigint> {
+        await sum(loops, false);
+
+        return performanceCounter(1);
+    }
+
+    @update([IDL.Nat32], IDL.Nat64)
+    async updatePerformanceCounter1(loops: number): Promise<bigint> {
+        await sum(loops, true);
+
+        return performanceCounter(1);
+    }
+}
+
+async function sum(loops: number, shouldChunk: boolean): Promise<void> {
+    let _sum = 0;
+
+    for (let i = 0; i < loops; i++) {
+        _sum += (i % 100) * (i % 100);
+
+        if (shouldChunk && i % 100_000 === 0) {
+            await chunk();
+        }
+    }
+}

tests/property/ic_api/performance_counter/test/pretest.ts

@@ -0,0 +1,17 @@
+import { execSync } from 'child_process';
+
+function pretest(): void {
+    execSync(`dfx canister uninstall-code canister || true`, {
+        stdio: 'inherit'
+    });
+
+    execSync(`dfx deploy canister`, {
+        stdio: 'inherit'
+    });
+
+    execSync(`dfx generate canister`, {
+        stdio: 'inherit'
+    });
+}
+
+pretest();

tests/property/ic_api/performance_counter/test/test.ts

@@ -0,0 +1,5 @@
+import { runTests } from 'azle/test';
+
+import { getTests } from './tests';
+
+runTests(getTests());

tests/property/ic_api/performance_counter/test/tests.ts

@@ -0,0 +1,231 @@
+import { defaultParams, expect, getCanisterActor, it, Test } from 'azle/test';
+import fc from 'fast-check';
+
+import { _SERVICE as Actor } from './dfx_generated/canister/canister.did';
+
+export function getTests(): Test {
+    return () => {
+        it('should calculate performanceCounter(0) instructions accurately from a query method', async () => {
+            const actor = await getCanisterActor<Actor>('canister');
+
+            await fc.assert(
+                fc.asyncProperty(
+                    fc.nat({
+                        max: 100 // Our algorithm for determinstically checking the number of instructions based on the number of loops breaks down soon after 100 iterations
+                    }),
+                    async (loops) => {
+                        const instructions =
+                            await actor.queryPerformanceCounter0(loops);
+
+                        const {
+                            loops0Instructions,
+                            zeroToOneDelta,
+                            oneToTwoDelta
+                        } = await getBaseInstructionCountsQuery0();
+
+                        expect(instructions).toStrictEqual(
+                            loops0Instructions +
+                                (loops === 0
+                                    ? 0n
+                                    : loops === 1
+                                    ? zeroToOneDelta
+                                    : zeroToOneDelta +
+                                      (BigInt(loops) - 1n) * oneToTwoDelta)
+                        );
+                    }
+                ),
+                defaultParams
+            );
+        });
+
+        // This test is much different than the query test because for some reason
+        // the number of instructions per call and even per call with the same number of loops
+        // is not consistent in a few ways. For example calling loops 0 repeatedly returns
+        // a different number of instructions in the first few calls
+        it('should calculate performanceCounter(0) instructions accurately from an update method', async () => {
+            const actor = await getCanisterActor<Actor>('canister');
+
+            await fc.assert(
+                fc.asyncProperty(
+                    fc.nat({
+                        max: 1_000_000
+                    }),
+                    async (loops) => {
+                        const instructionsLoops0 =
+                            await actor.updatePerformanceCounter0(0);
+
+                        const instructions0 =
+                            await actor.updatePerformanceCounter0(loops);
+                        const instructions1 =
+                            await actor.updatePerformanceCounter0(loops);
+
+                        const instructionsAfter0 =
+                            await actor.updatePerformanceCounter0(loops + 1);
+                        const instructionsAfter1 =
+                            await actor.updatePerformanceCounter0(loops + 1);
+
+                        expect(instructionsLoops0).not.toStrictEqual(0n);
+
+                        expect(
+                            bigIntAbs(instructions0 - instructions1)
+                        ).toBeLessThan(1_000n);
+
+                        expect(
+                            bigIntAbs(instructionsAfter0 - instructionsAfter1)
+                        ).toBeLessThan(1_000n);
+
+                        expect(instructions0).toBeLessThan(instructionsAfter0);
+                        expect(instructions1).toBeLessThan(instructionsAfter1);
+                    }
+                ),
+                defaultParams
+            );
+        });
+
+        it('should calculate performanceCounter(1) instructions accurately from a query method', async () => {
+            const actor = await getCanisterActor<Actor>('canister');
+
+            await fc.assert(
+                fc.asyncProperty(
+                    fc.nat({
+                        max: 100 // Our algorithm for determinstically checking the number of instructions based on the number of loops breaks down soon after 100 iterations
+                    }),
+                    async (loops) => {
+                        const instructions =
+                            await actor.queryPerformanceCounter1(loops);
+
+                        const {
+                            loops0Instructions,
+                            zeroToOneDelta,
+                            oneToTwoDelta
+                        } = await getBaseInstructionCountsQuery1();
+
+                        expect(instructions).toStrictEqual(
+                            loops0Instructions +
+                                (loops === 0
+                                    ? 0n
+                                    : loops === 1
+                                    ? zeroToOneDelta
+                                    : zeroToOneDelta +
+                                      (BigInt(loops) - 1n) * oneToTwoDelta)
+                        );
+                    }
+                ),
+                defaultParams
+            );
+        });
+
+        // This test is much different than the query test because for some reason
+        // the number of instructions per call and even per call with the same number of loops
+        // is not consistent in a few ways. For example calling loops 0 repeatedly returns
+        // a different number of instructions in the first few calls
+        it('should calculate performanceCounter(1) instructions accurately from an update method', async () => {
+            const actor = await getCanisterActor<Actor>('canister');
+
+            await fc.assert(
+                fc.asyncProperty(
+                    fc.nat({
+                        max: 1_000_000
+                    }),
+                    async (loops) => {
+                        const instructionsLoops0 =
+                            await actor.updatePerformanceCounter1(0);
+
+                        const instructions0 =
+                            await actor.updatePerformanceCounter1(loops);
+                        const instructions1 =
+                            await actor.updatePerformanceCounter1(loops);
+
+                        const instructionsAfter0 =
+                            await actor.updatePerformanceCounter1(loops + 100);
+                        const instructionsAfter1 =
+                            await actor.updatePerformanceCounter1(loops + 100);
+
+                        expect(instructionsLoops0).not.toStrictEqual(0n);
+
+                        expect(
+                            percentageDifferenceBigInt(
+                                instructions0,
+                                instructions1
+                            )
+                        ).toBeLessThanOrEqual(5n);
+
+                        expect(
+                            percentageDifferenceBigInt(
+                                instructionsAfter0,
+                                instructionsAfter1
+                            )
+                        ).toBeLessThanOrEqual(5n);
+
+                        expect(instructions0).toBeLessThan(instructionsAfter0);
+                        expect(instructions1).toBeLessThan(instructionsAfter1);
+                    }
+                ),
+                defaultParams
+            );
+        });
+    };
+}
+
+type BaseInstructionCounts = {
+    loops0Instructions: bigint;
+    loops1Instructions: bigint;
+    loops2Instructions: bigint;
+    zeroToOneDelta: bigint;
+    oneToTwoDelta: bigint;
+};
+
+async function getBaseInstructionCountsQuery0(): Promise<BaseInstructionCounts> {
+    const actor = await getCanisterActor<Actor>('canister');
+
+    await actor.queryPerformanceCounter0(0);
+    await actor.queryPerformanceCounter0(0);
+
+    const loops0Instructions = await actor.queryPerformanceCounter0(0);
+    const loops1Instructions = await actor.queryPerformanceCounter0(1);
+    const loops2Instructions = await actor.queryPerformanceCounter0(2);
+
+    const zeroToOneDelta = loops1Instructions - loops0Instructions;
+    const oneToTwoDelta = loops2Instructions - loops1Instructions;
+
+    return {
+        loops0Instructions,
+        loops1Instructions,
+        loops2Instructions,
+        zeroToOneDelta,
+        oneToTwoDelta
+    };
+}
+
+async function getBaseInstructionCountsQuery1(): Promise<BaseInstructionCounts> {
+    const actor = await getCanisterActor<Actor>('canister');
+
+    await actor.queryPerformanceCounter1(0);
+    await actor.queryPerformanceCounter1(0);
+
+    const loops0Instructions = await actor.queryPerformanceCounter1(0);
+    const loops1Instructions = await actor.queryPerformanceCounter1(1);
+    const loops2Instructions = await actor.queryPerformanceCounter1(2);
+
+    const zeroToOneDelta = loops1Instructions - loops0Instructions;
+    const oneToTwoDelta = loops2Instructions - loops1Instructions;
+
+    return {
+        loops0Instructions,
+        loops1Instructions,
+        loops2Instructions,
+        zeroToOneDelta,
+        oneToTwoDelta
+    };
+}
+
+function bigIntAbs(x: bigint): bigint {
+    return x < 0 ? -x : x;
+}
+
+function percentageDifferenceBigInt(value1: bigint, value2: bigint): bigint {
+    const difference = value1 > value2 ? value1 - value2 : value2 - value1;
+    const average = (value1 + value2) / 2n;
+    const percentageDifference = (difference * 100n) / average;
+    return percentageDifference;
+}

tests/property/ic_api/performance_counter/tsconfig.json

@@ -0,0 +1,10 @@
+{
+    "compilerOptions": {
+        "strict": true,
+        "target": "ES2020",
+        "module": "ES2020",
+        "moduleResolution": "node",
+        "allowJs": true,
+        "outDir": "HACK_BECAUSE_OF_ALLOW_JS"
+    }
+}