Various system tweaks.

Makefile.toml

@@ -21,7 +21,7 @@ cargo test -p argus-lib --lib export_bindings --locked
 [tasks.evaluation]
 script = """
 cd ide && depot build && cd ..
-node ide/packages/evaluation/dist/evaluation.cjs ./data
+node ide/packages/evaluation/dist/evaluation.cjs -s ./data
 python3 -m webbrowser http://localhost:8080/eval
 ./scripts/evaluation.scm
 """

ide/packages/common/src/lib.ts

@@ -30,9 +30,11 @@ export const ConfigConsts = {
 export type PanoptesInitialData = {
   data: [Filename, ObligationsInBody[]][];
   target?: ErrorJumpTargetInfo;
+  evalMode?: EvaluationMode;
 };
 
 export type SystemSpec = Omit<IssueOptions, "logText">;
+export type EvaluationMode = "release" | "rank" | "random";
 
 export type PanoptesConfig = PanoptesInitialData &
   (

ide/packages/evaluation/src/main.ts

@@ -1,17 +1,28 @@
 import { BodyBundle } from "@argus/common/bindings";
-import { Filename, Result } from "@argus/common/lib";
+import { EvaluationMode, Filename, Result } from "@argus/common/lib";
 import { ExecNotifyOpts, execNotify as _execNotify } from "@argus/system";
 import fs from "fs";
 import _ from "lodash";
 import path from "path";
-import { chromium } from "playwright";
+import {
+  BrowserContext,
+  ElementHandle,
+  Page,
+  chromium,
+  selectors,
+} from "playwright";
 import tmp from "tmp";
 //@ts-ignore
 import uniqueFilename from "unique-filename";
 
 import { webHtml } from "./page";
+import { rootCauses } from "./rootCauses";
 import { PORT, fileServer } from "./serve";
 
+declare global {
+  var debugging: boolean;
+}
+
 // See: https://doc.rust-lang.org/cargo/reference/external-tools.html
 //      https://doc.rust-lang.org/rustc/json.html
 // for up-to-date information on the structure of diagnostics output.
@@ -45,38 +56,81 @@ async function sleep(waitTime: number) {
   return new Promise(resolve => setTimeout(resolve, waitTime));
 }
 
+async function forFileInBundle<T>(
+  bundles: BodyBundle[],
+  f: (filename: Filename, bundles: BodyBundle[]) => Promise<T>
+) {
+  const groupedByFilename = _.groupBy(bundles, b => b.filename);
+  return await Promise.all(
+    _.map(groupedByFilename, async (bundles, filename) => {
+      return f(filename, bundles);
+    })
+  );
+}
+
+async function openPage(
+  context: BrowserContext,
+  filename: string,
+  bundles: BodyBundle[],
+  evalMode: EvaluationMode
+) {
+  const tmpobj = tmp.fileSync({ postfix: ".html" });
+  const html = webHtml("EVAL", filename, bundles, evalMode);
+  fs.writeSync(tmpobj.fd, html);
+  const page = await context.newPage();
+  await page.goto(`file://${tmpobj.name}`, {
+    waitUntil: "domcontentloaded",
+    timeout: 30_000,
+  });
+  return page;
+}
+
+async function expandBottomUpView(page: Page) {
+  let bs = await page.getByText("Bottom Up").all();
+  try {
+    // await Promise.all(
+    //   _.map(bs, async b => {
+    //     try {
+    //       await b.waitFor({ state: "visible" });
+    //     } catch (e: any) {}
+    //   })
+    // );
+    await Promise.all(
+      _.map(bs, async b => {
+        try {
+          await b.click();
+        } catch (e: any) {}
+      })
+    );
+  } catch (e: any) {
+    console.debug("Error clicking bottom up", e);
+  }
+}
+
 // Take bundled argus output and take a screenshot of the tree loaded into the browser.
 async function argusScreenshots(
   outDir: fs.PathLike,
   bundles: BodyBundle[],
   title: string = "Argus Output"
 ) {
-  const browser = await chromium.launch({ headless: true });
+  const browser = await chromium.launch({ headless: !global.debugging });
   const context = await browser.newContext();
 
   const innerDoScreenshot = async (
-    tmpobj: tmp.FileResult,
     out: string,
     filename: Filename,
     bundles: BodyBundle[]
   ) => {
-    const html = webHtml(title, filename, bundles);
-    fs.writeSync(tmpobj.fd, html);
-    const page = await context.newPage();
-    await page.goto(`file://${tmpobj.name}`);
-    await sleep(6000);
-    await page.screenshot({ path: out, fullPage: false });
+    const page = await openPage(context, filename, bundles, "rank");
+    await expandBottomUpView(page);
+    await page.screenshot({ path: out, fullPage: true });
   };
 
-  const groupedByFilename = _.groupBy(bundles, b => b.filename);
-  return await Promise.all(
-    _.map(groupedByFilename, async (bundles, filename) => {
-      const tmpobj = tmp.fileSync({ postfix: ".html" });
-      const outfile = uniqueFilename(outDir) + ".png";
-      await innerDoScreenshot(tmpobj, outfile, filename, bundles);
-      return { filename, outfile };
-    })
-  );
+  return forFileInBundle(bundles, async (filename, bundles) => {
+    const outfile = uniqueFilename(outDir) + ".png";
+    await innerDoScreenshot(outfile, filename, bundles);
+    return { filename, outfile };
+  });
 }
 
 async function argusData(dir: string) {
@@ -116,6 +170,145 @@ const testCases = [
   "uom",
 ] as const;
 
+async function runForJson<T>(func: () => Promise<T>) {
+  return JSON.stringify(JSON.stringify(await func()));
+}
+
+async function runRandom(N: number) {
+  fileServer().listen(PORT);
+  const workspaceDir = path.resolve(__dirname, "..", "workspaces");
+  const browser = await chromium.launch({ headless: !global.debugging });
+  const context = await browser.newContext();
+  context.setDefaultTimeout(5_000);
+
+  const innerF = async (workspace: string, causes: any[]) => {
+    const fullSubdir = path.resolve(workspaceDir, workspace);
+    const argusBundles = await argusData(fullSubdir);
+
+    if (!("Ok" in argusBundles)) throw new Error("Argus failed");
+    const fileBundles = (await forFileInBundle(
+      argusBundles.Ok,
+      async (filename, bundles) => [filename, bundles]
+    )) as [string, BodyBundle[]][];
+
+    let results = [];
+    for (const [filename, bundles] of fileBundles) {
+      const cause = _.find(causes, cause => filename.endsWith(cause.file));
+
+      if (!cause) {
+        console.error(`No cause found for ${filename} in ${workspace}`);
+        continue;
+      }
+      console.debug(`Running ${N} samples for ${filename}`);
+      let ranks = await Promise.all(
+        _.times(N, async () => {
+          const page = await openPage(context, filename, bundles, "random");
+          await sleep(3000);
+          await expandBottomUpView(page);
+          await sleep(3000);
+          try {
+            const goals = await page
+              .locator(".BottomUpArea .EvalGoal")
+              .filter({ hasText: cause.message })
+              .all();
+
+            const ranksStr = await Promise.all(
+              _.map(goals, goal => goal.getAttribute("data-rank"))
+            );
+
+            return _.min(_.map(_.compact(ranksStr), r => Number(r)));
+          } catch (e) {
+            return undefined;
+          } finally {
+            await page.close();
+          }
+        })
+      );
+
+      const noUndef = _.compact(ranks);
+      if (noUndef.length === 0) {
+        console.error(`No ranks found for ${filename} in ${workspace}`);
+      }
+
+      results.push({
+        workspace,
+        filename,
+        cause: cause.message,
+        ranks: noUndef,
+      });
+    }
+
+    return results;
+  };
+
+  let results = [];
+  for (const { workspace, causes } of rootCauses) {
+    results.push(await innerF(workspace, causes));
+  }
+
+  return _.flatten(results);
+}
+
+async function runEvaluation() {
+  fileServer().listen(PORT);
+  const workspaceDir = path.resolve(__dirname, "..", "workspaces");
+  const browser = await chromium.launch({ headless: !global.debugging });
+  const context = await browser.newContext();
+  const results = await Promise.all(
+    _.map(rootCauses, async ({ workspace, causes }) => {
+      const fullSubdir = path.resolve(workspaceDir, workspace);
+      const argusBundles = await argusData(fullSubdir);
+
+      if (!("Ok" in argusBundles)) throw new Error("Argus failed");
+      return forFileInBundle(argusBundles.Ok, async (filename, bundles) => {
+        const cause = _.find(causes, cause => filename.endsWith(cause.file)) as
+          | { file: string; message: string }
+          | undefined;
+
+        if (!cause) return;
+        const page = await openPage(context, filename, bundles, "rank");
+
+        await sleep(5000);
+        await expandBottomUpView(page);
+        await sleep(5000);
+
+        const goals = await page
+          .locator(".BottomUpArea .EvalGoal")
+          .filter({ hasText: cause.message })
+          .all();
+
+        console.debug(
+          `Found ${filename}:${goals.length} goals ${cause.message}`
+        );
+
+        const ranksStr = await Promise.all(
+          _.map(goals, goal => goal.getAttribute("data-rank"))
+        );
+        const rank = _.min(_.map(_.compact(ranksStr), r => Number(r)));
+
+        const numberTreeNodes = _.max(
+          _.flatten(
+            _.map(bundles, bundle =>
+              _.map(_.values(bundle.trees), tree => tree.nodes.length)
+            )
+          )
+        );
+
+        await page.close();
+        return {
+          workspace,
+          filename,
+          cause: cause.message,
+          numberTreeNodes,
+          rank,
+        };
+      });
+    })
+  );
+
+  return _.filter(_.compact(_.flatten(results)), v => v.rank !== undefined);
+}
+
 async function outputInDir(resultsDir: string) {
   fileServer().listen(PORT);
   ensureDir(resultsDir);
@@ -158,15 +351,35 @@ async function outputInDir(resultsDir: string) {
   );
 }
 
-async function main() {
-  const cacheDirectory = process.argv[2];
-  if (cacheDirectory === undefined) {
-    throw new Error("Must provide a cache directory");
-  }
-
+async function runScreenshots(cacheDirectory: string) {
   const results = await outputInDir(cacheDirectory);
   const mapFile = path.resolve(cacheDirectory, "results-map.json");
   fs.writeFileSync(mapFile, JSON.stringify(results));
+}
+
+async function main() {
+  global.debugging = _.includes(process.argv, "--debug");
+  const argv = _.filter(process.argv, arg => arg !== "--debug");
+
+  switch (argv[2]) {
+    case "-r": {
+      const N = Number(argv[3] ?? "10");
+      await runForJson(() => runRandom(N)).then(console.log);
+      break;
+    }
+    case "-h": {
+      await runForJson(() => runEvaluation()).then(console.log);
+      break;
+    }
+    case "-s": {
+      const cacheDirectory = argv[3];
+      if (cacheDirectory === undefined)
+        throw new Error("Must provide a cache directory");
+      await runScreenshots(cacheDirectory);
+    }
+    default:
+      throw new Error("Invalid CLI argument");
+  }
   process.exit(0);
 }