feat(core): start without session persistence

core/dbs/NetworkDb.ts

@@ -57,6 +57,7 @@ export default class NetworkDb {
     if (this.db) {
       clearInterval(this.saveInterval);
       this.flush();
+      if (env.enableSqliteWal) this.db.pragma('wal_checkpoint(TRUNCATE)');
       this.db.close();
     }
     this.db = null;

core/dbs/SessionDb.ts

@@ -43,7 +43,7 @@ export default class SessionDb {
   }
 
   public get isOpen(): boolean {
-    return this.db?.open;
+    return this.db?.open ?? false;
   }
 
   public readonly path: string;
@@ -76,12 +76,10 @@ export default class SessionDb {
   public output: OutputTable;
   public readonly sessionId: string;
 
-  public keepAlive = false;
-
   private readonly batchInsert?: Transaction;
   private readonly saveInterval: NodeJS.Timeout;
 
-  private db: SqliteDatabase;
+  private db: SqliteDatabase | null;
   private readonly tables: SqliteTable<any>[] = [];
 
   constructor(sessionId: string, path: string, dbOptions: IDbOptions = {}) {
@@ -142,20 +140,17 @@ export default class SessionDb {
 
   public close(): void {
     clearInterval(this.saveInterval);
+    if (!this.db) return;
 
-    if (this.db?.open) {
+    if (this.db.open) {
       this.flush();
     }
 
-    if (env.enableSqliteWal && !this.db.readonly) {
+    if (env.enableSqliteWal) {
+      // use delete to clean up wal files since we might move this around
       this.db.pragma('journal_mode = DELETE');
     }
 
-    if (this.keepAlive) {
-      this.db.readonly = true;
-      return;
-    }
-
     this.db.close();
     this.db = null;
   }

core/env.ts

@@ -11,6 +11,6 @@ export default {
   noChromeSandbox: parseEnvBool(env.ULX_NO_CHROME_SANDBOX) ?? AgentEnv.noChromeSandbox,
   disableGpu: parseEnvBool(env.ULX_DISABLE_GPU) ?? AgentEnv.disableGpu,
   enableSqliteWal: parseEnvBool(env.ULX_ENABLE_SQLITE_WAL) ?? false,
+  disableSessionPersistence: parseEnvBool(env.ULX_DISABLE_SESSION_PERSISTENCE),
+  dataDir: env.ULX_DATA_DIR,
 };
-
-export const dataDir = env.ULX_DATA_DIR;

core/index.ts

@@ -23,7 +23,7 @@ import * as Path from 'path';
 import ConnectionToHeroClient from './connections/ConnectionToHeroClient';
 import DefaultSessionRegistry from './dbs/DefaultSessionRegistry';
 import NetworkDb from './dbs/NetworkDb';
-import * as Env from './env';
+import Env from './env';
 import ISessionRegistry from './interfaces/ISessionRegistry';
 import Session from './lib/Session';
 import Tab from './lib/Tab';
@@ -94,6 +94,8 @@ export default class HeroCore extends TypedEventEmitter<THeroCoreEvents & { clos
       options.dataDir = Path.join(process.cwd(), options.dataDir);
     }
 
+    options.disableSessionPersistence ??= Env.disableSessionPersistence;
+
     try {
       Fs.mkdirSync(`${options.dataDir}`, { recursive: true });
     } catch {}

core/lib/Session.ts

@@ -158,6 +158,7 @@ export default class Session
         }
       }
     }
+    options.sessionPersistence ??= core.options.disableSessionPersistence !== true;
 
     this.createdTime = Date.now();
     this.id = this.assignId(options.sessionId);

docs/help/troubleshooting.md

@@ -18,10 +18,10 @@ Browsers will be saved to a shared location on each OS. Each browser version wil
 
 By default, Hero logs everything to a [Session](../advanced-concepts/sessions.md) database that is created per Hero instance. The SessionLogs table contains all debug logs.
 
-To output logs to the console during operation, you can set the environmental variable `DEBUG=true`.
+To output logs to the console during operation, you can set the environmental variable `DEBUG=ulx`. If you want to include all Devtools Protocol Messages as well, you can add `DEBUG=ulx,ulx:devtools`, or `DEBUG=ulx*`.
 
 ```js
-process.env.DEBUG = true;
+process.env.DEBUG = "ulx";
 import Hero from '@ulixee/hero';
 
 (async () => {

docs/overview/configuration.md

@@ -31,6 +31,14 @@ Configures the storage location for files created by Core.
 
 Configurable via [`Core.start()`](#core-start.md) or the first [`ConnectionToCore`](../advanced-client/connection-to-core.md).
 
+### Disable Session Persistence <div class="specs"><i>Core</i><i>Hero</i></div>
+
+Configuration to disable session persistence. This will instruct Hero to clean up sessions AFTER they are closed.
+
+Configurable as a Hero Core option via [`Core.start({ disableSessionPersistence: true })`](#core-start.md) or [Hero](../basic-client/hero.md) instances via `sessionPersistence: false`.
+
+`Environmental variable`: `ULX_DISABLE_SESSION_PERSISTENCE=true`
+
 ### Blocked Resource Types <div class="specs"><i>Connection</i><i>Hero</i></div> {#blocked-resources}
 
 One of the best ways to optimize Hero's memory and CPU is setting `blockedResourceTypes` to block resources you don't need. The following are valid options.

end-to-end/test/basic.test.ts

@@ -1,10 +1,11 @@
-import { Helpers, Hero } from '@ulixee/hero-testing';
+import HeroClient, { ConnectionToHeroCore } from '@ulixee/hero';
 import HeroCore, { Session } from '@ulixee/hero-core';
+import { Helpers, Hero } from '@ulixee/hero-testing';
+import { ITestKoaServer } from '@ulixee/hero-testing/helpers';
 
 import TransportBridge from '@ulixee/net/lib/TransportBridge';
-import HeroClient, { ConnectionToHeroCore } from '@ulixee/hero';
 
-let koaServer;
+let koaServer: ITestKoaServer;
 let core: HeroCore;
 beforeAll(async () => {
   core = new HeroCore();
@@ -218,4 +219,36 @@ describe('basic Full Client tests', () => {
     process.stderr.write = stdout;
     expect(warningHandler).not.toHaveBeenCalled();
   });
+
+  it('should run connections in parallel', async () => {
+    const bridge = new TransportBridge();
+    const connectionToCore = new ConnectionToHeroCore(bridge.transportToCore);
+
+    const heroCore = new HeroCore({
+      maxConcurrentClientCount: 10,
+      maxConcurrentClientsPerBrowser: 10,
+    });
+    Helpers.needsClosing.push(heroCore);
+    heroCore.addConnection(bridge.transportToClient);
+
+    koaServer.get('/random-delay', async ctx => {
+      await new Promise(resolve => setTimeout(resolve, Math.random() * 1000));
+      ctx.body = { test: true };
+    });
+
+    const resultOrder = [];
+    await Promise.all(
+      new Array(10).fill(0).map(async (_, i) => {
+        const hero = new Hero({
+          connectionToCore,
+        });
+        await hero.meta;
+        await hero.goto(`${koaServer.baseUrl}/random-delay`);
+        Helpers.needsClosing.push(hero);
+        resultOrder.push(i);
+      }),
+    );
+    expect(resultOrder).toHaveLength(10);
+    expect(resultOrder).not.toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+  });
 });

interfaces/ICoreConfigureOptions.ts

@@ -8,4 +8,5 @@ export default interface ICoreConfigureOptions {
   defaultUnblockedPlugins?: IUnblockedPluginClass[];
   shouldShutdownOnSignals?: boolean;
   sessionRegistry?: ISessionRegistry;
+  disableSessionPersistence?: boolean;
 }