diff --git a/core/dbs/NetworkDb.ts b/core/dbs/NetworkDb.ts index 34cb5eef3..10e99665a 100644 --- a/core/dbs/NetworkDb.ts +++ b/core/dbs/NetworkDb.ts @@ -57,6 +57,7 @@ export default class NetworkDb { if (this.db) { clearInterval(this.saveInterval); this.flush(); + if (env.enableSqliteWal) this.db.pragma('wal_checkpoint(TRUNCATE)'); this.db.close(); } this.db = null; diff --git a/core/dbs/SessionDb.ts b/core/dbs/SessionDb.ts index bfd87563e..69b3d3618 100644 --- a/core/dbs/SessionDb.ts +++ b/core/dbs/SessionDb.ts @@ -43,7 +43,7 @@ export default class SessionDb { } public get isOpen(): boolean { - return this.db?.open; + return this.db?.open ?? false; } public readonly path: string; @@ -76,12 +76,10 @@ export default class SessionDb { public output: OutputTable; public readonly sessionId: string; - public keepAlive = false; - private readonly batchInsert?: Transaction; private readonly saveInterval: NodeJS.Timeout; - private db: SqliteDatabase; + private db: SqliteDatabase | null; private readonly tables: SqliteTable[] = []; constructor(sessionId: string, path: string, dbOptions: IDbOptions = {}) { @@ -142,20 +140,17 @@ export default class SessionDb { public close(): void { clearInterval(this.saveInterval); + if (!this.db) return; - if (this.db?.open) { + if (this.db.open) { this.flush(); } - if (env.enableSqliteWal && !this.db.readonly) { + if (env.enableSqliteWal) { + // use delete to clean up wal files since we might move this around this.db.pragma('journal_mode = DELETE'); } - if (this.keepAlive) { - this.db.readonly = true; - return; - } - this.db.close(); this.db = null; } diff --git a/core/env.ts b/core/env.ts index f97adbf99..c8d62ab5a 100644 --- a/core/env.ts +++ b/core/env.ts @@ -11,6 +11,6 @@ export default { noChromeSandbox: parseEnvBool(env.ULX_NO_CHROME_SANDBOX) ?? AgentEnv.noChromeSandbox, disableGpu: parseEnvBool(env.ULX_DISABLE_GPU) ?? AgentEnv.disableGpu, enableSqliteWal: parseEnvBool(env.ULX_ENABLE_SQLITE_WAL) ?? false, + disableSessionPersistence: parseEnvBool(env.ULX_DISABLE_SESSION_PERSISTENCE), + dataDir: env.ULX_DATA_DIR, }; - -export const dataDir = env.ULX_DATA_DIR; diff --git a/core/index.ts b/core/index.ts index 6d8a44110..2410a3009 100644 --- a/core/index.ts +++ b/core/index.ts @@ -23,7 +23,7 @@ import * as Path from 'path'; import ConnectionToHeroClient from './connections/ConnectionToHeroClient'; import DefaultSessionRegistry from './dbs/DefaultSessionRegistry'; import NetworkDb from './dbs/NetworkDb'; -import * as Env from './env'; +import Env from './env'; import ISessionRegistry from './interfaces/ISessionRegistry'; import Session from './lib/Session'; import Tab from './lib/Tab'; @@ -94,6 +94,8 @@ export default class HeroCore extends TypedEventEmitter { diff --git a/docs/overview/configuration.md b/docs/overview/configuration.md index 29006680e..5dec02b07 100644 --- a/docs/overview/configuration.md +++ b/docs/overview/configuration.md @@ -31,6 +31,14 @@ Configures the storage location for files created by Core. Configurable via [`Core.start()`](#core-start.md) or the first [`ConnectionToCore`](../advanced-client/connection-to-core.md). +### Disable Session Persistence
CoreHero
+ +Configuration to disable session persistence. This will instruct Hero to clean up sessions AFTER they are closed. + +Configurable as a Hero Core option via [`Core.start({ disableSessionPersistence: true })`](#core-start.md) or [Hero](../basic-client/hero.md) instances via `sessionPersistence: false`. + +`Environmental variable`: `ULX_DISABLE_SESSION_PERSISTENCE=true` + ### Blocked Resource Types
ConnectionHero
{#blocked-resources} One of the best ways to optimize Hero's memory and CPU is setting `blockedResourceTypes` to block resources you don't need. The following are valid options. diff --git a/end-to-end/test/basic.test.ts b/end-to-end/test/basic.test.ts index e1c1ab7ca..bea9ec6e9 100644 --- a/end-to-end/test/basic.test.ts +++ b/end-to-end/test/basic.test.ts @@ -1,10 +1,11 @@ -import { Helpers, Hero } from '@ulixee/hero-testing'; +import HeroClient, { ConnectionToHeroCore } from '@ulixee/hero'; import HeroCore, { Session } from '@ulixee/hero-core'; +import { Helpers, Hero } from '@ulixee/hero-testing'; +import { ITestKoaServer } from '@ulixee/hero-testing/helpers'; import TransportBridge from '@ulixee/net/lib/TransportBridge'; -import HeroClient, { ConnectionToHeroCore } from '@ulixee/hero'; -let koaServer; +let koaServer: ITestKoaServer; let core: HeroCore; beforeAll(async () => { core = new HeroCore(); @@ -218,4 +219,36 @@ describe('basic Full Client tests', () => { process.stderr.write = stdout; expect(warningHandler).not.toHaveBeenCalled(); }); + + it('should run connections in parallel', async () => { + const bridge = new TransportBridge(); + const connectionToCore = new ConnectionToHeroCore(bridge.transportToCore); + + const heroCore = new HeroCore({ + maxConcurrentClientCount: 10, + maxConcurrentClientsPerBrowser: 10, + }); + Helpers.needsClosing.push(heroCore); + heroCore.addConnection(bridge.transportToClient); + + koaServer.get('/random-delay', async ctx => { + await new Promise(resolve => setTimeout(resolve, Math.random() * 1000)); + ctx.body = { test: true }; + }); + + const resultOrder = []; + await Promise.all( + new Array(10).fill(0).map(async (_, i) => { + const hero = new Hero({ + connectionToCore, + }); + await hero.meta; + await hero.goto(`${koaServer.baseUrl}/random-delay`); + Helpers.needsClosing.push(hero); + resultOrder.push(i); + }), + ); + expect(resultOrder).toHaveLength(10); + expect(resultOrder).not.toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + }); }); diff --git a/interfaces/ICoreConfigureOptions.ts b/interfaces/ICoreConfigureOptions.ts index c22fe04e3..298a781eb 100644 --- a/interfaces/ICoreConfigureOptions.ts +++ b/interfaces/ICoreConfigureOptions.ts @@ -8,4 +8,5 @@ export default interface ICoreConfigureOptions { defaultUnblockedPlugins?: IUnblockedPluginClass[]; shouldShutdownOnSignals?: boolean; sessionRegistry?: ISessionRegistry; + disableSessionPersistence?: boolean; }