From 6de42d0ac8087a755bc79d041574483f4f555800 Mon Sep 17 00:00:00 2001 From: Dimi Kot Date: Sat, 13 Jan 2024 03:00:15 -0800 Subject: [PATCH] Upgrade Prettier; allow using from both CommonJS and ESM code --- .eslintrc.base.js | 3 ++- .github/workflows/ci.yml | 9 +++++---- .gitignore | 1 + .npmignore | 2 ++ README.md | 2 ++ docs/README.md | 2 ++ docs/classes/Magination.md | 22 +++++++++++++--------- docs/classes/Source.md | 16 ++++++++++------ docs/interfaces/Cache.md | 8 ++++---- docs/interfaces/Hasher.md | 2 +- docs/interfaces/Page.md | 2 +- docs/interfaces/SourceOptions.md | 8 ++++---- package.json | 7 +++++-- src/Magination.ts | 8 ++++---- src/Source.ts | 14 +++++++------- src/__tests__/Source.test.ts | 6 +++--- src/__tests__/index.ts | 6 +++--- src/index.ts | 1 - tsconfig.json | 7 +++---- 19 files changed, 72 insertions(+), 54 deletions(-) diff --git a/.eslintrc.base.js b/.eslintrc.base.js index 005db65..5d45f7a 100644 --- a/.eslintrc.base.js +++ b/.eslintrc.base.js @@ -54,10 +54,11 @@ module.exports = (projectRoot) => ({ }, ignorePatterns: [ "node_modules", + "dist", "webpack.config.ts", "**/bin/**", "*.d.ts", - "**/jest.config.js", + "*.js", ], rules: { // TODO: slowly enable no-extraneous-dependencies rule below. For now, it's diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dcf806..8e9f154 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,8 @@ jobs: uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} - - run: npm install - - run: npm run build - - run: npm run test - - run: npm run lint + - run: npm install -g pnpm --force + - run: pnpm install + - run: pnpm run build + - run: pnpm run lint + - run: pnpm run test diff --git a/.gitignore b/.gitignore index f2d1a87..d73b8ab 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ dist node_modules package-lock.json yarn.lock +pnpm-lock.yaml .DS_Store *.log *.tmp diff --git a/.npmignore b/.npmignore index 3e3650d..14484a5 100644 --- a/.npmignore +++ b/.npmignore @@ -1,10 +1,12 @@ __tests__ .npmrc tsconfig.tsbuildinfo +.github node_modules package-lock.json yarn.lock +pnpm-lock.yaml .DS_Store *.log *.tmp diff --git a/README.md b/README.md index 1228c9a..aa19934 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ See also [Full API documentation](https://github.com/clickup/magination/blob/master/docs/modules.md). +![CI run](https://github.com/clickup/magination/actions/workflows/ci.yml/badge.svg?branch=main) + The problem: for some search request (e.g. search-by-keywords), we have multiple search queries (sources) with different performance that deliver results (hits) with different relevancy. We want to merge that search hits using "cursor based diff --git a/docs/README.md b/docs/README.md index 6cc1a15..763340a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,8 @@ See also [Full API documentation](https://github.com/clickup/magination/blob/master/docs/modules.md). +![CI run](https://github.com/clickup/magination/actions/workflows/ci.yml/badge.svg?branch=main) + The problem: for some search request (e.g. search-by-keywords), we have multiple search queries (sources) with different performance that deliver results (hits) with different relevancy. We want to merge that search hits using "cursor based diff --git a/docs/classes/Magination.md b/docs/classes/Magination.md index 3da8665..b424f1d 100644 --- a/docs/classes/Magination.md +++ b/docs/classes/Magination.md @@ -1,6 +1,6 @@ [@clickup/magination](../README.md) / [Exports](../modules.md) / Magination -# Class: Magination +# Class: Magination\ Represents a union of multiple pagination sources into one continuous pages stream with cursor. @@ -9,21 +9,21 @@ stream with cursor. | Name | Type | | :------ | :------ | -| `TSource` | extends [`Source`](Source.md)<`any`\> | -| `THit` | `TSource` extends [`Source`](Source.md) ? `THit` : `never` | +| `TSource` | extends [`Source`](Source.md)\<`any`\> | +| `THit` | `TSource` extends [`Source`](Source.md)\ ? `THit` : `never` | ## Constructors ### constructor -• **new Magination**<`TSource`, `THit`\>(`sources`) +• **new Magination**\<`TSource`, `THit`\>(`sources`): [`Magination`](Magination.md)\<`TSource`, `THit`\> #### Type parameters | Name | Type | | :------ | :------ | -| `TSource` | extends [`Source`](Source.md)<`any`\> | -| `THit` | `TSource` extends [`Source`](Source.md)<`THit`\> ? `THit` : `never` | +| `TSource` | extends [`Source`](Source.md)\<`any`\> | +| `THit` | `TSource` extends [`Source`](Source.md)\<`THit`\> ? `THit` : `never` | #### Parameters @@ -31,6 +31,10 @@ stream with cursor. | :------ | :------ | | `sources` | readonly `TSource`[] | +#### Returns + +[`Magination`](Magination.md)\<`TSource`, `THit`\> + #### Defined in [src/Magination.ts:30](https://github.com/clickup/magination/blob/master/src/Magination.ts#L30) @@ -39,7 +43,7 @@ stream with cursor. ### load -▸ **load**(`«destructured»`): `AsyncGenerator`<[`Page`](../interfaces/Page.md)<`THit`\> & { `prevCursor`: ``null`` \| `string` ; `source`: `TSource` }, `any`, `unknown`\> +▸ **load**(`«destructured»`): `AsyncGenerator`\<[`Page`](../interfaces/Page.md)\<`THit`\> & \{ `prevCursor`: ``null`` \| `string` ; `source`: `TSource` }, `any`, `unknown`\> Returns a finite generator of pages which runs all of the sources in parallel and then return a page of resulting hits. Basically, loads the @@ -76,12 +80,12 @@ hits along with the new cursor. | :------ | :------ | | `«destructured»` | `Object` | | › `cache` | [`Cache`](../interfaces/Cache.md) | -| › `hasher` | [`Hasher`](../interfaces/Hasher.md)<`THit`\> | +| › `hasher` | [`Hasher`](../interfaces/Hasher.md)\<`THit`\> | | › `cursor` | ``null`` \| `string` | #### Returns -`AsyncGenerator`<[`Page`](../interfaces/Page.md)<`THit`\> & { `prevCursor`: ``null`` \| `string` ; `source`: `TSource` }, `any`, `unknown`\> +`AsyncGenerator`\<[`Page`](../interfaces/Page.md)\<`THit`\> & \{ `prevCursor`: ``null`` \| `string` ; `source`: `TSource` }, `any`, `unknown`\> #### Defined in diff --git a/docs/classes/Source.md b/docs/classes/Source.md index 25fd4c3..56b2613 100644 --- a/docs/classes/Source.md +++ b/docs/classes/Source.md @@ -1,6 +1,6 @@ [@clickup/magination](../README.md) / [Exports](../modules.md) / Source -# Class: Source +# Class: Source\ Represents a single stream of hits subject for pagination. @@ -20,7 +20,7 @@ do exclusion on the hits returned, so they are never repeated. ### constructor -• **new Source**<`THit`\>(`name`, `options`) +• **new Source**\<`THit`\>(`name`, `options`): [`Source`](Source.md)\<`THit`\> #### Type parameters @@ -33,7 +33,11 @@ do exclusion on the hits returned, so they are never repeated. | Name | Type | | :------ | :------ | | `name` | `string` | -| `options` | [`SourceOptions`](../interfaces/SourceOptions.md)<`THit`\> | +| `options` | [`SourceOptions`](../interfaces/SourceOptions.md)\<`THit`\> | + +#### Returns + +[`Source`](Source.md)\<`THit`\> #### Defined in @@ -53,7 +57,7 @@ do exclusion on the hits returned, so they are never repeated. ### load -▸ **load**(`«destructured»`): `Promise`<[`Page`](../interfaces/Page.md)<`THit`\>\> +▸ **load**(`«destructured»`): `Promise`\<[`Page`](../interfaces/Page.md)\<`THit`\>\> #### Parameters @@ -63,11 +67,11 @@ do exclusion on the hits returned, so they are never repeated. | › `cache` | [`Cache`](../interfaces/Cache.md) | | › `cursor` | ``null`` \| `string` | | › `excludeHits` | `THit`[] | -| › `hasher` | [`Hasher`](../interfaces/Hasher.md)<`THit`\> | +| › `hasher` | [`Hasher`](../interfaces/Hasher.md)\<`THit`\> | #### Returns -`Promise`<[`Page`](../interfaces/Page.md)<`THit`\>\> +`Promise`\<[`Page`](../interfaces/Page.md)\<`THit`\>\> #### Defined in diff --git a/docs/interfaces/Cache.md b/docs/interfaces/Cache.md index 47d1b88..b64c4c9 100644 --- a/docs/interfaces/Cache.md +++ b/docs/interfaces/Cache.md @@ -8,7 +8,7 @@ A key-value store which allows to store various metadata for a cursor. ### read -▸ **read**(`key`): `Promise`<``null`` \| `object`\> +▸ **read**(`key`): `Promise`\<``null`` \| `object`\> #### Parameters @@ -18,7 +18,7 @@ A key-value store which allows to store various metadata for a cursor. #### Returns -`Promise`<``null`` \| `object`\> +`Promise`\<``null`` \| `object`\> #### Defined in @@ -28,7 +28,7 @@ ___ ### write -▸ **write**(`key`, `value`): `Promise`<`unknown`\> +▸ **write**(`key`, `value`): `Promise`\<`unknown`\> #### Parameters @@ -39,7 +39,7 @@ ___ #### Returns -`Promise`<`unknown`\> +`Promise`\<`unknown`\> #### Defined in diff --git a/docs/interfaces/Hasher.md b/docs/interfaces/Hasher.md index 42413cb..0e3abcc 100644 --- a/docs/interfaces/Hasher.md +++ b/docs/interfaces/Hasher.md @@ -1,6 +1,6 @@ [@clickup/magination](../README.md) / [Exports](../modules.md) / Hasher -# Interface: Hasher +# Interface: Hasher\ ## Type parameters diff --git a/docs/interfaces/Page.md b/docs/interfaces/Page.md index 70e4a65..a14c0f3 100644 --- a/docs/interfaces/Page.md +++ b/docs/interfaces/Page.md @@ -1,6 +1,6 @@ [@clickup/magination](../README.md) / [Exports](../modules.md) / Page -# Interface: Page +# Interface: Page\ A chunk of hits and a cursor which allows to fetch more hits. If the cursor is null, there are no more hits left to fetch. diff --git a/docs/interfaces/SourceOptions.md b/docs/interfaces/SourceOptions.md index 1f5b042..60dcea4 100644 --- a/docs/interfaces/SourceOptions.md +++ b/docs/interfaces/SourceOptions.md @@ -1,6 +1,6 @@ [@clickup/magination](../README.md) / [Exports](../modules.md) / SourceOptions -# Interface: SourceOptions +# Interface: SourceOptions\ ## Type parameters @@ -32,11 +32,11 @@ ___ ### search -• **search**: (`cursor`: ``null`` \| `string`, `excludeHits`: `THit`[], `count`: `number`) => `Promise`<[`Page`](Page.md)<`THit`\>\> +• **search**: (`cursor`: ``null`` \| `string`, `excludeHits`: `THit`[], `count`: `number`) => `Promise`\<[`Page`](Page.md)\<`THit`\>\> #### Type declaration -▸ (`cursor`, `excludeHits`, `count`): `Promise`<[`Page`](Page.md)<`THit`\>\> +▸ (`cursor`, `excludeHits`, `count`): `Promise`\<[`Page`](Page.md)\<`THit`\>\> ##### Parameters @@ -48,7 +48,7 @@ ___ ##### Returns -`Promise`<[`Page`](Page.md)<`THit`\>\> +`Promise`\<[`Page`](Page.md)\<`THit`\>\> #### Defined in diff --git a/package.json b/package.json index d781fc6..56abd5c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@clickup/magination", "description": "An opinionated framework to build cursor-based pagination over multiple streams of hits", - "version": "2.10.292", + "version": "2.10.296", "license": "MIT", "keywords": [ "pagination", @@ -16,7 +16,7 @@ "lint": "eslint . --ext .ts --cache --cache-location dist/.eslintcache", "test": "jest", "docs": "rm -rf docs && typedoc --plugin typedoc-plugin-markdown --plugin typedoc-plugin-merge-modules && sed -i '' -E 's#packages/[^/]+/##g' $(find docs -type f -name '*.md')", - "clean": "rm -rf dist node_modules yarn.lock package-lock.json", + "clean": "rm -rf dist node_modules yarn.lock package-lock.json pnpm-lock.yaml *.log", "copy-package-to-public-dir": "copy-package-to-public-dir.sh", "backport-package-from-public-dir": "backport-package-from-public-dir.sh", "deploy": "npm run build && npm run lint && npm run test && npm publish --access=public" @@ -28,6 +28,7 @@ "devDependencies": { "@types/jest": "^29.5.5", "@types/lodash": "^4.14.175", + "@types/node": "^20.4.1", "@types/uniqid": "^4.1.3", "@typescript-eslint/eslint-plugin": "^5.59.6", "@typescript-eslint/parser": "^5.59.6", @@ -41,6 +42,8 @@ "eslint-plugin-typescript-sort-keys": "^2.3.0", "eslint-plugin-unused-imports": "^2.0.0", "eslint": "^8.40.0", + "jest": "^29.7.0", + "prettier": "3.2.1", "ts-jest": "^29.1.1", "typedoc-plugin-markdown": "^3.16.0", "typedoc-plugin-merge-modules": "^5.1.0", diff --git a/src/Magination.ts b/src/Magination.ts index 05ede06..c290f9f 100644 --- a/src/Magination.ts +++ b/src/Magination.ts @@ -23,7 +23,7 @@ type MaginationSlot = { */ export default class Magination< TSource extends Source, - THit = TSource extends Source ? THit : never + THit = TSource extends Source ? THit : never, > { private sources; @@ -119,12 +119,12 @@ export default class Magination< cursor, excludeHits: [], hasher, - }) - ) + }), + ), ); const excludeHashes = new Set( - flatten(slot.frames.slice(0, num + 1).map((frame) => frame.hitHashes)) + flatten(slot.frames.slice(0, num + 1).map((frame) => frame.hitHashes)), ); // Yield pages in order of sources. On each yield, remember all of the diff --git a/src/Source.ts b/src/Source.ts index 7fd160d..421b30e 100644 --- a/src/Source.ts +++ b/src/Source.ts @@ -16,7 +16,7 @@ export interface SourceOptions { search: ( cursor: string | null, excludeHits: THit[], - count: number + count: number, ) => Promise>; } @@ -32,7 +32,7 @@ export interface SourceOptions { export default class Source { constructor( public readonly name: string, - private options: SourceOptions + private options: SourceOptions, ) {} async load({ @@ -50,8 +50,8 @@ export default class Source { typeof this.options.preloadSize === "function" ? this.options.preloadSize(offset) : typeof this.options.preloadSize === "number" - ? this.options.preloadSize - : this.options.pageSize + 1; // +1 to efficiently detect whether we have more pages + ? this.options.preloadSize + : this.options.pageSize + 1; // +1 to efficiently detect whether we have more pages let [slotKey, pos] = parseCursor(cursor); // Load cache slot or create a new empty one. @@ -70,7 +70,7 @@ export default class Source { pos = Math.min(pos, slot.hits.length); const excludeHashes = new Set( - [...slot.hits.slice(0, pos), ...excludeHits].map((hit) => hasher(hit)) + [...slot.hits.slice(0, pos), ...excludeHits].map((hit) => hasher(hit)), ); // Extract hits from the cache starting from pos to form results page. @@ -85,7 +85,7 @@ export default class Source { const res = await this.options.search( slot.cursor, [...slot.hits, ...excludeHits], - preloadSize(pos) + preloadSize(pos), ); slot.hits.push(...res.hits); slot.cursor = res.cursor; @@ -113,7 +113,7 @@ export default class Source { inoutExcludeHashes: Set, hasher: Hasher, cachedHits: THit[], - pos: number + pos: number, ): number { while (outHits.length < this.options.pageSize && pos < cachedHits.length) { const hit = cachedHits[pos]; diff --git a/src/__tests__/Source.test.ts b/src/__tests__/Source.test.ts index 854c85f..87235a5 100644 --- a/src/__tests__/Source.test.ts +++ b/src/__tests__/Source.test.ts @@ -12,7 +12,7 @@ test("empty results", async () => { search: mockSearch([]), }); expect( - await source.load({ cache, cursor: null, excludeHits: [], hasher }) + await source.load({ cache, cursor: null, excludeHits: [], hasher }), ).toEqual({ hits: [], cursor: null, @@ -23,7 +23,7 @@ test("empty results", async () => { cursor: "abc:100", excludeHits: [], hasher, - }) + }), ).toEqual({ hits: [], cursor: null, @@ -37,7 +37,7 @@ test("less results than page size", async () => { search: mockSearch(["a", "b"]), }); expect( - await source.load({ cache, cursor: null, excludeHits: [], hasher }) + await source.load({ cache, cursor: null, excludeHits: [], hasher }), ).toEqual({ hits: ["a", "b"], cursor: null, diff --git a/src/__tests__/index.ts b/src/__tests__/index.ts index 8e112ce..d772cea 100644 --- a/src/__tests__/index.ts +++ b/src/__tests__/index.ts @@ -24,16 +24,16 @@ export function mockSearch(...chunks: THit[][]) { async function search( cursor: string | null, excludeHits: THit[], - count: number + count: number, ): Promise> { search.calls.push({ cursor, excludeHits, count }); const chunk = cursor !== null ? parseInt(cursor.replace(/\D+/, "")) : 0; const excludeHitsStr = new Set( - excludeHits.map((excl) => JSON.stringify(excl)) + excludeHits.map((excl) => JSON.stringify(excl)), ); return { hits: (chunks[chunk] ?? []).filter( - (hit) => !excludeHitsStr.has(JSON.stringify(hit)) + (hit) => !excludeHitsStr.has(JSON.stringify(hit)), ), cursor: chunk < chunks.length - 1 ? `chunk${chunk + 1}` : null, }; diff --git a/src/index.ts b/src/index.ts index 8e336c7..f12be5c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,6 @@ import Cache from "./Cache"; import Magination from "./Magination"; import Page from "./Page"; - import Source, { SourceOptions } from "./Source"; export { Magination, Source, SourceOptions, Page, Cache }; diff --git a/tsconfig.json b/tsconfig.json index 65ddd6e..783953d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,9 +9,8 @@ "esModuleInterop": true, "experimentalDecorators": true, "incremental": true, - "lib": ["es2019"], - "module": "commonjs", - "moduleResolution": "node", + "lib": ["ES2019"], + "module": "Node16", "noEmitOnError": true, "noErrorTruncation": true, "noImplicitOverride": true, @@ -25,7 +24,7 @@ "skipLibCheck": true, "sourceMap": true, "strict": true, - "target": "es2019", + "target": "ES2019", "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo", "types": ["node", "jest"] }