fix(perf): chunk the output of the query, to avoid overloadign the se…

…rver

package.json

@@ -63,7 +63,7 @@
     "ts-jest": "^22.4.6",
     "tsc": "^1.20150623.0",
     "tslint": "^5.8.0",
-    "typescript": "^2.6.1",
+    "typescript": "^3.7.0",
     "typescript-babel-jest": "^1.0.5"
   },
   "dependencies": {
@@ -73,6 +73,7 @@
     "gravatar-url": "^2.0.0",
     "lodash": "^4.17.10",
     "ms": "^2.1.1",
-    "pg": "^7.4.3"
+    "pg": "^7.4.3",
+    "pg-cursor": "^2.1.5"
   }
 }

src/algolia/index.ts

@@ -117,7 +117,7 @@ export class Index<UserData extends object> {
     return this._client.copyIndex(source, destination);
   }
 
-  async waitTask({ taskID }: algoliasearch.Task) {
+  async waitTask({ taskID }: Pick<algoliasearch.Task, 'taskID'>) {
     log.info(`Waiting for Algolia task ${taskID} on ${this.indexName}`);
     return this._index.waitTask(taskID);
   }

src/ambient.d.ts

@@ -2,3 +2,10 @@ declare module '*.sql' {
   const query: string;
   export default query;
 }
+
+declare module 'pg-cursor' {
+  export default class Cursor<Row> {
+    constructor(text: string, values?: any[], config?: object) {}
+    read(rows: number, cb: (err: Error | undefined, rows: Row[]) => void) {}
+  }
+}

src/database.ts

@@ -1,4 +1,5 @@
 import { Client } from 'pg';
+import Cursor from 'pg-cursor';
 import { Pod, SpecificationData } from './types';
 import log from './log';
 import { formatPod } from './formatPod/index';
@@ -58,22 +59,44 @@ export interface ParsedRow {
   };
 }
 
-export async function fetchAll(): Promise<Pod[]> {
+export async function fetchAll(onBatch: (pods: Pod[]) => void) {
   log.info('Commencing query for all pods');
   await trunk.connect();
-  const { rows }: { rows: Row[] } = await trunk.query(allPodsQuery);
+  // TS: the cursor api seems to change the return of query to be itself
+  // let's leave it as `any` for now.
+  const cursor: any = trunk.query(new Cursor<Row>(allPodsQuery) as any);
 
-  log.info(`Found ${rows.length} pods`);
+  type PromiseCallbackParameters = Parameters<
+    ConstructorParameters<typeof Promise>[0]
+  >;
+  type Resolve = PromiseCallbackParameters[0];
+  type Reject = PromiseCallbackParameters[0];
 
-  const pods: Pod[] = rows
-    .map(({ objectID, specificationData, downloads }: Row) => ({
-      objectID,
-      specificationData: JSON.parse(specificationData),
-      downloads,
-    }))
-    .map(formatPod);
+  function batch(resolve: Resolve, reject: Reject) {
+    cursor.read(10000, (err: Error | undefined, rows: Row[]) => {
+      if (err) {
+        reject(err);
+      }
 
-  log.info(`Will now index ${pods.length} pods`);
+      if (rows.length === 0) {
+        resolve();
+        return;
+      }
 
-  return pods;
+      log.info(`Found ${rows.length} pods`);
+
+      const pods: Pod[] = rows
+        .map(({ objectID, specificationData, downloads }: Row) => ({
+          objectID,
+          specificationData: JSON.parse(specificationData),
+          downloads,
+        }))
+        .map(formatPod);
+
+      onBatch(pods);
+      batch(resolve, reject);
+    });
+  }
+
+  return new Promise(batch);
 }

src/index.ts

@@ -5,6 +5,7 @@ import { Index } from './algolia/index';
 import { fetchAll } from './database';
 import { settings, synonyms, rules } from './settings';
 import ms from 'ms';
+import { Task } from 'algoliasearch';
 
 interface IndexState {
   bootstrapLastFinished: number;
@@ -46,9 +47,17 @@ export async function bootstrap() {
   await mainIndex.state.save({ bootstrapDidFinish: false });
   await bootstrapIndex.waitTask(await bootstrapIndex.destroy());
 
-  await bootstrapIndex.waitTask(
-    await bootstrapIndex.savePods(await fetchAll())
-  );
+  const taskPromises: Promise<Task>[] = [];
+
+  await fetchAll(pods => {
+    taskPromises.push(bootstrapIndex.savePods(pods));
+  });
+
+  const tasks = await Promise.all(taskPromises);
+  const taskIds = tasks.map(task => task.taskID);
+  const latestTask = Math.max(...taskIds);
+
+  await bootstrapIndex.waitTask({ taskID: latestTask });
 
   await bootstrapIndex.waitTask(
     await bootstrapIndex.setAllSettings({

yarn.lock

@@ -3462,6 +3462,11 @@ [email protected]:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7"
 
+pg-cursor@^2.1.5:
+  version "2.1.5"
+  resolved "https://registry.yarnpkg.com/pg-cursor/-/pg-cursor-2.1.5.tgz#912096479cd8263b9eba38e84d9d64c641a55c0a"
+  integrity sha512-+LSPIxw7awEhQrb8sqT2Sxr+8RoEAzURMX18oaOQebItqaO94WUh3deHQs9vh3m14E2lz7q7ebRl1s/ODfJAkQ==
+
 pg-pool@~2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-2.0.3.tgz#c022032c8949f312a4f91fb6409ce04076be3257"
@@ -4361,10 +4366,15 @@ typescript-babel-jest@^1.0.5:
     babel-jest "20.0.3"
     typescript "^2.4.1"
 
-typescript@^2.4.1, typescript@^2.6.1:
+typescript@^2.4.1:
   version "2.8.3"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.8.3.tgz#5d817f9b6f31bb871835f4edf0089f21abe6c170"
 
+typescript@^3.7.0:
+  version "3.7.5"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae"
+  integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==
+
 uglify-js@^2.6:
   version "2.8.29"
   resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd"