From b0ea9ac0f82b6bda1ec7ff815b15f8bcc95a0c47 Mon Sep 17 00:00:00 2001
From: Ronit Agarwala <ronitagarwala01@gmail.com>
Date: Tue, 21 Nov 2023 13:07:15 -0500
Subject: [PATCH] Change batch index loop

-Switch to Array.forEach method for faster indexing
-Use lodash chunk method for batching
---
 data.ts           | 8 ++++----
 package-lock.json | 9 +++++++++
 package.json      | 1 +
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/data.ts b/data.ts
index ff8fee2..cf60cd2 100644
--- a/data.ts
+++ b/data.ts
@@ -12,6 +12,7 @@ import { pathToFileURL } from 'url'
 import { Client } from '@opensearch-project/opensearch'
 import type { ClientOptions } from '@opensearch-project/opensearch'
 import { exists } from './paths'
+import _ from 'lodash'
 
 const jsonFilename = 'sandbox-search.json'
 const jsFilename = 'sandbox-search.js'
@@ -45,10 +46,9 @@ export async function populate(path: string, opts: ClientOptions) {
   if (data) {
     const client = new Client(opts)
     const batch_size = 10
-    for (let i = 0; i < data.length; i += batch_size) {
-      const batch = data.slice(i, i + batch_size)
+    const batches = _.chunk(data, batch_size)
+    batches.forEach(async (batch: object[]) => {
       await client.bulk({ body: batch })
-      console.log(`Indexed ${batch.length + i} records`)
-    }
+    })
   }
 }
diff --git a/package-lock.json b/package-lock.json
index 7a2f7a3..b02e4ec 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -11,6 +11,7 @@
       "dependencies": {
         "@opensearch-project/opensearch": "^2.2.0",
         "env-paths": "^3.0.0",
+        "lodash": "^4.17.21",
         "make-fetch-happen": "^11.0.3",
         "rimraf": "^4.1.2",
         "tar": "^6.1.13",
@@ -32,6 +33,9 @@
         "lint-staged": "^13.1.2",
         "npm-run-all": "^4.1.5",
         "prettier": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=18"
       }
     },
     "node_modules/@esbuild/android-arm": {
@@ -2795,6 +2799,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
diff --git a/package.json b/package.json
index bc7b329..209eaa0 100644
--- a/package.json
+++ b/package.json
@@ -27,6 +27,7 @@
   "dependencies": {
     "@opensearch-project/opensearch": "^2.2.0",
     "env-paths": "^3.0.0",
+    "lodash": "^4.17.21",
     "make-fetch-happen": "^11.0.3",
     "rimraf": "^4.1.2",
     "tar": "^6.1.13",