KEEP-1063: handle data-entries limit

Keeper-Wallet · Feb 3, 2023 · d0facd5 · d0facd5
1 parent 46871e7
commit d0facd5
Show file tree

Hide file tree

Showing 6 changed files with 107 additions and 89 deletions.
diff --git a/src/_core/chunk.ts b/src/_core/chunk.ts
@@ -0,0 +1,5 @@
+export function* chunk<T>(array: T[], size: number) {
+  for (let i = 0; i < array.length; i += size) {
+    yield array.slice(i, i + size);
+  }
+}
diff --git a/src/_core/fetchInBatches.ts b/src/_core/fetchInBatches.ts
@@ -0,0 +1,20 @@
+import invariant from 'tiny-invariant';
+
+import { chunk } from './chunk';
+
+export async function fetchInBatches<I, T>(
+  allItems: I[],
+  chunkSize: number,
+  fetchFn: (items: I[]) => Promise<T[]>
+) {
+  const result: T[] = [];
+
+  for (const items of chunk(allItems, chunkSize)) {
+    const data = await fetchFn(items);
+    invariant(Array.isArray(data));
+
+    result.push(...data);
+  }
+
+  return result;
+}
diff --git a/src/nfts/utils.ts b/src/nfts/utils.ts
@@ -1,15 +1,3 @@
-import { DataTransactionEntry } from '@waves/ts-types';
-
 export function capitalize(str: string | undefined) {
   return str ? str.charAt(0).toUpperCase() + str.slice(1) : str;
 }
-
-export function reduceDataEntries(entries: DataTransactionEntry[]) {
-  return entries.reduce<Record<string, DataTransactionEntry['value']>>(
-    (data, item) => {
-      data[item.key] = item.value;
-      return data;
-    },
-    {}
-  );
-}
diff --git a/src/nfts/vendors/ducklings.ts b/src/nfts/vendors/ducklings.ts
@@ -1,11 +1,15 @@
+import {
+  dataEntriesToRecord,
+  fetchDataEntries,
+} from '../../nodeApi/dataEntries';
 import {
   CreateParams,
   FetchInfoParams,
   NftAssetDetail,
   NftVendor,
   NftVendorId,
 } from '../types';
-import { capitalize, reduceDataEntries } from '../utils';
+import { capitalize } from '../utils';
 
 const DUCKLINGS_DAPP = '3PKmLiGEfqLWMC1H9xhzqvAZKUXfFm8uoeg';
 
@@ -37,22 +41,11 @@ export class DucklingsNftVendor implements NftVendor<DucklingsNftInfo> {
 
     const nftIds = nfts.map(nft => nft.assetId);
 
-    return fetch(ducklingDataUrl(nodeUrl), {
-      method: 'POST',
-      headers: {
-        Accept: 'application/json',
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        keys: nftIds.map(ducklingLevelKey),
-      }),
-    })
-      .then(response =>
-        response.ok
-          ? response.json()
-          : response.text().then(text => Promise.reject(new Error(text)))
-      )
-      .then(reduceDataEntries)
+    return fetchDataEntries(
+      ducklingDataUrl(nodeUrl),
+      nftIds.map(ducklingLevelKey)
+    )
+      .then(dataEntriesToRecord)
       .then(dataEntries =>
         nftIds.map((id): DucklingsNftInfo => {
           // eslint-disable-next-line radix

diff --git a/src/nfts/vendors/signArt.ts b/src/nfts/vendors/signArt.ts
@@ -1,11 +1,17 @@
+import { DataTransactionEntryString } from '@waves/ts-types/src/parts';
+import invariant from 'tiny-invariant';
+
+import {
+  dataEntriesToRecord,
+  fetchDataEntries,
+} from '../../nodeApi/dataEntries';
 import {
   CreateParams,
   FetchInfoParams,
   NftAssetDetail,
   NftVendor,
   NftVendorId,
 } from '../types';
-import { reduceDataEntries } from '../utils';
 
 const SIGN_ART_DAPP = '3PDBLdsUrcsiPxNbt8g2gQVoefKgzt3kJzV';
 const SIGN_ART_USER_DAPP = '3PGSWDgad4RtceQYXBpq2x73mXLRJYLRqRP';
@@ -47,72 +53,42 @@ export class SignArtNftVendor implements NftVendor<SignArtNftInfo> {
 
     const nftIds = nfts.map(nft => nft.assetId);
 
-    return fetch(signArtDataUrl(nodeUrl), {
-      method: 'POST',
-      headers: {
-        Accept: 'application/json',
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        keys: nftIds.map(id => nftIdKey(id)),
-      }),
-    })
-      .then(response =>
-        response.ok
-          ? response.json()
-          : response.text().then(text => Promise.reject(new Error(text)))
-      )
-      .then(reduceDataEntries)
+    return fetchDataEntries<DataTransactionEntryString>(
+      signArtDataUrl(nodeUrl),
+      nftIds.map(id => nftIdKey(id))
+    )
+      .then(dataEntriesToRecord)
       .then(dataEntries =>
         nftIds.map(id => {
           const value = dataEntries[nftIdKey(id)];
-          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-          const [, artworkId, creator] = (value as string).match(
-            /art_sold_\d+_of_\d+_(\w+)_(\w+)/i
-          )!;
+
+          const match = value.match(/art_sold_\d+_of_\d+_(\w+)_(\w+)/i);
+          invariant(match);
+          const [, artworkId, creator] = match;
+
           return { artworkId, creator };
         })
       )
       .then(artworks =>
         Promise.all([
-          fetch(signArtDataUrl(nodeUrl), {
-            method: 'POST',
-            headers: {
-              Accept: 'application/json',
-              'Content-Type': 'application/json',
-            },
-            body: JSON.stringify({
-              keys: nftIds.flatMap((id, index) => {
-                const info = artworks[index];
-                return [
-                  `art_name_${info.artworkId}_${info.creator}`,
-                  `art_desc_${info.artworkId}_${info.creator}`,
-                  `art_display_cid_${info.artworkId}_${info.creator}`,
-                  `art_type_${info.artworkId}_${info.creator}`,
-                ];
-              }),
-            }),
-          }).then(response =>
-            response.ok
-              ? response.json()
-              : response.text().then(text => Promise.reject(new Error(text)))
+          fetchDataEntries<DataTransactionEntryString>(
+            signArtDataUrl(nodeUrl),
+            nftIds.flatMap((id, index) => {
+              const info = artworks[index];
+              return [
+                `art_name_${info.artworkId}_${info.creator}`,
+                `art_desc_${info.artworkId}_${info.creator}`,
+                `art_display_cid_${info.artworkId}_${info.creator}`,
+                `art_type_${info.artworkId}_${info.creator}`,
+              ];
+            })
           ),
-          fetch(signArtUserDataUrl(nodeUrl), {
-            method: 'POST',
-            headers: {
-              Accept: 'application/json',
-              'Content-Type': 'application/json',
-            },
-            body: JSON.stringify({
-              keys: nftIds.map((id, index) => {
-                const info = artworks[index];
-                return `user_name_${info.creator}`;
-              }),
-            }),
-          }).then(response =>
-            response.ok
-              ? response.json()
-              : response.text().then(text => Promise.reject(new Error(text)))
+          fetchDataEntries<DataTransactionEntryString>(
+            signArtUserDataUrl(nodeUrl),
+            nftIds.map((id, index) => {
+              const info = artworks[index];
+              return `user_name_${info.creator}`;
+            })
           ),
         ])
       )
@@ -123,8 +99,9 @@ export class SignArtNftVendor implements NftVendor<SignArtNftInfo> {
           const artDesc = artworksEntries[entriesPerAsset * index + 1];
           const artDisplayCid = artworksEntries[entriesPerAsset * index + 2];
           const userName = userNameEntries[index];
-          const [, artworkId, creator] =
-            artName.key.match(/art_name_(\w+)_(\w+)/i);
+          const match = artName.key.match(/art_name_(\w+)_(\w+)/i);
+          invariant(match);
+          const [, artworkId, creator] = match;
 
           return {
             id,

diff --git a/src/nodeApi/dataEntries.ts b/src/nodeApi/dataEntries.ts
@@ -0,0 +1,35 @@
+import { DataTransactionEntry } from '@waves/ts-types';
+
+import { fetchInBatches } from '../_core/fetchInBatches';
+
+export function dataEntriesToRecord<T extends DataTransactionEntry>(
+  entries: T[]
+) {
+  return entries.reduce<Record<string, T['value']>>((data, item) => {
+    data[item.key] = item.value;
+    return data;
+  }, {});
+}
+
+const NODE_DATA_KEYS_REQUEST_LIMIT = 1000;
+
+export function fetchDataEntries<T extends DataTransactionEntry>(
+  url: string,
+  allKeys: string[]
+) {
+  return fetchInBatches(allKeys, NODE_DATA_KEYS_REQUEST_LIMIT, keys =>
+    fetch(url, {
+      method: 'POST',
+      headers: {
+        Accept: 'application/json',
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ keys }),
+    }).then(
+      (response): Promise<T[]> =>
+        response.ok
+          ? response.json()
+          : response.text().then(text => Promise.reject(new Error(text)))
+    )
+  );
+}