From d0facd597ad0086cf7eda51efc315dc6ab8d852d Mon Sep 17 00:00:00 2001 From: Timofei Domnikov Date: Tue, 31 Jan 2023 20:36:22 +0300 Subject: [PATCH] KEEP-1063: handle data-entries limit --- src/_core/chunk.ts | 5 ++ src/_core/fetchInBatches.ts | 20 ++++++++ src/nfts/utils.ts | 12 ----- src/nfts/vendors/ducklings.ts | 27 ++++------ src/nfts/vendors/signArt.ts | 97 +++++++++++++---------------------- src/nodeApi/dataEntries.ts | 35 +++++++++++++ 6 files changed, 107 insertions(+), 89 deletions(-) create mode 100644 src/_core/chunk.ts create mode 100644 src/_core/fetchInBatches.ts create mode 100644 src/nodeApi/dataEntries.ts diff --git a/src/_core/chunk.ts b/src/_core/chunk.ts new file mode 100644 index 000000000..48d741a1a --- /dev/null +++ b/src/_core/chunk.ts @@ -0,0 +1,5 @@ +export function* chunk(array: T[], size: number) { + for (let i = 0; i < array.length; i += size) { + yield array.slice(i, i + size); + } +} diff --git a/src/_core/fetchInBatches.ts b/src/_core/fetchInBatches.ts new file mode 100644 index 000000000..19ba0a670 --- /dev/null +++ b/src/_core/fetchInBatches.ts @@ -0,0 +1,20 @@ +import invariant from 'tiny-invariant'; + +import { chunk } from './chunk'; + +export async function fetchInBatches( + allItems: I[], + chunkSize: number, + fetchFn: (items: I[]) => Promise +) { + const result: T[] = []; + + for (const items of chunk(allItems, chunkSize)) { + const data = await fetchFn(items); + invariant(Array.isArray(data)); + + result.push(...data); + } + + return result; +} diff --git a/src/nfts/utils.ts b/src/nfts/utils.ts index 6d3c9bb4d..56726906a 100644 --- a/src/nfts/utils.ts +++ b/src/nfts/utils.ts @@ -1,15 +1,3 @@ -import { DataTransactionEntry } from '@waves/ts-types'; - export function capitalize(str: string | undefined) { return str ? str.charAt(0).toUpperCase() + str.slice(1) : str; } - -export function reduceDataEntries(entries: DataTransactionEntry[]) { - return entries.reduce>( - (data, item) => { - data[item.key] = item.value; - return data; - }, - {} - ); -} diff --git a/src/nfts/vendors/ducklings.ts b/src/nfts/vendors/ducklings.ts index b1e808500..b4602fad8 100644 --- a/src/nfts/vendors/ducklings.ts +++ b/src/nfts/vendors/ducklings.ts @@ -1,3 +1,7 @@ +import { + dataEntriesToRecord, + fetchDataEntries, +} from '../../nodeApi/dataEntries'; import { CreateParams, FetchInfoParams, @@ -5,7 +9,7 @@ import { NftVendor, NftVendorId, } from '../types'; -import { capitalize, reduceDataEntries } from '../utils'; +import { capitalize } from '../utils'; const DUCKLINGS_DAPP = '3PKmLiGEfqLWMC1H9xhzqvAZKUXfFm8uoeg'; @@ -37,22 +41,11 @@ export class DucklingsNftVendor implements NftVendor { const nftIds = nfts.map(nft => nft.assetId); - return fetch(ducklingDataUrl(nodeUrl), { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - keys: nftIds.map(ducklingLevelKey), - }), - }) - .then(response => - response.ok - ? response.json() - : response.text().then(text => Promise.reject(new Error(text))) - ) - .then(reduceDataEntries) + return fetchDataEntries( + ducklingDataUrl(nodeUrl), + nftIds.map(ducklingLevelKey) + ) + .then(dataEntriesToRecord) .then(dataEntries => nftIds.map((id): DucklingsNftInfo => { // eslint-disable-next-line radix diff --git a/src/nfts/vendors/signArt.ts b/src/nfts/vendors/signArt.ts index dfb5f1ecf..077535627 100644 --- a/src/nfts/vendors/signArt.ts +++ b/src/nfts/vendors/signArt.ts @@ -1,3 +1,10 @@ +import { DataTransactionEntryString } from '@waves/ts-types/src/parts'; +import invariant from 'tiny-invariant'; + +import { + dataEntriesToRecord, + fetchDataEntries, +} from '../../nodeApi/dataEntries'; import { CreateParams, FetchInfoParams, @@ -5,7 +12,6 @@ import { NftVendor, NftVendorId, } from '../types'; -import { reduceDataEntries } from '../utils'; const SIGN_ART_DAPP = '3PDBLdsUrcsiPxNbt8g2gQVoefKgzt3kJzV'; const SIGN_ART_USER_DAPP = '3PGSWDgad4RtceQYXBpq2x73mXLRJYLRqRP'; @@ -47,72 +53,42 @@ export class SignArtNftVendor implements NftVendor { const nftIds = nfts.map(nft => nft.assetId); - return fetch(signArtDataUrl(nodeUrl), { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - keys: nftIds.map(id => nftIdKey(id)), - }), - }) - .then(response => - response.ok - ? response.json() - : response.text().then(text => Promise.reject(new Error(text))) - ) - .then(reduceDataEntries) + return fetchDataEntries( + signArtDataUrl(nodeUrl), + nftIds.map(id => nftIdKey(id)) + ) + .then(dataEntriesToRecord) .then(dataEntries => nftIds.map(id => { const value = dataEntries[nftIdKey(id)]; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const [, artworkId, creator] = (value as string).match( - /art_sold_\d+_of_\d+_(\w+)_(\w+)/i - )!; + + const match = value.match(/art_sold_\d+_of_\d+_(\w+)_(\w+)/i); + invariant(match); + const [, artworkId, creator] = match; + return { artworkId, creator }; }) ) .then(artworks => Promise.all([ - fetch(signArtDataUrl(nodeUrl), { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - keys: nftIds.flatMap((id, index) => { - const info = artworks[index]; - return [ - `art_name_${info.artworkId}_${info.creator}`, - `art_desc_${info.artworkId}_${info.creator}`, - `art_display_cid_${info.artworkId}_${info.creator}`, - `art_type_${info.artworkId}_${info.creator}`, - ]; - }), - }), - }).then(response => - response.ok - ? response.json() - : response.text().then(text => Promise.reject(new Error(text))) + fetchDataEntries( + signArtDataUrl(nodeUrl), + nftIds.flatMap((id, index) => { + const info = artworks[index]; + return [ + `art_name_${info.artworkId}_${info.creator}`, + `art_desc_${info.artworkId}_${info.creator}`, + `art_display_cid_${info.artworkId}_${info.creator}`, + `art_type_${info.artworkId}_${info.creator}`, + ]; + }) ), - fetch(signArtUserDataUrl(nodeUrl), { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - keys: nftIds.map((id, index) => { - const info = artworks[index]; - return `user_name_${info.creator}`; - }), - }), - }).then(response => - response.ok - ? response.json() - : response.text().then(text => Promise.reject(new Error(text))) + fetchDataEntries( + signArtUserDataUrl(nodeUrl), + nftIds.map((id, index) => { + const info = artworks[index]; + return `user_name_${info.creator}`; + }) ), ]) ) @@ -123,8 +99,9 @@ export class SignArtNftVendor implements NftVendor { const artDesc = artworksEntries[entriesPerAsset * index + 1]; const artDisplayCid = artworksEntries[entriesPerAsset * index + 2]; const userName = userNameEntries[index]; - const [, artworkId, creator] = - artName.key.match(/art_name_(\w+)_(\w+)/i); + const match = artName.key.match(/art_name_(\w+)_(\w+)/i); + invariant(match); + const [, artworkId, creator] = match; return { id, diff --git a/src/nodeApi/dataEntries.ts b/src/nodeApi/dataEntries.ts new file mode 100644 index 000000000..a9e1468c1 --- /dev/null +++ b/src/nodeApi/dataEntries.ts @@ -0,0 +1,35 @@ +import { DataTransactionEntry } from '@waves/ts-types'; + +import { fetchInBatches } from '../_core/fetchInBatches'; + +export function dataEntriesToRecord( + entries: T[] +) { + return entries.reduce>((data, item) => { + data[item.key] = item.value; + return data; + }, {}); +} + +const NODE_DATA_KEYS_REQUEST_LIMIT = 1000; + +export function fetchDataEntries( + url: string, + allKeys: string[] +) { + return fetchInBatches(allKeys, NODE_DATA_KEYS_REQUEST_LIMIT, keys => + fetch(url, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ keys }), + }).then( + (response): Promise => + response.ok + ? response.json() + : response.text().then(text => Promise.reject(new Error(text))) + ) + ); +}