From b207d663735f3aa78b7d0ec01077c0b8bac3ea19 Mon Sep 17 00:00:00 2001 From: Adam Gall Date: Thu, 7 Mar 2024 01:14:18 -0500 Subject: [PATCH] Refactor to compartmentalize code --- .env | 4 + netlify/functions/tokenPrices.mts | 412 ++++++++++++++++-------------- 2 files changed, 222 insertions(+), 194 deletions(-) diff --git a/.env b/.env index 977667056e..c36bb2b353 100644 --- a/.env +++ b/.env @@ -16,5 +16,9 @@ NEXT_PUBLIC_SITE_URL="https://app.dev.fractalframework.xyz/" NEXT_PUBLIC_WALLET_CONNECT_PROJECT_ID="" # CoinGecko API key COINGECKO_API_KEY="" +# Minutes to cache prices for valid token addresses +TOKEN_PRICE_VALID_CACHE_MINUTES="" +# Minutes to cache "0" for invalid token addresses +TOKEN_PRICE_INVALID_CACHE_MINUTES="" # Shutter Public Key NEXT_PUBLIC_SHUTTER_EON_PUBKEY=0x0e6493bbb4ee8b19aa9b70367685049ff01dc9382c46aed83f8bc07d2a5ba3e6030bd83b942c1fd3dff5b79bef3b40bf6b666e51e7f0be14ed62daaffad47435265f5c9403b1a801921981f7d8659a9bd91fe92fb1cf9afdb16178a532adfaf51a237103874bb03afafe9cab2118dae1be5f08a0a28bf488c1581e9db4bc23ca \ No newline at end of file diff --git a/netlify/functions/tokenPrices.mts b/netlify/functions/tokenPrices.mts index 7338220442..ccaf81d4cb 100644 --- a/netlify/functions/tokenPrices.mts +++ b/netlify/functions/tokenPrices.mts @@ -1,6 +1,10 @@ -import { getStore } from '@netlify/blobs'; +// eslint-disable-next-line import/named +import { Store, getStore } from '@netlify/blobs'; import { ethers } from 'ethers'; +const PUBLIC_DEMO_API_BASE_URL = 'https://api.coingecko.com/api/v3/'; +const AUTH_QUERY_PARAM = `?x_cg_demo_api_key=${process.env.COINGECKO_API_KEY}`; + type TokenPriceWithMetadata = { data: { tokenAddress: string; @@ -11,232 +15,252 @@ type TokenPriceWithMetadata = { }; }; -export default async function getTokenPrices(request: Request) { - if (!process.env.COINGECKO_API_KEY) { - console.error('CoinGecko API key is missing'); - return Response.json({ error: 'Unknown error while fetching prices' }, { status: 503 }); - } - - // First we want to pull the tokens off of the request's query param. - const tokensStringParam = new URL(request.url).searchParams.get('tokens'); - if (!tokensStringParam) { - return Response.json({ error: 'Tokens to request were not provided' }, { status: 400 }); - } +type Config = { + store: Store; + now: number; + invalidAddressCacheTime: number; + validAddressCacheTime: number; +}; - // Sanitize user input - const rawTokenAddresses = tokensStringParam.split(','); +function sanitizeUserInput(tokensString: string) { + const rawTokenAddresses = tokensString.split(','); const needEthereum = rawTokenAddresses.map(address => address.toLowerCase()).includes('ethereum'); const validTokenAddresses = rawTokenAddresses.filter(address => ethers.utils.isAddress(address)); const lowerCaseTokenAddresses = validTokenAddresses.map(address => address.toLowerCase()); const tokens = [...new Set(lowerCaseTokenAddresses)]; if (needEthereum) tokens.push('ethereum'); + return { tokens, needEthereum }; +} - // Let's immediately build up our repsonse object, containing each - // token address and an value of 0. We'll modify this along the way - // populating it with more relevant prices. - const responseBody: Record = tokens.reduce((p, c) => ({ ...p, [c]: 0 }), {}); +async function splitData( + config: Config, + tokens: string[], + responseBodyCallback: (address: string, price: number) => void +) { + // Try to get all of the tokens from our store. + // Any token address that we don't have a record for will + // come back as null. + const possibleCachedTokenPrices = await Promise.all( + tokens.map( + tokenAddress => + config.store.getWithMetadata(tokenAddress, { + type: 'json', + }) as Promise | null + ) + ); + + // Filter out the null values, leaving us with an array of + // TokenPricesWithMetadata. All of these TokenPrices will be either + // expired or unexpired. + const cachedTokenPrices = possibleCachedTokenPrices.filter( + (possible): possible is TokenPriceWithMetadata => possible !== null + ); + + // Let's pull out all of the expired addresses from our cache. + const expiredCachedTokenAddresses = cachedTokenPrices + .filter(tokenPrice => tokenPrice.metadata.expiration < config.now) + .map(tokenPrice => tokenPrice.data.tokenAddress); + + // Finally let's get a list of all of the token addresses that + // we don't have any record of in our cache. + const uncachedTokenAddresses = tokens.filter( + address => + !cachedTokenPrices.find(cachedTokenPrice => cachedTokenPrice.data.tokenAddress === address) + ); + + // We'll update our response with those cached expired and + // unexpired prices. + cachedTokenPrices.forEach(tokenPrice => { + responseBodyCallback(tokenPrice.data.tokenAddress, tokenPrice.data.price); + }); + + return { expiredCachedTokenAddresses, uncachedTokenAddresses }; +} - const store = getStore('token-prices'); +function getTokenPricesUrl(tokens: string[]) { + const tokenPricesUrl = `${PUBLIC_DEMO_API_BASE_URL}simple/token_price/ethereum/${AUTH_QUERY_PARAM}&vs_currencies=usd&contract_addresses=${tokens.join( + ',' + )}`; + return tokenPricesUrl; +} - try { - const now = Math.floor(Date.now() / 1000); - - // Try to get all of the tokens from our store. - // Any token address that we don't have a record for will - // come back as null. - const possibleCachedTokenPrices = await Promise.all( - tokens.map( - tokenAddress => - store.getWithMetadata(tokenAddress, { - type: 'json', - }) as Promise | null - ) - ); +function getEthereumPriceUrl() { + const ethPriceUrl = `${PUBLIC_DEMO_API_BASE_URL}simple/price${AUTH_QUERY_PARAM}&ids=ethereum&vs_currencies=usd`; + return ethPriceUrl; +} - // Filter out the null values, leaving us with an array of - // TokenPricesWithMetadata. All of these TokenPrices will be either - // expired or unexpired. - const allCachedTokenPrices = possibleCachedTokenPrices.filter( - (possible): possible is TokenPriceWithMetadata => possible !== null - ); +async function storeTokenPrice( + config: Config, + tokenAddress: string, + price: number, + expiration: number +) { + await config.store.setJSON( + tokenAddress, + { tokenAddress, price }, + { metadata: { expiration: config.now + expiration } } + ); +} - console.log('allCachedTokenPrices'); - console.log( - allCachedTokenPrices.map(a => ({ - address: a.data.tokenAddress, - price: a.data.price, - now: ' ' + now, - expiration: a.metadata.expiration, - expired: a.metadata.expiration < now, - })) - ); +async function processTokenPricesResponse( + config: Config, + tokenPricesResponseJson: Record, + responseBodyCallback: (address: string, price: number) => void +) { + const coinGeckoResponseAddresses = Object.keys(tokenPricesResponseJson); + for await (const tokenAddress of coinGeckoResponseAddresses) { + const price = tokenPricesResponseJson[tokenAddress].usd; + const sanitizedAddress = tokenAddress.toLowerCase(); + + // Sometimes no USD price is returned. If this happens, + // we should consider it as though CoinGecko doesn't support + // this address and not query it again for a while. + if (price === undefined) { + await storeTokenPrice(config, sanitizedAddress, 0, config.invalidAddressCacheTime); + } else { + // Otherwise, update the cache with the new price and update + // the response object. + responseBodyCallback(sanitizedAddress, price); + await storeTokenPrice(config, sanitizedAddress, price, config.validAddressCacheTime); + } + } - // Let's pull out all of the unexpired TokenPrices from our cache. - const cachedUnexpiredTokenPrices = allCachedTokenPrices.filter( - tokenPrice => tokenPrice.metadata.expiration >= now - ); + return coinGeckoResponseAddresses; +} - console.log('cachedUnexpiredTokenPrices'); - console.log( - cachedUnexpiredTokenPrices.map(a => ({ - address: a.data.tokenAddress, - })) - ); +async function processUnknownAddresses( + config: Config, + needPricesTokenAddresses: string[], + responseAddresses: string[] +) { + const unknownAddresses = needPricesTokenAddresses + .filter(x => !responseAddresses.includes(x)) + .map(address => address.toLowerCase()); + for await (const tokenAddress of unknownAddresses) { + await storeTokenPrice(config, tokenAddress, 0, config.invalidAddressCacheTime); + } +} - // We'll update our response with those unexpired cached prices. - cachedUnexpiredTokenPrices.forEach(tokenPrice => { - responseBody[tokenPrice.data.tokenAddress] = tokenPrice.data.price; - }); +async function coinGeckoRequestAndResponse( + config: Config, + url: string, + responseBodyCallback: (address: string, price: number) => void +) { + // Make the request to CoinGecko. + // Response is of shape: + // { + // [tokenAddress]: { usd: 1234 }, + // } + let ethPriceResponseJson: Record; + try { + const ethPriceResponse = await fetch(url); + ethPriceResponseJson = await ethPriceResponse.json(); + } catch (e) { + throw e; + } - // Let's pull out all of the expired TokenPrices from our cache. - const cachedExpiredTokenPrices = allCachedTokenPrices.filter( - tokenPrice => tokenPrice.metadata.expiration < now - ); + // Update the cache with the new price and update + // the response object. + const responseAddresses = processTokenPricesResponse( + config, + ethPriceResponseJson, + responseBodyCallback + ); - console.log('cachedExpiredTokenPrices'); - console.log( - cachedExpiredTokenPrices.map(a => ({ - address: a.data.tokenAddress, - })) - ); - - // We'll update our response with those expired cached prices. - // This is done now in the offchance that we won't be able to contact - // CoinGecko later. Ideally these will be updated after getting - // fresher data from CoinGecko - cachedExpiredTokenPrices.forEach(tokenPrice => { - responseBody[tokenPrice.data.tokenAddress] = tokenPrice.data.price; - }); - - // Finally let's get a list of all of the token addresses that - // we don't have any price for in our cache, expired or not. - const allUncachedTokenAddresses = tokens.filter( - address => - !allCachedTokenPrices.find( - cachedTokenPrice => cachedTokenPrice.data.tokenAddress === address - ) - ); + return responseAddresses; +} - console.log('allUncachedTokenAddresses'); - console.log( - allUncachedTokenAddresses.map(a => ({ - address: a, - })) - ); +export default async function getTokenPrices(request: Request) { + if (!process.env.COINGECKO_API_KEY) { + console.error('CoinGecko API key is missing'); + return Response.json({ error: 'Error while fetching prices' }, { status: 503 }); + } - // If there are no expired token prices, and no token addresses that we - // don't have a cached value for at all, we can early return! - if (allUncachedTokenAddresses.length === 0 && cachedExpiredTokenPrices.length === 0) { - console.log('early exit'); - console.log({ responseBody }); - return Response.json({ data: responseBody }); - } + if (!process.env.TOKEN_PRICE_INVALID_CACHE_MINUTES) { + console.error('TOKEN_PRICE_INVALID_CACHE_MINUTES is not set'); + return Response.json({ error: 'Error while fetching prices' }, { status: 503 }); + } - // If we got here, then we have either some expired prices for given tokens, - // or no prices at all for given tokens. + if (!process.env.TOKEN_PRICE_VALID_CACHE_MINUTES) { + console.error('TOKEN_PRICE_VALID_CACHE_MINUTES is not set'); + return Response.json({ error: 'Error while fetching prices' }, { status: 503 }); + } - // First, let's build up our list of token addresses to query CoinGecko with, - // which is all uncached tokens and tokens that have expired. - // Remove "ethereum" if it's in this list - const needPricesTokenAddresses = [ - ...allUncachedTokenAddresses, - ...cachedExpiredTokenPrices.map(tokenPrice => tokenPrice.data.tokenAddress), - ].filter(address => address !== 'ethereum'); + const tokensStringParam = new URL(request.url).searchParams.get('tokens'); + if (!tokensStringParam) { + return Response.json({ error: 'Tokens missing from request' }, { status: 400 }); + } - try { - // Build up the request URL. - const PUBLIC_DEMO_API_BASE_URL = 'https://api.coingecko.com/api/v3/'; - const AUTH_QUERY_PARAM = `?x_cg_demo_api_key=${process.env.COINGECKO_API_KEY}`; - const tokenPricesUrl = `${PUBLIC_DEMO_API_BASE_URL}simple/token_price/ethereum/${AUTH_QUERY_PARAM}&vs_currencies=usd&contract_addresses=${needPricesTokenAddresses.join( - ',' - )}`; - - console.log('making CoinGecko API call:', tokenPricesUrl); - - // Make the request. - const tokenPricesResponse = await fetch(tokenPricesUrl); - const tokenPricesResponseJson = await tokenPricesResponse.json(); - - // Create the metadata for our new token prices, with an - // expiration in the future. - const tokenPriceMetadata = { metadata: { expiration: now + 48 } }; - - // With our response... - const coinGeckoResponseAddresses = Object.keys(tokenPricesResponseJson); - coinGeckoResponseAddresses.forEach(tokenAddress => { - const price: number | undefined = tokenPricesResponseJson[tokenAddress].usd; - - const sanitizedAddress = tokenAddress.toLowerCase(); - - // Sometimes no USD price is returned - if (price === undefined) { - store.setJSON( - sanitizedAddress, - { tokenAddress: sanitizedAddress, price: 0 }, - { metadata: { expiration: now + 60 * 10 } } - ); - return; - } - - // 1. Replace the token addresses of our existing response object with the new prices. - responseBody[sanitizedAddress] = price; - // 2. Store these fresh prices in our Blob store. - store.setJSON( - sanitizedAddress, - { tokenAddress: sanitizedAddress, price }, - tokenPriceMetadata - ); - }); + const store = getStore('token-prices'); + const now = Math.floor(Date.now() / 1000); + const invalidAddressCacheTime = parseInt(process.env.TOKEN_PRICE_INVALID_CACHE_MINUTES); + const validAddressCacheTime = parseInt(process.env.TOKEN_PRICE_VALID_CACHE_MINUTES); + const config = { store, now, invalidAddressCacheTime, validAddressCacheTime }; - // CoinGecko will only respond back with prices for token addresses - // that it knows about. We should store a price of 0 in our store with - // a long expiration for all addresses that CoinGecko isn't tracking - // (likely spam tokens), so as to not continually query CoinGecko with - // these addresses - const likelySpamAddresses = needPricesTokenAddresses.filter( - x => !coinGeckoResponseAddresses.includes(x) - ); - console.log({ likelySpamAddresses }); - likelySpamAddresses.forEach(tokenAddress => { - const sanitizedAddress = tokenAddress.toLowerCase(); - store.setJSON( - sanitizedAddress, - { tokenAddress: sanitizedAddress, price: 0 }, - { metadata: { expiration: now + 60 * 10 } } - ); - }); + const { tokens, needEthereum } = sanitizeUserInput(tokensStringParam); - // Do we need to get the price of our chain's gas token (ethereum)? - if (needEthereum) { - // Build up the request URL. - const ethPriceUrl = `${PUBLIC_DEMO_API_BASE_URL}simple/price${AUTH_QUERY_PARAM}&ids=ethereum&vs_currencies=usd`; + // Let's immediately build up our repsonse object, containing each + // token address and an value of 0. We'll modify this along the way + // populating it with more relevant prices. + const responseBody: Record = tokens.reduce((p, c) => ({ ...p, [c]: 0 }), {}); - console.log('making CoinGecko API call:', ethPriceUrl); + const { expiredCachedTokenAddresses, uncachedTokenAddresses } = await splitData( + config, + tokens, + (address, price) => { + responseBody[address] = price; + } + ); - // Make the request. - const ethPriceResponse = await fetch(ethPriceUrl); - const ethPriceResponseJson = await ethPriceResponse.json(); + // If there are no expired token prices, and no token addresses that we + // don't have a cached value for at all, we can early return! + if (expiredCachedTokenAddresses.length === 0 && uncachedTokenAddresses.length === 0) { + return Response.json({ data: responseBody }); + } - // Get the price data. - const price: number | undefined = ethPriceResponseJson.ethereum.usd; + // If we got here, then we have either some expired prices for given tokens, + // or no prices at all for given tokens. - if (price !== undefined) { - // 1. Replace the token addresses of our existing response object with the new prices. - responseBody.ethereum = price; + // First, let's build up our list of token addresses to query CoinGecko with, + // which is all uncached tokens and tokens that have expired. + // Remove "ethereum" if it's in this list. + const needPricesTokenAddresses = [ + ...uncachedTokenAddresses, + ...expiredCachedTokenAddresses, + ].filter(address => address !== 'ethereum'); - // 2. Store this fresh prices in our Blob store. - store.setJSON('ethereum', { tokenAddress: 'ethereum', price }, tokenPriceMetadata); - } + let responseAddresses: string[]; + try { + responseAddresses = await coinGeckoRequestAndResponse( + config, + getTokenPricesUrl(needPricesTokenAddresses), + (address, price) => { + responseBody[address] = price; } + ); + } catch (e) { + console.error('Error while querying CoinGecko', e); + return Response.json({ error: 'Error while fetching prices', data: responseBody }); + } - console.log({ responseBody }); - return Response.json({ data: responseBody }); + // In the previous request, CoinGecko will only respond back with prices + // for token addresses that it knows about. We should store a price of 0 + // in our store with a long expiration for all addresses that CoinGecko + // isn't tracking (likely spam tokens), so as to not continually query + // CoinGecko with these addresses + await processUnknownAddresses(config, needPricesTokenAddresses, responseAddresses); + + // Do we need to get the price of our chain's gas token (ethereum)? + if (needEthereum) { + try { + await coinGeckoRequestAndResponse(config, getEthereumPriceUrl(), (address, price) => { + responseBody[address] = price; + }); } catch (e) { console.error('Error while querying CoinGecko', e); return Response.json({ error: 'Error while fetching prices', data: responseBody }); } - } catch (e) { - console.error('Error while fetching prices', e); - return Response.json({ error: 'Error while fetching prices', data: responseBody }); } + + return Response.json({ data: responseBody }); }