From faa9c41a6288d1654f335a469f19fd2e19462632 Mon Sep 17 00:00:00 2001 From: mktcode Date: Mon, 5 Aug 2024 16:37:44 +0200 Subject: [PATCH 01/41] feat: add test for token list resolver --- test/e2e/api.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index 25652891..4511b12c 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -3,6 +3,7 @@ import redis from '../../src/helpers/redis'; import { KEY_PREFIX } from '../../src/addressResolvers/cache'; const HOST = `http://localhost:${process.env.PORT || 3003}`; +const TOKEN_ADDRESS = '0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e'; async function purge(): Promise { if (!redis) return; @@ -16,6 +17,14 @@ async function purge(): Promise { describe('E2E api', () => { describe('GET type/TYPE/ID', () => { + it('returns an image for tokenlists resolver', async () => { + const response = await axios.get(`${HOST}/token/${TOKEN_ADDRESS}?resolver=tokenlists`); + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe('image/webp'); + expect(response.headers['x-stamp-token-resolver']).toBe('tokenlists'); + expect(response.headers['x-stamp-token-fallback']).toBe(undefined); + }); + it.todo('returns a 500 status on invalid query'); describe('when the image is not cached', () => { From bc0321acdfe6bbd90b192c3fd0bab6c4c14631e5 Mon Sep 17 00:00:00 2001 From: mktcode Date: Mon, 5 Aug 2024 18:03:49 +0200 Subject: [PATCH 02/41] feat: allow additional headers for image responses, add fallback header I think this will be helpful for testing and for apps using it too. --- src/api.ts | 4 +++- src/utils.ts | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/api.ts b/src/api.ts index 765c0eb9..766f5617 100644 --- a/src/api.ts +++ b/src/api.ts @@ -122,7 +122,9 @@ router.get(`/:type(${TYPE_CONSTRAINTS})/:id`, async (req, res) => { const fallbackImage = await resolvers[fallback](address, network); const resizedImage = await resize(fallbackImage, w, h); - setHeader(res, 'SHORT_CACHE'); + setHeader(res, 'SHORT_CACHE', { + [`x-stamp-${type}-fallback`]: fallback + }); return res.send(resizedImage); } } diff --git a/src/utils.ts b/src/utils.ts index 37308264..fad2c837 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -138,14 +138,20 @@ export function getCacheKey({ return sha256(JSON.stringify(data)); } -export function setHeader(res: Response, cacheType: 'SHORT_CACHE' | 'LONG_CACHE' = 'LONG_CACHE') { +export function setHeader( + res: Response, + cacheType: 'SHORT_CACHE' | 'LONG_CACHE' = 'LONG_CACHE', + extraHeaders: Record = {} +) { const ttl = cacheType === 'SHORT_CACHE' ? constants.shortTtl : constants.ttl; - res.set({ + const headers = { 'Content-Type': 'image/webp', 'Cache-Control': `public, max-age=${ttl}`, Expires: new Date(Date.now() + ttl * 1e3).toUTCString() - }); + }; + + res.set({ ...headers, ...extraHeaders }); } export const getBaseAssetIconUrl = (chainId: string) => { From ed233a7bbd15c1dad300588689f4ae13fdeaad02 Mon Sep 17 00:00:00 2001 From: mktcode Date: Mon, 5 Aug 2024 18:08:36 +0200 Subject: [PATCH 03/41] feat: add "empty" tokenlists resolver What's missing is the actual processing/resolving. --- src/constants.json | 2 +- src/helpers/tokenlists.ts | 20 ++++++++++++++++++++ src/resolvers/index.ts | 4 +++- src/resolvers/tokenlists.ts | 26 ++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 src/helpers/tokenlists.ts create mode 100644 src/resolvers/tokenlists.ts diff --git a/src/constants.json b/src/constants.json index 567600c1..b3331d5f 100644 --- a/src/constants.json +++ b/src/constants.json @@ -6,7 +6,7 @@ "resolvers": { "avatar": ["snapshot", "ens", "lens", "farcaster", "starknet"], "user-cover": ["user-cover"], - "token": ["trustwallet", "zapper"], + "token": ["trustwallet", "zapper", "tokenlists"], "space": ["space"], "space-sx": ["space-sx"], "space-cover-sx": ["space-cover-sx"] diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts new file mode 100644 index 00000000..fa64b319 --- /dev/null +++ b/src/helpers/tokenlists.ts @@ -0,0 +1,20 @@ +// (draft from top of my head) +export type TokenLists = { + [network: string]: { + name: string; + logoURI: string; + tokens: { + address: string; + chainId: number; + name: string; + symbol: string; + decimals: number; + }[]; + }[]; +}; + +const tokenlists: TokenLists = {}; + +// TODO: process static tokenlists from remote sources + +export default tokenlists; diff --git a/src/resolvers/index.ts b/src/resolvers/index.ts index 909c2b4c..eeda6541 100644 --- a/src/resolvers/index.ts +++ b/src/resolvers/index.ts @@ -10,6 +10,7 @@ import lens from './lens'; import zapper from './zapper'; import starknet from './starknet'; import farcaster from './farcaster'; +import tokenlists from './tokenlists'; export default { blockie, @@ -25,5 +26,6 @@ export default { lens, zapper, starknet, - farcaster + farcaster, + tokenlists }; diff --git a/src/resolvers/tokenlists.ts b/src/resolvers/tokenlists.ts new file mode 100644 index 00000000..4baa9c2f --- /dev/null +++ b/src/resolvers/tokenlists.ts @@ -0,0 +1,26 @@ +import { getAddress } from '@ethersproject/address'; +import { resize, chainIdToName } from '../utils'; +import { max } from '../constants.json'; +import { fetchHttpImage } from './utils'; +import tokenlists from '../helpers/tokenlists'; + +function searchTokenlists(address: string, chainId: string) { + const checksum = getAddress(address); + const networkName = chainIdToName(chainId) || 'ethereum'; + + const url = tokenlists[networkName]?.find(list => + list.tokens.find(token => getAddress(token.address) === checksum) + )?.logoURI; + + return url; +} + +export default async function resolve(address: string, chainId: string) { + const url = searchTokenlists(address, chainId); + if (!url) return false; + + const image = await fetchHttpImage(url); + if (!image) return false; + + return await resize(image, max, max); +} From a97a80aaaf4095a6189defd8e3dd6cf99003f792 Mon Sep 17 00:00:00 2001 From: mktcode Date: Mon, 5 Aug 2024 18:48:09 +0200 Subject: [PATCH 04/41] refactor: improve error handling in new tokenlists resolver (still wip) --- src/helpers/tokenlists.ts | 25 ++++++++++--------------- src/resolvers/tokenlists.ts | 29 +++++++++++++++-------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index fa64b319..46849135 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -1,20 +1,15 @@ // (draft from top of my head) -export type TokenLists = { - [network: string]: { - name: string; - logoURI: string; - tokens: { - address: string; - chainId: number; - name: string; - symbol: string; - decimals: number; - }[]; - }[]; -}; +export type AggregatedTokenList = { + chainId: number; + address: string; + symbol: string; + name: string; + logoURI: string; + decimals: number; +}[]; -const tokenlists: TokenLists = {}; +const aggregatedTokenList: AggregatedTokenList = []; // TODO: process static tokenlists from remote sources -export default tokenlists; +export default aggregatedTokenList; diff --git a/src/resolvers/tokenlists.ts b/src/resolvers/tokenlists.ts index 4baa9c2f..954d1859 100644 --- a/src/resolvers/tokenlists.ts +++ b/src/resolvers/tokenlists.ts @@ -1,26 +1,27 @@ import { getAddress } from '@ethersproject/address'; -import { resize, chainIdToName } from '../utils'; +import { resize } from '../utils'; import { max } from '../constants.json'; import { fetchHttpImage } from './utils'; -import tokenlists from '../helpers/tokenlists'; +import aggregatedTokenList from '../helpers/tokenlists'; -function searchTokenlists(address: string, chainId: string) { +function findImageUrl(address: string, chainId: string) { const checksum = getAddress(address); - const networkName = chainIdToName(chainId) || 'ethereum'; - const url = tokenlists[networkName]?.find(list => - list.tokens.find(token => getAddress(token.address) === checksum) - )?.logoURI; + const token = aggregatedTokenList.find(token => { + return token.chainId === parseInt(chainId) && getAddress(token.address) === checksum; + }); + if (!token) throw new Error('Token not found'); - return url; + return token.logoURI; } export default async function resolve(address: string, chainId: string) { - const url = searchTokenlists(address, chainId); - if (!url) return false; + try { + const url = findImageUrl(address, chainId); + const image = await fetchHttpImage(url); - const image = await fetchHttpImage(url); - if (!image) return false; - - return await resize(image, max, max); + return await resize(image, max, max); + } catch (e) { + return false; + } } From b0b4e869e24569b18a85430149b3e81d333164a0 Mon Sep 17 00:00:00 2001 From: mktcode Date: Mon, 5 Aug 2024 18:56:27 +0200 Subject: [PATCH 05/41] feat: Improve handling of additional headers in setHeader function --- src/utils.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index fad2c837..810f0d37 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -151,7 +151,15 @@ export function setHeader( Expires: new Date(Date.now() + ttl * 1e3).toUTCString() }; - res.set({ ...headers, ...extraHeaders }); + Object.entries(extraHeaders).forEach(([key, value]) => { + if (typeof value === 'string') { + headers[key] = value; + } else { + console.warn('Invalid header value passed to setHeader:', key, value); + } + }); + + res.set(headers); } export const getBaseAssetIconUrl = (chainId: string) => { From 6b11cd39eca373ca63c449d8d5325e901a37074c Mon Sep 17 00:00:00 2001 From: mktcode Date: Mon, 5 Aug 2024 19:22:07 +0200 Subject: [PATCH 06/41] feat: set resolver headers in setHeader function --- src/api.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/api.ts b/src/api.ts index 766f5617..39c0a460 100644 --- a/src/api.ts +++ b/src/api.ts @@ -131,7 +131,9 @@ router.get(`/:type(${TYPE_CONSTRAINTS})/:id`, async (req, res) => { // Resize and return image const resizedImage = await resize(baseImage, w, h); - setHeader(res); + const extraHeaders = {}; + if (resolver) extraHeaders[`x-stamp-${type}-resolver`] = resolver; + setHeader(res, 'LONG_CACHE', extraHeaders); res.send(resizedImage); if (disableCache) return; From 9300e3f5c3571102f46fdbdc3c276b5a1e598354 Mon Sep 17 00:00:00 2001 From: mktcode Date: Mon, 5 Aug 2024 22:26:57 +0200 Subject: [PATCH 07/41] feat: Add check for extraHeaders in setHeader function --- src/utils.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 810f0d37..0a7b1137 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -151,13 +151,15 @@ export function setHeader( Expires: new Date(Date.now() + ttl * 1e3).toUTCString() }; - Object.entries(extraHeaders).forEach(([key, value]) => { - if (typeof value === 'string') { - headers[key] = value; - } else { - console.warn('Invalid header value passed to setHeader:', key, value); - } - }); + if (typeof extraHeaders === 'object' && extraHeaders !== null) { + Object.entries(extraHeaders).forEach(([key, value]) => { + if (typeof value === 'string') { + headers[key] = value; + } else { + console.warn('Invalid header value passed to setHeader:', key, value); + } + }); + } res.set(headers); } From 2e48ec685dff58ab63034b30ae7485828dd902e2 Mon Sep 17 00:00:00 2001 From: mktcode Date: Mon, 5 Aug 2024 23:21:37 +0200 Subject: [PATCH 08/41] feat: Add aggregated token list initialization --- src/helpers/tokenlists.ts | 19 +++++++++++++++++-- src/index.ts | 9 +++++++-- test/e2e/api.test.ts | 6 ++++-- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 46849135..066ec20f 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -1,4 +1,3 @@ -// (draft from top of my head) export type AggregatedTokenList = { chainId: number; address: string; @@ -10,6 +9,22 @@ export type AggregatedTokenList = { const aggregatedTokenList: AggregatedTokenList = []; -// TODO: process static tokenlists from remote sources +export async function initAggregatedTokenList() { + // let's start with a single one (refactor in next commits) + const response = await fetch( + 'https://raw.githubusercontent.com/compound-finance/token-list/master/compound.tokenlist.json' + ); + const tokens = await response.json(); + aggregatedTokenList.push( + ...tokens.tokens.map((token: any) => ({ + chainId: 1, + address: token.address, + symbol: token.symbol, + name: token.name, + logoURI: token.logoURI, + decimals: token.decimals + })) + ); +} export default aggregatedTokenList; diff --git a/src/index.ts b/src/index.ts index 9a9a39df..56c50df0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry'; import initMetrics from './helpers/metrics'; import api from './api'; import { name, version } from '../package.json'; +import { initAggregatedTokenList } from './helpers/tokenlists'; const app = express(); const PORT = process.env.PORT || 3008; @@ -20,7 +21,7 @@ app.use(cors({ maxAge: 86400 })); app.use(compression()); app.use('/', api); -app.get('/', (req, res) => { +app.get('/', (_req, res) => { const commit = process.env.COMMIT_HASH ?? undefined; res.json({ name, version, commit }); }); @@ -31,4 +32,8 @@ app.use((_, res) => { res.status(400).json({ message: 'Not found' }); }); -app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); +async function boot() { + await initAggregatedTokenList(); +} + +boot().then(() => app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`))); diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index 4511b12c..8eebc884 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -3,7 +3,7 @@ import redis from '../../src/helpers/redis'; import { KEY_PREFIX } from '../../src/addressResolvers/cache'; const HOST = `http://localhost:${process.env.PORT || 3003}`; -const TOKEN_ADDRESS = '0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e'; +const cUSDC_TOKEN_ADDRESS_ON_MAIN = '0x39AA39c021dfbaE8faC545936693aC917d5E7563'; async function purge(): Promise { if (!redis) return; @@ -18,7 +18,9 @@ async function purge(): Promise { describe('E2E api', () => { describe('GET type/TYPE/ID', () => { it('returns an image for tokenlists resolver', async () => { - const response = await axios.get(`${HOST}/token/${TOKEN_ADDRESS}?resolver=tokenlists`); + const response = await axios.get( + `${HOST}/token/${cUSDC_TOKEN_ADDRESS_ON_MAIN}?resolver=tokenlists` + ); expect(response.status).toBe(200); expect(response.headers['content-type']).toBe('image/webp'); expect(response.headers['x-stamp-token-resolver']).toBe('tokenlists'); From 014b9f04264dc047b0c10b86b3bd14b13e7dcf28 Mon Sep 17 00:00:00 2001 From: mktcode Date: Tue, 6 Aug 2024 22:45:25 +0200 Subject: [PATCH 09/41] feat: Initialize aggregated token list from multiple sources --- src/helpers/tokenlists.ts | 42 ++++++++++++++++++++++++++------------- src/index.ts | 7 +++++-- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 066ec20f..0b83687d 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -9,22 +9,36 @@ export type AggregatedTokenList = { const aggregatedTokenList: AggregatedTokenList = []; +function normalizeTokenListUri(tokenListUri: string) { + if (!tokenListUri.startsWith('http') && tokenListUri.endsWith('.eth')) { + tokenListUri = `https://${tokenListUri}.limo`; + } + return tokenListUri; +} + export async function initAggregatedTokenList() { - // let's start with a single one (refactor in next commits) - const response = await fetch( - 'https://raw.githubusercontent.com/compound-finance/token-list/master/compound.tokenlist.json' - ); - const tokens = await response.json(); - aggregatedTokenList.push( - ...tokens.tokens.map((token: any) => ({ - chainId: 1, - address: token.address, - symbol: token.symbol, - name: token.name, - logoURI: token.logoURI, - decimals: token.decimals - })) + console.info('Initializing aggregated token list from tokenlists.org'); + + const tokenListsResponse = await fetch( + 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json' ); + const tokenLists = await tokenListsResponse.json(); + const uris = Object.keys(tokenLists); + + // TODO: parallelize + for (let tokenListUri of uris) { + tokenListUri = normalizeTokenListUri(tokenListUri); + console.info(`Fetching list from ${tokenListUri}`); + + try { + const response = await fetch(tokenListUri); + const tokens = await response.json(); + // TODO: validate response, use zod + aggregatedTokenList.push(...tokens.tokens.filter((token: any) => token.logoURI)); + } catch (e) { + console.warn(`Failed to fetch token list from ${tokenListUri}`); + } + } } export default aggregatedTokenList; diff --git a/src/index.ts b/src/index.ts index 56c50df0..5811e424 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry'; import initMetrics from './helpers/metrics'; import api from './api'; import { name, version } from '../package.json'; -import { initAggregatedTokenList } from './helpers/tokenlists'; +import aggregatedTokenList, { initAggregatedTokenList } from './helpers/tokenlists'; const app = express(); const PORT = process.env.PORT || 3008; @@ -36,4 +36,7 @@ async function boot() { await initAggregatedTokenList(); } -boot().then(() => app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`))); +boot().then(() => { + console.info('Aggregated token list initialized', aggregatedTokenList.length); + app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); +}); From 138800137ee05ba8cc5d37860ae91c930addd693 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 03:48:06 +0200 Subject: [PATCH 10/41] feat: add todo comment --- src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.ts b/src/index.ts index 5811e424..4aed2f53 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,6 +33,7 @@ app.use((_, res) => { }); async function boot() { + // TODO: init other stuff here too (redis) and remove if !redis return elsewhere await initAggregatedTokenList(); } From ad2f62dd2a6e1119c7e448bde98b0055464d43e9 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 11:05:10 +0200 Subject: [PATCH 11/41] refactor: init tokens from tokenlists in resolver --- src/helpers/tokenlists.ts | 10 ++++++---- src/index.ts | 11 +---------- src/resolvers/tokenlists.ts | 8 +++++--- test/e2e/api.test.ts | 2 ++ 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 0b83687d..bdc152f5 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -7,8 +7,6 @@ export type AggregatedTokenList = { decimals: number; }[]; -const aggregatedTokenList: AggregatedTokenList = []; - function normalizeTokenListUri(tokenListUri: string) { if (!tokenListUri.startsWith('http') && tokenListUri.endsWith('.eth')) { tokenListUri = `https://${tokenListUri}.limo`; @@ -19,6 +17,8 @@ function normalizeTokenListUri(tokenListUri: string) { export async function initAggregatedTokenList() { console.info('Initializing aggregated token list from tokenlists.org'); + const aggregatedTokenList: AggregatedTokenList = []; + const tokenListsResponse = await fetch( 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json' ); @@ -39,6 +39,8 @@ export async function initAggregatedTokenList() { console.warn(`Failed to fetch token list from ${tokenListUri}`); } } -} -export default aggregatedTokenList; + console.info(`Aggregated token list initialized with ${aggregatedTokenList.length} tokens`); + + return aggregatedTokenList; +} diff --git a/src/index.ts b/src/index.ts index 4aed2f53..3a0221b7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,6 @@ import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry'; import initMetrics from './helpers/metrics'; import api from './api'; import { name, version } from '../package.json'; -import aggregatedTokenList, { initAggregatedTokenList } from './helpers/tokenlists'; const app = express(); const PORT = process.env.PORT || 3008; @@ -32,12 +31,4 @@ app.use((_, res) => { res.status(400).json({ message: 'Not found' }); }); -async function boot() { - // TODO: init other stuff here too (redis) and remove if !redis return elsewhere - await initAggregatedTokenList(); -} - -boot().then(() => { - console.info('Aggregated token list initialized', aggregatedTokenList.length); - app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); -}); +app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); diff --git a/src/resolvers/tokenlists.ts b/src/resolvers/tokenlists.ts index 954d1859..ab133834 100644 --- a/src/resolvers/tokenlists.ts +++ b/src/resolvers/tokenlists.ts @@ -2,11 +2,13 @@ import { getAddress } from '@ethersproject/address'; import { resize } from '../utils'; import { max } from '../constants.json'; import { fetchHttpImage } from './utils'; -import aggregatedTokenList from '../helpers/tokenlists'; +import { initAggregatedTokenList } from '../helpers/tokenlists'; -function findImageUrl(address: string, chainId: string) { +async function findImageUrl(address: string, chainId: string) { const checksum = getAddress(address); + const aggregatedTokenList = await initAggregatedTokenList(); + const token = aggregatedTokenList.find(token => { return token.chainId === parseInt(chainId) && getAddress(token.address) === checksum; }); @@ -17,7 +19,7 @@ function findImageUrl(address: string, chainId: string) { export default async function resolve(address: string, chainId: string) { try { - const url = findImageUrl(address, chainId); + const url = await findImageUrl(address, chainId); const image = await fetchHttpImage(url); return await resize(image, max, max); diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index 8eebc884..30fcb518 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -15,6 +15,8 @@ async function purge(): Promise { transaction.exec(); } +jest.setTimeout(120_000); // for token resolver + describe('E2E api', () => { describe('GET type/TYPE/ID', () => { it('returns an image for tokenlists resolver', async () => { From c98a1b4052418bce8075014f2640dfd78f1fe8fe Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 11:06:17 +0200 Subject: [PATCH 12/41] chore: comment --- test/e2e/api.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index 30fcb518..2286df33 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -15,7 +15,9 @@ async function purge(): Promise { transaction.exec(); } -jest.setTimeout(120_000); // for token resolver +// for token resolver, which currently takes very long +// not sure this is the best way to handle this +jest.setTimeout(120_000); describe('E2E api', () => { describe('GET type/TYPE/ID', () => { From 45b03d7f79a13fe02cb45fb84109a12d348cd633 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 11:33:00 +0200 Subject: [PATCH 13/41] feat: parallelize tokenlist fetching --- src/helpers/tokenlists.ts | 52 +++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index bdc152f5..65e0b4b8 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -14,31 +14,45 @@ function normalizeTokenListUri(tokenListUri: string) { return tokenListUri; } +const TOKENLISTS_URL = + 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; + +async function fetchList() { + const response = await fetch(TOKENLISTS_URL); + const tokenLists = await response.json(); + const uris = Object.keys(tokenLists); + + return uris; +} + +async function fetchTokens(tokenListUri: string) { + tokenListUri = normalizeTokenListUri(tokenListUri); + console.info(`Fetching list from ${tokenListUri}`); + + try { + const response = await fetch(tokenListUri); + const tokens = await response.json(); + // TODO: validate response, use zod + return tokens.tokens.filter((token: any) => token.logoURI); + } catch (e) { + console.warn(`Failed to fetch token list from ${tokenListUri}`); + return []; + } +} + export async function initAggregatedTokenList() { console.info('Initializing aggregated token list from tokenlists.org'); const aggregatedTokenList: AggregatedTokenList = []; - const tokenListsResponse = await fetch( - 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json' - ); - const tokenLists = await tokenListsResponse.json(); - const uris = Object.keys(tokenLists); + const uris = await fetchList(); - // TODO: parallelize - for (let tokenListUri of uris) { - tokenListUri = normalizeTokenListUri(tokenListUri); - console.info(`Fetching list from ${tokenListUri}`); - - try { - const response = await fetch(tokenListUri); - const tokens = await response.json(); - // TODO: validate response, use zod - aggregatedTokenList.push(...tokens.tokens.filter((token: any) => token.logoURI)); - } catch (e) { - console.warn(`Failed to fetch token list from ${tokenListUri}`); - } - } + await Promise.all( + uris.map(async tokenListUri => { + const tokens = await fetchTokens(tokenListUri); + aggregatedTokenList.push(...tokens); + }) + ); console.info(`Aggregated token list initialized with ${aggregatedTokenList.length} tokens`); From 2f6091f86c6ab4fb95dcc242dbc4f2c855a4dbcf Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 16:17:37 +0200 Subject: [PATCH 14/41] refactor: removed addition headers --- src/api.ts | 8 ++------ src/utils.ts | 22 +++------------------- test/e2e/api.test.ts | 2 -- 3 files changed, 5 insertions(+), 27 deletions(-) diff --git a/src/api.ts b/src/api.ts index 39c0a460..765c0eb9 100644 --- a/src/api.ts +++ b/src/api.ts @@ -122,18 +122,14 @@ router.get(`/:type(${TYPE_CONSTRAINTS})/:id`, async (req, res) => { const fallbackImage = await resolvers[fallback](address, network); const resizedImage = await resize(fallbackImage, w, h); - setHeader(res, 'SHORT_CACHE', { - [`x-stamp-${type}-fallback`]: fallback - }); + setHeader(res, 'SHORT_CACHE'); return res.send(resizedImage); } } // Resize and return image const resizedImage = await resize(baseImage, w, h); - const extraHeaders = {}; - if (resolver) extraHeaders[`x-stamp-${type}-resolver`] = resolver; - setHeader(res, 'LONG_CACHE', extraHeaders); + setHeader(res); res.send(resizedImage); if (disableCache) return; diff --git a/src/utils.ts b/src/utils.ts index 0a7b1137..37308264 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -138,30 +138,14 @@ export function getCacheKey({ return sha256(JSON.stringify(data)); } -export function setHeader( - res: Response, - cacheType: 'SHORT_CACHE' | 'LONG_CACHE' = 'LONG_CACHE', - extraHeaders: Record = {} -) { +export function setHeader(res: Response, cacheType: 'SHORT_CACHE' | 'LONG_CACHE' = 'LONG_CACHE') { const ttl = cacheType === 'SHORT_CACHE' ? constants.shortTtl : constants.ttl; - const headers = { + res.set({ 'Content-Type': 'image/webp', 'Cache-Control': `public, max-age=${ttl}`, Expires: new Date(Date.now() + ttl * 1e3).toUTCString() - }; - - if (typeof extraHeaders === 'object' && extraHeaders !== null) { - Object.entries(extraHeaders).forEach(([key, value]) => { - if (typeof value === 'string') { - headers[key] = value; - } else { - console.warn('Invalid header value passed to setHeader:', key, value); - } - }); - } - - res.set(headers); + }); } export const getBaseAssetIconUrl = (chainId: string) => { diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index 2286df33..3ac43fa1 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -27,8 +27,6 @@ describe('E2E api', () => { ); expect(response.status).toBe(200); expect(response.headers['content-type']).toBe('image/webp'); - expect(response.headers['x-stamp-token-resolver']).toBe('tokenlists'); - expect(response.headers['x-stamp-token-fallback']).toBe(undefined); }); it.todo('returns a 500 status on invalid query'); From d9057202df7c01377bd405e9f5bf05a981a1e0c1 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 16:20:26 +0200 Subject: [PATCH 15/41] refactor: reduce timeout in test to more reasonable value --- test/e2e/api.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index 3ac43fa1..5fc19e99 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -17,7 +17,7 @@ async function purge(): Promise { // for token resolver, which currently takes very long // not sure this is the best way to handle this -jest.setTimeout(120_000); +jest.setTimeout(30_000); describe('E2E api', () => { describe('GET type/TYPE/ID', () => { From 8e206a39b953f0e7127af215d7b8a8671238469a Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 16:35:57 +0200 Subject: [PATCH 16/41] feat: Update token list initialization for improved performance --- src/helpers/tokenlists.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 65e0b4b8..a10ddaa9 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -7,6 +7,10 @@ export type AggregatedTokenList = { decimals: number; }[]; +const TTL = 1000 * 60 * 60 * 24; +let aggregatedTokenList: AggregatedTokenList | undefined; +let lastUpdateTimestamp: number | undefined; + function normalizeTokenListUri(tokenListUri: string) { if (!tokenListUri.startsWith('http') && tokenListUri.endsWith('.eth')) { tokenListUri = `https://${tokenListUri}.limo`; @@ -41,20 +45,28 @@ async function fetchTokens(tokenListUri: string) { } export async function initAggregatedTokenList() { + if (aggregatedTokenList && lastUpdateTimestamp && lastUpdateTimestamp > Date.now() - TTL) { + console.info('Using in-memory aggregated token list'); + return aggregatedTokenList; + } + console.info('Initializing aggregated token list from tokenlists.org'); - const aggregatedTokenList: AggregatedTokenList = []; + const list: AggregatedTokenList = []; const uris = await fetchList(); await Promise.all( uris.map(async tokenListUri => { const tokens = await fetchTokens(tokenListUri); - aggregatedTokenList.push(...tokens); + list.push(...tokens); }) ); + aggregatedTokenList = list; + console.info(`Aggregated token list initialized with ${aggregatedTokenList.length} tokens`); + lastUpdateTimestamp = Date.now(); return aggregatedTokenList; } From 87cbbf5f7e4d5e777a73252cb0ff32ebae79fc61 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 17:42:47 +0200 Subject: [PATCH 17/41] feat: Replace thumbnail image URLs with larger versions in token list --- src/helpers/tokenlists.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index a10ddaa9..d93f50d8 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -44,6 +44,22 @@ async function fetchTokens(tokenListUri: string) { } } +const REPLACE_SIZE_REGEX: { pattern: RegExp; replacement: string }[] = [ + { + pattern: /assets.coingecko.com\/coins\/images\/\d+\/thumb/, + replacement: 'assets.coingecko.com/coins/images/$1/large' + } +]; + +function replaceSizePartsInImageUrls(list: AggregatedTokenList) { + return list.map(token => { + token.logoURI = REPLACE_SIZE_REGEX.reduce((acc, { pattern, replacement }) => { + return acc.replace(pattern, replacement); + }, token.logoURI); + return token; + }); +} + export async function initAggregatedTokenList() { if (aggregatedTokenList && lastUpdateTimestamp && lastUpdateTimestamp > Date.now() - TTL) { console.info('Using in-memory aggregated token list'); @@ -63,7 +79,9 @@ export async function initAggregatedTokenList() { }) ); - aggregatedTokenList = list; + const listWithReplacedSizes = replaceSizePartsInImageUrls(list); + + aggregatedTokenList = listWithReplacedSizes; console.info(`Aggregated token list initialized with ${aggregatedTokenList.length} tokens`); lastUpdateTimestamp = Date.now(); From 527895ff3c3861d94351e6f38b458d02487a8632 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 17:49:20 +0200 Subject: [PATCH 18/41] chore: updated comment --- test/e2e/api.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index 5fc19e99..08553ba5 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -15,8 +15,8 @@ async function purge(): Promise { transaction.exec(); } -// for token resolver, which currently takes very long -// not sure this is the best way to handle this +// for token resolver, which needs a moment on first run +// I think the tests need to return promises. jest.setTimeout(30_000); describe('E2E api', () => { From c7da141017278398a4b9f5012f17c92563f46652 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 7 Aug 2024 18:14:02 +0200 Subject: [PATCH 19/41] refactor: remove variable assignment --- src/helpers/tokenlists.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index d93f50d8..e5756ab2 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -79,9 +79,7 @@ export async function initAggregatedTokenList() { }) ); - const listWithReplacedSizes = replaceSizePartsInImageUrls(list); - - aggregatedTokenList = listWithReplacedSizes; + aggregatedTokenList = replaceSizePartsInImageUrls(list); console.info(`Aggregated token list initialized with ${aggregatedTokenList.length} tokens`); lastUpdateTimestamp = Date.now(); From 226a3a194e7d16d0d9aeabb3553b2d6bc7247eb7 Mon Sep 17 00:00:00 2001 From: mktcode Date: Fri, 9 Aug 2024 18:26:32 +0200 Subject: [PATCH 20/41] Update test/e2e/api.test.ts --- test/e2e/api.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index db77952d..583dd829 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -17,7 +17,7 @@ async function purge(): Promise { // for token resolver, which needs a moment on first run // I think the tests need to return promises. -jest.setTimeout(30_000); +jest.setTimeout(60_000); describe('E2E api', () => { describe('GET type/TYPE/ID', () => { From e1697d98262957c557849ef7b806afda93adb5be Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 14 Aug 2024 12:24:50 +0200 Subject: [PATCH 21/41] refactor(tokenlists): roughly validate list response --- src/helpers/tokenlists.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index e5756ab2..d5e1f9cb 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -36,7 +36,10 @@ async function fetchTokens(tokenListUri: string) { try { const response = await fetch(tokenListUri); const tokens = await response.json(); - // TODO: validate response, use zod + if (!tokens.tokens || !Array.isArray(tokens.tokens)) { + throw new Error('Invalid token list'); + } + return tokens.tokens.filter((token: any) => token.logoURI); } catch (e) { console.warn(`Failed to fetch token list from ${tokenListUri}`); From 8422fba9b81797c232d165ae8956312308e8811b Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 14 Aug 2024 12:26:07 +0200 Subject: [PATCH 22/41] refactor(tokenlists): move back to on-boot approach --- src/helpers/tokenlists.ts | 50 ++++++++++++++++++++++++++----------- src/index.ts | 9 ++++++- src/resolvers/tokenlists.ts | 16 +----------- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index d5e1f9cb..b8e8c121 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -1,11 +1,15 @@ -export type AggregatedTokenList = { +import { getAddress } from '@ethersproject/address'; + +type TokenlistToken = { chainId: number; address: string; symbol: string; name: string; logoURI: string; decimals: number; -}[]; +}; + +type AggregatedTokenList = TokenlistToken[]; const TTL = 1000 * 60 * 60 * 24; let aggregatedTokenList: AggregatedTokenList | undefined; @@ -21,7 +25,7 @@ function normalizeTokenListUri(tokenListUri: string) { const TOKENLISTS_URL = 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; -async function fetchList() { +async function fetchListUris() { const response = await fetch(TOKENLISTS_URL); const tokenLists = await response.json(); const uris = Object.keys(tokenLists); @@ -63,17 +67,10 @@ function replaceSizePartsInImageUrls(list: AggregatedTokenList) { }); } -export async function initAggregatedTokenList() { - if (aggregatedTokenList && lastUpdateTimestamp && lastUpdateTimestamp > Date.now() - TTL) { - console.info('Using in-memory aggregated token list'); - return aggregatedTokenList; - } - - console.info('Initializing aggregated token list from tokenlists.org'); - +async function updateAggregatedTokenList() { const list: AggregatedTokenList = []; - const uris = await fetchList(); + const uris = await fetchListUris(); await Promise.all( uris.map(async tokenListUri => { @@ -83,9 +80,32 @@ export async function initAggregatedTokenList() { ); aggregatedTokenList = replaceSizePartsInImageUrls(list); - - console.info(`Aggregated token list initialized with ${aggregatedTokenList.length} tokens`); lastUpdateTimestamp = Date.now(); +} + +export async function initTokenLists() { + await updateAggregatedTokenList(); + + setInterval(() => { + if (lastUpdateTimestamp && Date.now() - lastUpdateTimestamp > TTL) { + updateAggregatedTokenList(); + } + }, TTL); + + return true; +} + +export async function findImageUrl(address: string, chainId: string) { + const checksum = getAddress(address); + + if (!aggregatedTokenList) { + throw new Error('Tokenlists not initialized'); + } + + const token = aggregatedTokenList.find(token => { + return token.chainId === parseInt(chainId) && getAddress(token.address) === checksum; + }); + if (!token) throw new Error('Token not found'); - return aggregatedTokenList; + return token.logoURI; } diff --git a/src/index.ts b/src/index.ts index 3a0221b7..906cb77c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry'; import initMetrics from './helpers/metrics'; import api from './api'; import { name, version } from '../package.json'; +import { initTokenLists } from './helpers/tokenlists'; const app = express(); const PORT = process.env.PORT || 3008; @@ -31,4 +32,10 @@ app.use((_, res) => { res.status(400).json({ message: 'Not found' }); }); -app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); +async function boot() { + await initTokenLists(); +} + +boot().then(() => { + app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); +}); diff --git a/src/resolvers/tokenlists.ts b/src/resolvers/tokenlists.ts index ab133834..72b5fbcf 100644 --- a/src/resolvers/tokenlists.ts +++ b/src/resolvers/tokenlists.ts @@ -1,21 +1,7 @@ -import { getAddress } from '@ethersproject/address'; import { resize } from '../utils'; import { max } from '../constants.json'; import { fetchHttpImage } from './utils'; -import { initAggregatedTokenList } from '../helpers/tokenlists'; - -async function findImageUrl(address: string, chainId: string) { - const checksum = getAddress(address); - - const aggregatedTokenList = await initAggregatedTokenList(); - - const token = aggregatedTokenList.find(token => { - return token.chainId === parseInt(chainId) && getAddress(token.address) === checksum; - }); - if (!token) throw new Error('Token not found'); - - return token.logoURI; -} +import { findImageUrl } from '../helpers/tokenlists'; export default async function resolve(address: string, chainId: string) { try { From dc6aa6b4171ae37ea7eceee49be55174921f2319 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 14 Aug 2024 12:35:13 +0200 Subject: [PATCH 23/41] fix(tokenlists): regex to capture variabe part in url replacements --- src/helpers/tokenlists.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index b8e8c121..97f6e964 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -53,7 +53,7 @@ async function fetchTokens(tokenListUri: string) { const REPLACE_SIZE_REGEX: { pattern: RegExp; replacement: string }[] = [ { - pattern: /assets.coingecko.com\/coins\/images\/\d+\/thumb/, + pattern: /assets.coingecko.com\/coins\/images\/(\d+)\/thumb/, replacement: 'assets.coingecko.com/coins/images/$1/large' } ]; @@ -67,6 +67,19 @@ function replaceSizePartsInImageUrls(list: AggregatedTokenList) { }); } +console.log( + replaceSizePartsInImageUrls([ + { + address: '0x6b175474e89094c44da98b954eedeac495271d0f', + chainId: 1, + decimals: 18, + logoURI: 'https://assets.coingecko.com/coins/images/1645646/thumb/dai.png', + name: 'Dai Stablecoin', + symbol: 'DAI' + } + ]) +); + async function updateAggregatedTokenList() { const list: AggregatedTokenList = []; From 62180b7205171de4c6cd3ea127732af52efd6165 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 14 Aug 2024 12:36:26 +0200 Subject: [PATCH 24/41] refactor(tokenlists): upsi --- src/helpers/tokenlists.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 97f6e964..f685dbe8 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -67,19 +67,6 @@ function replaceSizePartsInImageUrls(list: AggregatedTokenList) { }); } -console.log( - replaceSizePartsInImageUrls([ - { - address: '0x6b175474e89094c44da98b954eedeac495271d0f', - chainId: 1, - decimals: 18, - logoURI: 'https://assets.coingecko.com/coins/images/1645646/thumb/dai.png', - name: 'Dai Stablecoin', - symbol: 'DAI' - } - ]) -); - async function updateAggregatedTokenList() { const list: AggregatedTokenList = []; From bd130be101fb24675e07437a18bd441a876f77cd Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 14 Aug 2024 12:42:15 +0200 Subject: [PATCH 25/41] refactor(tokenlists): map ipfs logoUris to http gateway --- src/helpers/tokenlists.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index f685dbe8..bbb156ba 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -22,6 +22,13 @@ function normalizeTokenListUri(tokenListUri: string) { return tokenListUri; } +function normalizeTokenLogoUri(logoUri: string) { + if (logoUri.startsWith('ipfs://')) { + logoUri = `https://ipfs.io/ipfs/${logoUri.slice(7)}`; + } + return logoUri; +} + const TOKENLISTS_URL = 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; @@ -44,7 +51,7 @@ async function fetchTokens(tokenListUri: string) { throw new Error('Invalid token list'); } - return tokens.tokens.filter((token: any) => token.logoURI); + return tokens.tokens.filter((token: any) => normalizeTokenLogoUri(token.logoURI)); } catch (e) { console.warn(`Failed to fetch token list from ${tokenListUri}`); return []; From 743fefdf111a482447768f5b492136f2fbcaffc4 Mon Sep 17 00:00:00 2001 From: mktcode Date: Wed, 14 Aug 2024 13:55:13 +0200 Subject: [PATCH 26/41] refactor: add unit test for replaceSizePartsInImageUrls function --- package.json | 5 ++-- src/helpers/tokenlists.ts | 4 ++-- test/unit/tokenlists.test.ts | 45 ++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 test/unit/tokenlists.test.ts diff --git a/package.json b/package.json index 64d09c39..1ff1bc02 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "start:test": "dotenv -e test/.env.test yarn dev", "test": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test yarn jest'", "test:integration": "dotenv -e test/.env.test yarn jest --runInBand --collectCoverage=false test/integration", - "test:e2e": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'" + "test:e2e": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'", + "test:unit": "yarn jest --runInBand --collectCoverage=false test/unit" }, "dependencies": { "@adraffy/ens-normalize": "^1.10.0", @@ -57,4 +58,4 @@ "start-server-and-test": "^2.0.3", "ts-jest": "^28.0.4" } -} +} \ No newline at end of file diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index bbb156ba..1110261b 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -60,12 +60,12 @@ async function fetchTokens(tokenListUri: string) { const REPLACE_SIZE_REGEX: { pattern: RegExp; replacement: string }[] = [ { - pattern: /assets.coingecko.com\/coins\/images\/(\d+)\/thumb/, + pattern: /assets.coingecko.com\/coins\/images\/(\d+)\/(thumb|small)/, replacement: 'assets.coingecko.com/coins/images/$1/large' } ]; -function replaceSizePartsInImageUrls(list: AggregatedTokenList) { +export function replaceSizePartsInImageUrls(list: AggregatedTokenList) { return list.map(token => { token.logoURI = REPLACE_SIZE_REGEX.reduce((acc, { pattern, replacement }) => { return acc.replace(pattern, replacement); diff --git a/test/unit/tokenlists.test.ts b/test/unit/tokenlists.test.ts new file mode 100644 index 00000000..5bc2c590 --- /dev/null +++ b/test/unit/tokenlists.test.ts @@ -0,0 +1,45 @@ +import { replaceSizePartsInImageUrls } from '../../src/helpers/tokenlists'; + +test('replaceSizePartsInImageUrls should replace image size parts in URLs', () => { + const tokenList = [ + { + chainId: 1, + address: '0x1234567890abcdef', + symbol: 'ABC', + name: 'Token ABC', + logoURI: 'https://assets.coingecko.com/coins/images/123/thumb', + decimals: 18 + }, + { + chainId: 1, + address: '0xabcdef1234567890', + symbol: 'DEF', + name: 'Token DEF', + logoURI: 'https://assets.coingecko.com/coins/images/456/small', + decimals: 18 + } + ]; + + const expectedTokenList = [ + { + chainId: 1, + address: '0x1234567890abcdef', + symbol: 'ABC', + name: 'Token ABC', + logoURI: 'https://assets.coingecko.com/coins/images/123/large', + decimals: 18 + }, + { + chainId: 1, + address: '0xabcdef1234567890', + symbol: 'DEF', + name: 'Token DEF', + logoURI: 'https://assets.coingecko.com/coins/images/456/large', + decimals: 18 + } + ]; + + const result = replaceSizePartsInImageUrls(tokenList); + + expect(result).toEqual(expectedTokenList); +}); From 630644f102a029fc5fb4763012a429b17c4cf39d Mon Sep 17 00:00:00 2001 From: mktcode Date: Thu, 15 Aug 2024 13:19:45 +0200 Subject: [PATCH 27/41] refactor(tokenlists): update in resolver --- src/helpers/tokenlists.ts | 36 +++++++++++++++--------------------- src/index.ts | 9 +-------- src/resolvers/tokenlists.ts | 5 +++-- 3 files changed, 19 insertions(+), 31 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 1110261b..d015f918 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -12,9 +12,13 @@ type TokenlistToken = { type AggregatedTokenList = TokenlistToken[]; const TTL = 1000 * 60 * 60 * 24; -let aggregatedTokenList: AggregatedTokenList | undefined; +let aggregatedTokenList: AggregatedTokenList = []; let lastUpdateTimestamp: number | undefined; +function isExpired() { + return !lastUpdateTimestamp || Date.now() - lastUpdateTimestamp > TTL; +} + function normalizeTokenListUri(tokenListUri: string) { if (!tokenListUri.startsWith('http') && tokenListUri.endsWith('.eth')) { tokenListUri = `https://${tokenListUri}.limo`; @@ -42,7 +46,6 @@ async function fetchListUris() { async function fetchTokens(tokenListUri: string) { tokenListUri = normalizeTokenListUri(tokenListUri); - console.info(`Fetching list from ${tokenListUri}`); try { const response = await fetch(tokenListUri); @@ -74,7 +77,11 @@ export function replaceSizePartsInImageUrls(list: AggregatedTokenList) { }); } -async function updateAggregatedTokenList() { +export async function updateExpiredAggregatedTokenList() { + if (!isExpired()) { + return; + } + const list: AggregatedTokenList = []; const uris = await fetchListUris(); @@ -90,29 +97,16 @@ async function updateAggregatedTokenList() { lastUpdateTimestamp = Date.now(); } -export async function initTokenLists() { - await updateAggregatedTokenList(); - - setInterval(() => { - if (lastUpdateTimestamp && Date.now() - lastUpdateTimestamp > TTL) { - updateAggregatedTokenList(); - } - }, TTL); - - return true; -} - -export async function findImageUrl(address: string, chainId: string) { +export function findImageUrl(address: string, chainId: string) { const checksum = getAddress(address); - if (!aggregatedTokenList) { - throw new Error('Tokenlists not initialized'); - } - const token = aggregatedTokenList.find(token => { return token.chainId === parseInt(chainId) && getAddress(token.address) === checksum; }); - if (!token) throw new Error('Token not found'); + + if (!token) { + throw new Error('Token not found'); + } return token.logoURI; } diff --git a/src/index.ts b/src/index.ts index 906cb77c..3a0221b7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,6 @@ import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry'; import initMetrics from './helpers/metrics'; import api from './api'; import { name, version } from '../package.json'; -import { initTokenLists } from './helpers/tokenlists'; const app = express(); const PORT = process.env.PORT || 3008; @@ -32,10 +31,4 @@ app.use((_, res) => { res.status(400).json({ message: 'Not found' }); }); -async function boot() { - await initTokenLists(); -} - -boot().then(() => { - app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); -}); +app.listen(PORT, () => console.log(`Listening at http://localhost:${PORT}`)); diff --git a/src/resolvers/tokenlists.ts b/src/resolvers/tokenlists.ts index 72b5fbcf..d4b13e2a 100644 --- a/src/resolvers/tokenlists.ts +++ b/src/resolvers/tokenlists.ts @@ -1,11 +1,12 @@ import { resize } from '../utils'; import { max } from '../constants.json'; import { fetchHttpImage } from './utils'; -import { findImageUrl } from '../helpers/tokenlists'; +import { findImageUrl, updateExpiredAggregatedTokenList } from '../helpers/tokenlists'; export default async function resolve(address: string, chainId: string) { try { - const url = await findImageUrl(address, chainId); + await updateExpiredAggregatedTokenList(); + const url = findImageUrl(address, chainId); const image = await fetchHttpImage(url); return await resize(image, max, max); From ba51a78d2703a0511f3120abe174124bc7dbb0d1 Mon Sep 17 00:00:00 2001 From: mktcode Date: Thu, 15 Aug 2024 13:29:29 +0200 Subject: [PATCH 28/41] fix(tokenlists): npe caused by normalizeTokenLogoUri --- src/helpers/tokenlists.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index d015f918..76679d45 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -54,7 +54,14 @@ async function fetchTokens(tokenListUri: string) { throw new Error('Invalid token list'); } - return tokens.tokens.filter((token: any) => normalizeTokenLogoUri(token.logoURI)); + const tokensWithLogoUri = tokens.tokens.filter((token: any) => token.logoURI); + + return tokensWithLogoUri.map((token: any) => { + return { + ...token, + logoURI: normalizeTokenLogoUri(token.logoURI) + }; + }); } catch (e) { console.warn(`Failed to fetch token list from ${tokenListUri}`); return []; From 6fd921943526fa11d836a967a3f10371ae03f35a Mon Sep 17 00:00:00 2001 From: mktcode Date: Thu, 15 Aug 2024 13:52:02 +0200 Subject: [PATCH 29/41] refactor(tokenlists): set reasonable timeout for requests --- src/helpers/tokenlists.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 76679d45..58bfa9f6 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -11,6 +11,9 @@ type TokenlistToken = { type AggregatedTokenList = TokenlistToken[]; +const TOKENLISTS_URL = + 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; +const REQUEST_TIMEOUT = 2500; const TTL = 1000 * 60 * 60 * 24; let aggregatedTokenList: AggregatedTokenList = []; let lastUpdateTimestamp: number | undefined; @@ -33,11 +36,10 @@ function normalizeTokenLogoUri(logoUri: string) { return logoUri; } -const TOKENLISTS_URL = - 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; - async function fetchListUris() { - const response = await fetch(TOKENLISTS_URL); + const response = await fetch(TOKENLISTS_URL, { + signal: AbortSignal.timeout(REQUEST_TIMEOUT) + }); const tokenLists = await response.json(); const uris = Object.keys(tokenLists); @@ -48,7 +50,9 @@ async function fetchTokens(tokenListUri: string) { tokenListUri = normalizeTokenListUri(tokenListUri); try { - const response = await fetch(tokenListUri); + const response = await fetch(tokenListUri, { + signal: AbortSignal.timeout(REQUEST_TIMEOUT) + }); const tokens = await response.json(); if (!tokens.tokens || !Array.isArray(tokens.tokens)) { throw new Error('Invalid token list'); From 25f3bf2d3922b66cbd301878e05e5c9683f35b6b Mon Sep 17 00:00:00 2001 From: mktcode Date: Fri, 23 Aug 2024 20:56:33 +0200 Subject: [PATCH 30/41] chore(deps): update semver to version 7.6.3 --- yarn.lock | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index a5756d50..2c9724d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6290,7 +6290,7 @@ scrypt-js@3.0.1: resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-3.0.1.tgz#d314a57c2aef69d1ad98a138a21fe9eafa9ee312" integrity sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA== -semver@7.x, semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.5.3: +semver@7.x, semver@^7.3.2, semver@^7.3.5, semver@^7.5.3: version "7.6.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== @@ -6307,6 +6307,11 @@ semver@^6.0.0, semver@^6.1.2, semver@^6.3.0, semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== +semver@^7.3.7: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + semver@~7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" From 0b070a40e82449561940c9966db663b386a83c50 Mon Sep 17 00:00:00 2001 From: mktcode Date: Fri, 23 Aug 2024 20:56:55 +0200 Subject: [PATCH 31/41] chore: add e2e test for tokenlists --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index ca344d01..d14b0176 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "test": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test yarn jest'", "test:integration": "dotenv -e test/.env.test yarn jest --runInBand --collectCoverage=false test/integration", "test:e2e": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'", + "test:e2e:tokenlists": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/tokenlists.test.ts'", "test:unit": "yarn jest --runInBand --collectCoverage=false test/unit" }, "dependencies": { @@ -58,4 +59,4 @@ "start-server-and-test": "^2.0.3", "ts-jest": "^28.0.4" } -} \ No newline at end of file +} From 8521425aaf3b4c780c42f8afa8973cf02af8c1d7 Mon Sep 17 00:00:00 2001 From: mktcode Date: Fri, 23 Aug 2024 20:57:24 +0200 Subject: [PATCH 32/41] refactor: improve tokenlists resolver and add tests --- test/e2e/api.test.ts | 13 ------- test/e2e/tokenlists.test.ts | 72 ++++++++++++++++++++++++++++++++++++ test/unit/tokenlists.test.ts | 56 ++++++++-------------------- 3 files changed, 88 insertions(+), 53 deletions(-) create mode 100644 test/e2e/tokenlists.test.ts diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts index 583dd829..e24ad36e 100644 --- a/test/e2e/api.test.ts +++ b/test/e2e/api.test.ts @@ -3,7 +3,6 @@ import redis from '../../src/helpers/redis'; import { KEY_PREFIX } from '../../src/addressResolvers/cache'; const HOST = `http://localhost:${process.env.PORT || 3003}`; -const cUSDC_TOKEN_ADDRESS_ON_MAIN = '0x39AA39c021dfbaE8faC545936693aC917d5E7563'; async function purge(): Promise { if (!redis) return; @@ -15,20 +14,8 @@ async function purge(): Promise { transaction.exec(); } -// for token resolver, which needs a moment on first run -// I think the tests need to return promises. -jest.setTimeout(60_000); - describe('E2E api', () => { describe('GET type/TYPE/ID', () => { - it('returns an image for tokenlists resolver', async () => { - const response = await axios.get( - `${HOST}/token/${cUSDC_TOKEN_ADDRESS_ON_MAIN}?resolver=tokenlists` - ); - expect(response.status).toBe(200); - expect(response.headers['content-type']).toBe('image/webp'); - }); - it.todo('returns a 500 status on invalid query'); describe('when the image is not cached', () => { diff --git a/test/e2e/tokenlists.test.ts b/test/e2e/tokenlists.test.ts new file mode 100644 index 00000000..3562c9d8 --- /dev/null +++ b/test/e2e/tokenlists.test.ts @@ -0,0 +1,72 @@ +import sharp from 'sharp'; +import axios from 'axios'; +import crypto from 'crypto'; + +const HOST = `http://localhost:${process.env.PORT || 3003}`; +const cUSDC_TOKEN_ADDRESS_ON_MAIN = '0x39AA39c021dfbaE8faC545936693aC917d5E7563'; +const ERC3770_ADDRESS = 'oeth:0xe0BB0D3DE8c10976511e5030cA403dBf4c25165B'; +const EIP155_ADDRESS = 'eip155:1:0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e'; + +function getImageFingerprint(input: string) { + return crypto + .createHash('sha256') + .update(input) + .digest('hex'); +} + +function getImageResponse(identifier: string) { + return axios.get(`${HOST}/token/${identifier}?resolver=tokenlists`, { + responseType: 'arraybuffer', + headers: { + 'Cache-Control': 'no-cache', + Pragma: 'no-cache', + Expires: '0' + } + }); +} + +// tokenlists resolver needs a moment on first run +// there's probably a better way to handle this +jest.setTimeout(60_000); + +describe('tokenlist resolver', () => { + it('returns an image for standard address', async () => { + const response = await getImageResponse(cUSDC_TOKEN_ADDRESS_ON_MAIN); + + const image = sharp(response.data); + await image.toFile(`./test/e2e/cusdc-${new Date().getTime()}.webp`); + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe('image/webp'); + }); + + it('returns correct image for ERC3770 address', async () => { + const response = await getImageResponse(ERC3770_ADDRESS); + + const image = sharp(response.data); + await image.toFile(`./test/e2e/erc3770-${new Date().getTime()}.webp`); + const imageBuffer = await image.raw().toBuffer(); + + const fingerprint = getImageFingerprint(imageBuffer.toString('hex')); + const expectedFingerprint = 'f00ea652e3b6c2c7e72db8b1c0f35884094c7c598fe60b3c28956f328dbaded8'; + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe('image/webp'); + expect(fingerprint).toBe(expectedFingerprint); + }); + + it('returns an image for EIP155 address', async () => { + const response = await getImageResponse(EIP155_ADDRESS); + + const image = sharp(response.data); + await image.toFile(`./test/e2e/eip155-${new Date().getTime()}.webp`); + const imageBuffer = await image.raw().toBuffer(); + + const fingerprint = getImageFingerprint(imageBuffer.toString('hex')); + const expectedFingerprint = '8118786398e4756b2b7e8e224ec2bb5cbe3b26ee93ceff3b19d40f81c8ce45a2'; + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe('image/webp'); + expect(fingerprint).toBe(expectedFingerprint); + }); +}); diff --git a/test/unit/tokenlists.test.ts b/test/unit/tokenlists.test.ts index 5bc2c590..57c74e9f 100644 --- a/test/unit/tokenlists.test.ts +++ b/test/unit/tokenlists.test.ts @@ -1,45 +1,21 @@ -import { replaceSizePartsInImageUrls } from '../../src/helpers/tokenlists'; +import { replaceURIPatterns } from '../../src/helpers/tokenlists'; -test('replaceSizePartsInImageUrls should replace image size parts in URLs', () => { - const tokenList = [ - { - chainId: 1, - address: '0x1234567890abcdef', - symbol: 'ABC', - name: 'Token ABC', - logoURI: 'https://assets.coingecko.com/coins/images/123/thumb', - decimals: 18 - }, - { - chainId: 1, - address: '0xabcdef1234567890', - symbol: 'DEF', - name: 'Token DEF', - logoURI: 'https://assets.coingecko.com/coins/images/456/small', - decimals: 18 - } - ]; +jest.setTimeout(60_000); - const expectedTokenList = [ - { - chainId: 1, - address: '0x1234567890abcdef', - symbol: 'ABC', - name: 'Token ABC', - logoURI: 'https://assets.coingecko.com/coins/images/123/large', - decimals: 18 - }, - { - chainId: 1, - address: '0xabcdef1234567890', - symbol: 'DEF', - name: 'Token DEF', - logoURI: 'https://assets.coingecko.com/coins/images/456/large', - decimals: 18 - } - ]; +describe('tokenlists helper', () => { + it('replaceURIPatterns should replace image size parts in URLs', () => { + const uris = [ + 'https://assets.coingecko.com/coins/images/123/thumb', + 'https://assets.coingecko.com/coins/images/456/small' + ]; - const result = replaceSizePartsInImageUrls(tokenList); + const expectedUris = [ + 'https://assets.coingecko.com/coins/images/123/large', + 'https://assets.coingecko.com/coins/images/456/large' + ]; - expect(result).toEqual(expectedTokenList); + uris.forEach((uri, i) => { + expect(replaceURIPatterns(uri)).toBe(expectedUris[i]); + }); + }); }); From c7d2949e010cc6400ed6e9eceaca2181d841772a Mon Sep 17 00:00:00 2001 From: mktcode Date: Fri, 23 Aug 2024 20:57:36 +0200 Subject: [PATCH 33/41] refactor: improve tokenlists resolver and add tests --- src/helpers/tokenlists.ts | 148 +++++++++++++++++++++++--------------- 1 file changed, 89 insertions(+), 59 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 58bfa9f6..2e6a070c 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -1,5 +1,12 @@ import { getAddress } from '@ethersproject/address'; +const TOKENLISTS_URL = + 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; +const REQUEST_TIMEOUT = 5000; +const TTL = 1000 * 60 * 60 * 24; +let aggregatedTokenList: AggregatedTokenList = []; +let lastUpdateTimestamp: number | undefined; + type TokenlistToken = { chainId: number; address: string; @@ -9,83 +16,86 @@ type TokenlistToken = { decimals: number; }; -type AggregatedTokenList = TokenlistToken[]; +type AggregatedTokenListToken = Omit & { + logoURIs: string[]; +}; -const TOKENLISTS_URL = - 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; -const REQUEST_TIMEOUT = 2500; -const TTL = 1000 * 60 * 60 * 24; -let aggregatedTokenList: AggregatedTokenList = []; -let lastUpdateTimestamp: number | undefined; +type AggregatedTokenList = AggregatedTokenListToken[]; + +function isTokenlistToken(token: unknown): token is TokenlistToken { + if (typeof token !== 'object' || token === null) { + return false; + } + + const { chainId, address, symbol, name, logoURI, decimals } = token as TokenlistToken; + + return ( + typeof chainId === 'number' && + typeof address === 'string' && + typeof symbol === 'string' && + typeof name === 'string' && + typeof logoURI === 'string' && + typeof decimals === 'number' + ); +} function isExpired() { return !lastUpdateTimestamp || Date.now() - lastUpdateTimestamp > TTL; } -function normalizeTokenListUri(tokenListUri: string) { - if (!tokenListUri.startsWith('http') && tokenListUri.endsWith('.eth')) { - tokenListUri = `https://${tokenListUri}.limo`; +function normalizeUri(uri: string) { + if (!uri.startsWith('http') && uri.endsWith('.eth')) { + uri = `https://${uri}.limo`; } - return tokenListUri; + if (uri.startsWith('ipfs://')) { + uri = `https://ipfs.io/ipfs/${uri.slice(7)}`; + } + return uri; } -function normalizeTokenLogoUri(logoUri: string) { - if (logoUri.startsWith('ipfs://')) { - logoUri = `https://ipfs.io/ipfs/${logoUri.slice(7)}`; - } - return logoUri; +async function fetchUri(uri: string) { + return await fetch(normalizeUri(uri), { signal: AbortSignal.timeout(REQUEST_TIMEOUT) }); } async function fetchListUris() { - const response = await fetch(TOKENLISTS_URL, { - signal: AbortSignal.timeout(REQUEST_TIMEOUT) - }); - const tokenLists = await response.json(); - const uris = Object.keys(tokenLists); + try { + const response = await fetchUri(TOKENLISTS_URL); + const tokenLists = await response.json(); + const uris = Object.keys(tokenLists); - return uris; -} + uris.sort(); -async function fetchTokens(tokenListUri: string) { - tokenListUri = normalizeTokenListUri(tokenListUri); + return uris; + } catch (e) { + return []; + } +} +export async function fetchTokens(listUri: string) { try { - const response = await fetch(tokenListUri, { - signal: AbortSignal.timeout(REQUEST_TIMEOUT) - }); - const tokens = await response.json(); - if (!tokens.tokens || !Array.isArray(tokens.tokens)) { + const response = await fetchUri(listUri); + const { tokens } = await response.json(); + if (!tokens || !Array.isArray(tokens)) { throw new Error('Invalid token list'); } - - const tokensWithLogoUri = tokens.tokens.filter((token: any) => token.logoURI); - - return tokensWithLogoUri.map((token: any) => { - return { - ...token, - logoURI: normalizeTokenLogoUri(token.logoURI) - }; - }); + return tokens.filter(isTokenlistToken); } catch (e) { - console.warn(`Failed to fetch token list from ${tokenListUri}`); return []; } } -const REPLACE_SIZE_REGEX: { pattern: RegExp; replacement: string }[] = [ +const REPLACE_SIZE_REGEXES: { pattern: RegExp; replacement: string }[] = [ { pattern: /assets.coingecko.com\/coins\/images\/(\d+)\/(thumb|small)/, replacement: 'assets.coingecko.com/coins/images/$1/large' } ]; -export function replaceSizePartsInImageUrls(list: AggregatedTokenList) { - return list.map(token => { - token.logoURI = REPLACE_SIZE_REGEX.reduce((acc, { pattern, replacement }) => { - return acc.replace(pattern, replacement); - }, token.logoURI); - return token; - }); +export function replaceURIPatterns(uri: string) { + for (const { pattern, replacement } of REPLACE_SIZE_REGEXES) { + uri = uri.replace(pattern, replacement); + } + return uri; } export async function updateExpiredAggregatedTokenList() { @@ -93,18 +103,38 @@ export async function updateExpiredAggregatedTokenList() { return; } - const list: AggregatedTokenList = []; - - const uris = await fetchListUris(); + const updatedAggregatedList: AggregatedTokenList = []; + const tokenMap = new Map(); + + const tokenListUris = await fetchListUris(); + const tokenLists = await Promise.all(tokenListUris.map(fetchTokens)); + + for (const tokens of tokenLists) { + for (const token of tokens) { + const logoURI = normalizeUri(replaceURIPatterns(token.logoURI)); + const tokenKey = `${token.chainId}-${getAddress(token.address)}`; + + const existingToken = tokenMap.get(tokenKey); + if (existingToken) { + existingToken.logoURIs.push(logoURI); + } else { + const newToken: AggregatedTokenListToken = { + chainId: token.chainId, + address: token.address, + symbol: token.symbol, + name: token.name, + decimals: token.decimals, + logoURIs: [logoURI] + }; + tokenMap.set(tokenKey, newToken); + updatedAggregatedList.push(newToken); + } + } + } - await Promise.all( - uris.map(async tokenListUri => { - const tokens = await fetchTokens(tokenListUri); - list.push(...tokens); - }) - ); + updatedAggregatedList.forEach(token => token.logoURIs.sort()); - aggregatedTokenList = replaceSizePartsInImageUrls(list); + aggregatedTokenList = updatedAggregatedList; lastUpdateTimestamp = Date.now(); } @@ -119,5 +149,5 @@ export function findImageUrl(address: string, chainId: string) { throw new Error('Token not found'); } - return token.logoURI; + return token.logoURIs[0]; } From 9d6ceae2807d1e1a1fe30f220bfaa236512552f2 Mon Sep 17 00:00:00 2001 From: mktcode Date: Fri, 23 Aug 2024 21:01:45 +0200 Subject: [PATCH 34/41] refactor: remove test code, update expected fingerprint --- test/e2e/tokenlists.test.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/e2e/tokenlists.test.ts b/test/e2e/tokenlists.test.ts index 3562c9d8..a2b91ab5 100644 --- a/test/e2e/tokenlists.test.ts +++ b/test/e2e/tokenlists.test.ts @@ -33,9 +33,6 @@ describe('tokenlist resolver', () => { it('returns an image for standard address', async () => { const response = await getImageResponse(cUSDC_TOKEN_ADDRESS_ON_MAIN); - const image = sharp(response.data); - await image.toFile(`./test/e2e/cusdc-${new Date().getTime()}.webp`); - expect(response.status).toBe(200); expect(response.headers['content-type']).toBe('image/webp'); }); @@ -44,11 +41,10 @@ describe('tokenlist resolver', () => { const response = await getImageResponse(ERC3770_ADDRESS); const image = sharp(response.data); - await image.toFile(`./test/e2e/erc3770-${new Date().getTime()}.webp`); const imageBuffer = await image.raw().toBuffer(); const fingerprint = getImageFingerprint(imageBuffer.toString('hex')); - const expectedFingerprint = 'f00ea652e3b6c2c7e72db8b1c0f35884094c7c598fe60b3c28956f328dbaded8'; + const expectedFingerprint = 'ac601f072065d4d03e6ef906c1dc3074d7ad52b9c715d0db6941ec89bf2073a1'; expect(response.status).toBe(200); expect(response.headers['content-type']).toBe('image/webp'); @@ -59,7 +55,6 @@ describe('tokenlist resolver', () => { const response = await getImageResponse(EIP155_ADDRESS); const image = sharp(response.data); - await image.toFile(`./test/e2e/eip155-${new Date().getTime()}.webp`); const imageBuffer = await image.raw().toBuffer(); const fingerprint = getImageFingerprint(imageBuffer.toString('hex')); From 9c1c84c77bb10ba0f0d5c8dd1f32c24a6e6ee50a Mon Sep 17 00:00:00 2001 From: mktcode Date: Sat, 24 Aug 2024 14:38:10 +0200 Subject: [PATCH 35/41] refactor: remove unnecessary code in tokenlists.ts This sorting isn't relevant. Response times are. Sorting must be the last step. --- src/helpers/tokenlists.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 2e6a070c..8011437c 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -63,8 +63,6 @@ async function fetchListUris() { const tokenLists = await response.json(); const uris = Object.keys(tokenLists); - uris.sort(); - return uris; } catch (e) { return []; From 2e0d09f5cf941ea2b4e885dbb193c7a4b9694592 Mon Sep 17 00:00:00 2001 From: mktcode Date: Sat, 24 Aug 2024 15:21:02 +0200 Subject: [PATCH 36/41] refactor(tokenlists): sort logoURIs by size keywords --- src/helpers/tokenlists.ts | 45 +++++++++++++++++++++++++++++++++++- test/unit/tokenlists.test.ts | 36 +++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 8011437c..cdb9e9f3 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -96,6 +96,49 @@ export function replaceURIPatterns(uri: string) { return uri; } +const sizeKeywords = [ + 'xxl', + 'xl', + 'large', + 'lg', + 'big', + 'medium', + 'md', + 'small', + 'sm', + 'thumb', + 'icon', + 'xs', + 'xxs' +]; + +/** + * Sorts URIs by the size keyword in the URI. The order in the array above is the order of the sort. + */ +export function sortByKeywordMatch(a: string, b: string) { + try { + const aPath = new URL(a).pathname; + const bPath = new URL(b).pathname; + + const keywordRegex = new RegExp(`\\b(${sizeKeywords.join('|')})\\b`); + + const aMatch = aPath.match(keywordRegex); + const bMatch = bPath.match(keywordRegex); + + if (aMatch && bMatch) { + return sizeKeywords.indexOf(aMatch[1]) - sizeKeywords.indexOf(bMatch[1]); + } else if (aMatch) { + return -1; + } else if (bMatch) { + return 1; + } else { + return a.localeCompare(b); + } + } catch (e) { + return 0; + } +} + export async function updateExpiredAggregatedTokenList() { if (!isExpired()) { return; @@ -130,7 +173,7 @@ export async function updateExpiredAggregatedTokenList() { } } - updatedAggregatedList.forEach(token => token.logoURIs.sort()); + updatedAggregatedList.forEach(token => token.logoURIs.sort(sortByKeywordMatch)); aggregatedTokenList = updatedAggregatedList; lastUpdateTimestamp = Date.now(); diff --git a/test/unit/tokenlists.test.ts b/test/unit/tokenlists.test.ts index 57c74e9f..2bae38da 100644 --- a/test/unit/tokenlists.test.ts +++ b/test/unit/tokenlists.test.ts @@ -1,9 +1,9 @@ -import { replaceURIPatterns } from '../../src/helpers/tokenlists'; +import { replaceURIPatterns, sortByKeywordMatch } from '../../src/helpers/tokenlists'; jest.setTimeout(60_000); describe('tokenlists helper', () => { - it('replaceURIPatterns should replace image size parts in URLs', () => { + it('replaceURIPatterns should replace known image size related parts in URLs', () => { const uris = [ 'https://assets.coingecko.com/coins/images/123/thumb', 'https://assets.coingecko.com/coins/images/456/small' @@ -18,4 +18,36 @@ describe('tokenlists helper', () => { expect(replaceURIPatterns(uri)).toBe(expectedUris[i]); }); }); + + it('sortByKeywordMatch should sort URIs by size keywords', () => { + const uris = [ + 'https://assets.coingecko.com/coins/images/123/thumb', + 'https://assets.coingecko.com/coins/images/2021/xxs', + 'https://assets.coingecko.com/coins/images/456-small', + 'https://assets.coingecko.com/coins/images/789/medium', + 'https://assets.coingecko.com/coins/images/1011/large', + 'https://assets.xl.coingecko.com/coins/images/1213', + 'https://assets.coingecko.com/coins/images/1415/xxl', + 'https://assets.coingecko.com/coins/images/1617/icon', + 'https://assets.coingecko.com/coins/images/2021/lg/logo.png', + 'https://assets.coingecko.com/coins/images/2021/md-logo.png' + ]; + + const expectedUris = [ + 'https://assets.coingecko.com/coins/images/1415/xxl', + 'https://assets.coingecko.com/coins/images/1011/large', + 'https://assets.coingecko.com/coins/images/2021/lg/logo.png', + 'https://assets.coingecko.com/coins/images/789/medium', + 'https://assets.coingecko.com/coins/images/2021/md-logo.png', + 'https://assets.coingecko.com/coins/images/456-small', + 'https://assets.coingecko.com/coins/images/123/thumb', + 'https://assets.coingecko.com/coins/images/1617/icon', + 'https://assets.coingecko.com/coins/images/2021/xxs', + + // no keyword, should be at the end (domain part should be ignored) + 'https://assets.xl.coingecko.com/coins/images/1213' + ]; + + expect(uris.sort(sortByKeywordMatch)).toEqual(expectedUris); + }); }); From edfedb810a9db1e4df4113aa4b2e24930ed9a402 Mon Sep 17 00:00:00 2001 From: mktcode Date: Sat, 24 Aug 2024 15:41:00 +0200 Subject: [PATCH 37/41] refactor(tokenlists): use Map instead of Array --- src/helpers/tokenlists.ts | 43 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index cdb9e9f3..1479f375 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -1,12 +1,5 @@ import { getAddress } from '@ethersproject/address'; -const TOKENLISTS_URL = - 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; -const REQUEST_TIMEOUT = 5000; -const TTL = 1000 * 60 * 60 * 24; -let aggregatedTokenList: AggregatedTokenList = []; -let lastUpdateTimestamp: number | undefined; - type TokenlistToken = { chainId: number; address: string; @@ -20,7 +13,14 @@ type AggregatedTokenListToken = Omit & { logoURIs: string[]; }; -type AggregatedTokenList = AggregatedTokenListToken[]; +type AggregatedTokenList = Map; + +const TOKENLISTS_URL = + 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; +const REQUEST_TIMEOUT = 5000; +const TTL = 1000 * 60 * 60 * 24; +let aggregatedTokenList: AggregatedTokenList = new Map(); +let lastUpdateTimestamp: number | undefined; function isTokenlistToken(token: unknown): token is TokenlistToken { if (typeof token !== 'object' || token === null) { @@ -139,13 +139,16 @@ export function sortByKeywordMatch(a: string, b: string) { } } +function getTokenKey(address: string, chainId: string) { + return `${chainId}-${getAddress(address)}`; +} + export async function updateExpiredAggregatedTokenList() { if (!isExpired()) { return; } - const updatedAggregatedList: AggregatedTokenList = []; - const tokenMap = new Map(); + const newTokenMap = new Map(); const tokenListUris = await fetchListUris(); const tokenLists = await Promise.all(tokenListUris.map(fetchTokens)); @@ -153,9 +156,9 @@ export async function updateExpiredAggregatedTokenList() { for (const tokens of tokenLists) { for (const token of tokens) { const logoURI = normalizeUri(replaceURIPatterns(token.logoURI)); - const tokenKey = `${token.chainId}-${getAddress(token.address)}`; + const tokenKey = getTokenKey(token.address, token.chainId.toString()); - const existingToken = tokenMap.get(tokenKey); + const existingToken = newTokenMap.get(tokenKey); if (existingToken) { existingToken.logoURIs.push(logoURI); } else { @@ -167,27 +170,23 @@ export async function updateExpiredAggregatedTokenList() { decimals: token.decimals, logoURIs: [logoURI] }; - tokenMap.set(tokenKey, newToken); - updatedAggregatedList.push(newToken); + newTokenMap.set(tokenKey, newToken); } } } - updatedAggregatedList.forEach(token => token.logoURIs.sort(sortByKeywordMatch)); + newTokenMap.forEach(token => token.logoURIs.sort(sortByKeywordMatch)); - aggregatedTokenList = updatedAggregatedList; + aggregatedTokenList = newTokenMap; lastUpdateTimestamp = Date.now(); } export function findImageUrl(address: string, chainId: string) { - const checksum = getAddress(address); - - const token = aggregatedTokenList.find(token => { - return token.chainId === parseInt(chainId) && getAddress(token.address) === checksum; - }); + const tokenKey = getTokenKey(address, chainId); + const token = aggregatedTokenList.get(tokenKey); if (!token) { - throw new Error('Token not found'); + throw new Error('Token not found in aggregated tokenlist'); } return token.logoURIs[0]; From 0b5a6b3d98b5e5497b0b02a1deb6bce5dbdb0362 Mon Sep 17 00:00:00 2001 From: mktcode Date: Sat, 24 Aug 2024 16:00:10 +0200 Subject: [PATCH 38/41] refactor(tokenlists): remove unused data from aggregated tokenlist --- src/helpers/tokenlists.ts | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 1479f375..047cbb88 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -3,10 +3,7 @@ import { getAddress } from '@ethersproject/address'; type TokenlistToken = { chainId: number; address: string; - symbol: string; - name: string; logoURI: string; - decimals: number; }; type AggregatedTokenListToken = Omit & { @@ -27,16 +24,9 @@ function isTokenlistToken(token: unknown): token is TokenlistToken { return false; } - const { chainId, address, symbol, name, logoURI, decimals } = token as TokenlistToken; + const { chainId, address, logoURI } = token as TokenlistToken; - return ( - typeof chainId === 'number' && - typeof address === 'string' && - typeof symbol === 'string' && - typeof name === 'string' && - typeof logoURI === 'string' && - typeof decimals === 'number' - ); + return typeof chainId === 'number' && typeof address === 'string' && typeof logoURI === 'string'; } function isExpired() { @@ -165,9 +155,6 @@ export async function updateExpiredAggregatedTokenList() { const newToken: AggregatedTokenListToken = { chainId: token.chainId, address: token.address, - symbol: token.symbol, - name: token.name, - decimals: token.decimals, logoURIs: [logoURI] }; newTokenMap.set(tokenKey, newToken); From 0ee529ec4b2ebf1cbb24c921ee6ad3ea024d8994 Mon Sep 17 00:00:00 2001 From: mktcode Date: Sat, 24 Aug 2024 16:06:00 +0200 Subject: [PATCH 39/41] refactor(tokenlists): update AggregatedTokenList type to use string arrays directly --- src/helpers/tokenlists.ts | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 047cbb88..080d3572 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -6,11 +6,7 @@ type TokenlistToken = { logoURI: string; }; -type AggregatedTokenListToken = Omit & { - logoURIs: string[]; -}; - -type AggregatedTokenList = Map; +type AggregatedTokenList = Map; const TOKENLISTS_URL = 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; @@ -138,7 +134,7 @@ export async function updateExpiredAggregatedTokenList() { return; } - const newTokenMap = new Map(); + const newTokenMap = new Map(); const tokenListUris = await fetchListUris(); const tokenLists = await Promise.all(tokenListUris.map(fetchTokens)); @@ -150,19 +146,14 @@ export async function updateExpiredAggregatedTokenList() { const existingToken = newTokenMap.get(tokenKey); if (existingToken) { - existingToken.logoURIs.push(logoURI); + existingToken.push(logoURI); } else { - const newToken: AggregatedTokenListToken = { - chainId: token.chainId, - address: token.address, - logoURIs: [logoURI] - }; - newTokenMap.set(tokenKey, newToken); + newTokenMap.set(tokenKey, [logoURI]); } } } - newTokenMap.forEach(token => token.logoURIs.sort(sortByKeywordMatch)); + newTokenMap.forEach(token => token.sort(sortByKeywordMatch)); aggregatedTokenList = newTokenMap; lastUpdateTimestamp = Date.now(); @@ -176,5 +167,5 @@ export function findImageUrl(address: string, chainId: string) { throw new Error('Token not found in aggregated tokenlist'); } - return token.logoURIs[0]; + return token[0]; } From c0f8a9c2503971b999c86da45a7d90a0d4d5876e Mon Sep 17 00:00:00 2001 From: mktcode Date: Sat, 24 Aug 2024 16:22:44 +0200 Subject: [PATCH 40/41] chore(tokenlists): add comment --- src/helpers/tokenlists.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index 080d3572..dff67a62 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -75,6 +75,8 @@ const REPLACE_SIZE_REGEXES: { pattern: RegExp; replacement: string }[] = [ } ]; +// TODO: Since we do the sorting by keyword match, we should probably not change the URLs in place but add the "large" version to the list of URIs. +// Might be better for fallback mechanisms, instead of overwriting the version that was fetched. export function replaceURIPatterns(uri: string) { for (const { pattern, replacement } of REPLACE_SIZE_REGEXES) { uri = uri.replace(pattern, replacement); From 51bff9d637a1771fcf532fff1d67706f46c2e802 Mon Sep 17 00:00:00 2001 From: mktcode Date: Sat, 24 Aug 2024 17:36:55 +0200 Subject: [PATCH 41/41] refactor(tokenlists): add isUpdating flag to prevent concurrent updates --- src/helpers/tokenlists.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts index dff67a62..2d59a82c 100644 --- a/src/helpers/tokenlists.ts +++ b/src/helpers/tokenlists.ts @@ -14,6 +14,7 @@ const REQUEST_TIMEOUT = 5000; const TTL = 1000 * 60 * 60 * 24; let aggregatedTokenList: AggregatedTokenList = new Map(); let lastUpdateTimestamp: number | undefined; +let isUpdating = false; function isTokenlistToken(token: unknown): token is TokenlistToken { if (typeof token !== 'object' || token === null) { @@ -132,10 +133,12 @@ function getTokenKey(address: string, chainId: string) { } export async function updateExpiredAggregatedTokenList() { - if (!isExpired()) { + if (!isExpired() || isUpdating) { return; } + isUpdating = true; + const newTokenMap = new Map(); const tokenListUris = await fetchListUris(); @@ -159,6 +162,7 @@ export async function updateExpiredAggregatedTokenList() { aggregatedTokenList = newTokenMap; lastUpdateTimestamp = Date.now(); + isUpdating = false; } export function findImageUrl(address: string, chainId: string) {