From b25ccce9a9716772f7798049ec356909822dd4c7 Mon Sep 17 00:00:00 2001 From: "Antonio F. T" Date: Wed, 21 Feb 2024 18:03:23 +0100 Subject: [PATCH] Introduce IPFS_GATEWAY environment variable (#78) * Introduce IPFS_GATEWAY environment variable * Format * Fix compile error --- .github/.k8s/deploy.yml | 2 + .github/.k8s/deploy_goerli.yml | 2 + .github/.k8s/deploy_sepolia.yml | 2 + server/.env.example | 1 + server/src/state.rs | 8 +++- shared/src/core/mod.rs | 1 + shared/src/core/records.rs | 1 + shared/src/models/eip155/mod.rs | 38 +++++++++++++------ shared/src/models/ipfs/mod.rs | 23 +++++------ shared/src/models/lookup/image.rs | 19 ++++------ shared/src/models/lookup/mod.rs | 1 + shared/src/models/multicoin/decoding/p2pkh.rs | 12 ++++-- shared/src/models/multicoin/decoding/p2sh.rs | 36 +++++++++++------- shared/src/utils/sha256.rs | 2 +- worker/src/http_util.rs | 4 +- worker/src/lib.rs | 8 +++- 16 files changed, 102 insertions(+), 58 deletions(-) diff --git a/.github/.k8s/deploy.yml b/.github/.k8s/deploy.yml index 13ff852..a58f768 100644 --- a/.github/.k8s/deploy.yml +++ b/.github/.k8s/deploy.yml @@ -68,6 +68,8 @@ spec: value: redis://redis.enstate.svc.cluster.local:6379 - name: UNIVERSAL_RESOLVER value: 0x8cab227b1162f03b8338331adaad7aadc83b895e + - name: IPFS_GATEWAY + value: https://cloudflare-ipfs.com/ipfs/ resources: requests: cpu: 100m diff --git a/.github/.k8s/deploy_goerli.yml b/.github/.k8s/deploy_goerli.yml index 638f154..55b5df1 100644 --- a/.github/.k8s/deploy_goerli.yml +++ b/.github/.k8s/deploy_goerli.yml @@ -65,6 +65,8 @@ spec: value: https://rpc.ankr.com/eth_goerli,https://ethereum-goerli.publicnode.com,https://goerli.gateway.tenderly.co - name: UNIVERSAL_RESOLVER value: 0xfc4AC75C46C914aF5892d6d3eFFcebD7917293F1 + - name: IPFS_GATEWAY + value: https://cloudflare-ipfs.com/ipfs/ resources: requests: cpu: 100m diff --git a/.github/.k8s/deploy_sepolia.yml b/.github/.k8s/deploy_sepolia.yml index 9954101..029b762 100644 --- a/.github/.k8s/deploy_sepolia.yml +++ b/.github/.k8s/deploy_sepolia.yml @@ -65,6 +65,8 @@ spec: value: https://rpc.ankr.com/eth_sepolia,https://ethereum-sepolia.publicnode.com,https://sepolia.gateway.tenderly.co - name: UNIVERSAL_RESOLVER value: 0xBaBC7678D7A63104f1658c11D6AE9A21cdA09725 + - name: IPFS_GATEWAY, + value: https://cloudflare-ipfs.com/ipfs/ resources: requests: cpu: 100m diff --git a/server/.env.example b/server/.env.example index 0cdbc4b..6e94121 100644 --- a/server/.env.example +++ b/server/.env.example @@ -3,6 +3,7 @@ REDIS_URL=redis://localhost:6379 RPC_URL=https://rpc.ankr.com/eth OPENSEA_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxx UNIVERSAL_RESOLVER=0xc0497E381f536Be9ce14B0dD3817cBcAe57d2F62 +IPFS_GATEWAY=https://ipfs.io/ipfs/ # Optionally you can specify a comma-seperated list PROFILE_RECORDS, however if not provided there are sensible defaults # PROFILE_RECORDS=com.discord,com.twitter diff --git a/server/src/state.rs b/server/src/state.rs index c82ddbf..ea9d18c 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -1,13 +1,13 @@ -use enstate_shared::cache::{CacheLayer, PassthroughCacheLayer}; -use ethers_core::types::H160; use std::env; use std::sync::Arc; +use enstate_shared::cache::{CacheLayer, PassthroughCacheLayer}; use enstate_shared::core::ENSService; use enstate_shared::models::{ multicoin::cointype::{coins::CoinType, Coins}, records::Records, }; +use ethers_core::types::H160; use tracing::{info, warn}; use crate::provider::RoundRobin; @@ -63,6 +63,9 @@ impl AppState { let opensea_api_key = env::var("OPENSEA_API_KEY").expect("OPENSEA_API_KEY should've been set"); + let ipfs_gateway = + env::var("IPFS_GATEWAY").unwrap_or_else(|_| "https://ipfs.io/ipfs/".to_string()); + let universal_resolver = env::var("UNIVERSAL_RESOLVER") .expect("UNIVERSAL_RESOLVER should've been set") .parse::() @@ -73,6 +76,7 @@ impl AppState { cache, rpc: Box::new(provider), opensea_api_key, + ipfs_gateway, profile_records: Arc::from(profile_records), profile_chains: Arc::from(multicoin_chains), universal_resolver, diff --git a/shared/src/core/mod.rs b/shared/src/core/mod.rs index 004ce4e..4f304e9 100644 --- a/shared/src/core/mod.rs +++ b/shared/src/core/mod.rs @@ -53,6 +53,7 @@ pub struct ENSService { pub cache: Box, pub rpc: Box>>>, pub opensea_api_key: String, + pub ipfs_gateway: String, pub profile_records: Arc<[String]>, pub profile_chains: Arc<[CoinType]>, pub universal_resolver: H160, diff --git a/shared/src/core/records.rs b/shared/src/core/records.rs index fc5c5f8..8b74255 100644 --- a/shared/src/core/records.rs +++ b/shared/src/core/records.rs @@ -76,6 +76,7 @@ impl ENSService { let lookup_state = LookupState { rpc, opensea_api_key: self.opensea_api_key.clone(), + ipfs_gateway: self.ipfs_gateway.clone(), }; // Assume results & calldata have the same length diff --git a/shared/src/models/eip155/mod.rs b/shared/src/models/eip155/mod.rs index ab59cba..0a1313e 100644 --- a/shared/src/models/eip155/mod.rs +++ b/shared/src/models/eip155/mod.rs @@ -8,8 +8,8 @@ use thiserror::Error; use tracing::info; use crate::models::ipfs::{URLFetchError, OPENSEA_BASE_PREFIX}; +use crate::models::lookup::LookupState; use crate::models::multicoin::cointype::evm::ChainId; -use crate::core::CCIPProvider; use super::ipfs::IPFSURLUnparsed; @@ -56,8 +56,7 @@ pub async fn resolve_eip155( contract_type: EIP155ContractType, contract_address: &str, token_id: U256, - provider: &CCIPProvider, - opensea_api_key: &str, + state: &LookupState, ) -> Result { let chain_id: u64 = chain_id.into(); @@ -86,7 +85,7 @@ pub async fn resolve_eip155( typed_transaction.set_to(contract_h160); typed_transaction.set_data(Bytes::from(transaction_data)); - let res = provider.provider().call_raw(&typed_transaction).await?; + let res = state.rpc.provider().call_raw(&typed_transaction).await?; let res_data = res.to_vec(); @@ -115,13 +114,13 @@ pub async fn resolve_eip155( // TODO: Validate URL here let token_metadata_url = IPFSURLUnparsed::from_unparsed(token_metadata_url); - let token_metadata = token_metadata_url.fetch(opensea_api_key).await?; + let token_metadata = token_metadata_url.fetch(state).await?; let image = token_metadata.image.ok_or(EIP155Error::Other)?; info!("Image: {}", image); - let token_image_url = IPFSURLUnparsed::from_unparsed(image).to_url_or_gateway(); + let token_image_url = IPFSURLUnparsed::from_unparsed(image).to_url_or_gateway(state); Ok(token_image_url) } @@ -144,13 +143,18 @@ mod tests { .wrap_into(|it| CCIPReadMiddleware::new(Arc::from(it))); let opensea_api_key = env::var("OPENSEA_API_KEY").unwrap().to_string(); + let state = LookupState { + rpc: Arc::new(provider), + opensea_api_key, + ipfs_gateway: "https://ipfs.io/ipfs/".to_string(), + }; + let data = resolve_eip155( ChainId::Ethereum, EIP155ContractType::ERC721, "0xc92ceddfb8dd984a89fb494c376f9a48b999aafc", U256::from_dec_str("2257").unwrap(), - &provider, - &opensea_api_key, + &state, ) .await .unwrap(); @@ -165,13 +169,18 @@ mod tests { .wrap_into(|it| CCIPReadMiddleware::new(Arc::from(it))); let opensea_api_key = env::var("OPENSEA_API_KEY").unwrap().to_string(); + let state = LookupState { + rpc: Arc::new(provider), + opensea_api_key, + ipfs_gateway: "https://ipfs.io/ipfs/".to_string(), + }; + let data = resolve_eip155( ChainId::Ethereum, EIP155ContractType::ERC1155, "0xb32979486938aa9694bfc898f35dbed459f44424", U256::from_dec_str("10063").unwrap(), - &provider, - &opensea_api_key, + &state, ) .await .unwrap(); @@ -190,6 +199,12 @@ mod tests { .wrap_into(|it| CCIPReadMiddleware::new(Arc::from(it))); let opensea_api_key = env::var("OPENSEA_API_KEY").unwrap().to_string(); + let state = LookupState { + rpc: Arc::new(provider), + opensea_api_key, + ipfs_gateway: "https://ipfs.io/ipfs/".to_string(), + }; + let data = resolve_eip155( ChainId::Ethereum, EIP155ContractType::ERC1155, @@ -198,8 +213,7 @@ mod tests { "8112316025873927737505937898915153732580103913704334048512380490797008551937", ) .unwrap(), - &provider, - &opensea_api_key, + &state, ) .await .unwrap(); diff --git a/shared/src/models/ipfs/mod.rs b/shared/src/models/ipfs/mod.rs index ca70099..0ee48f8 100644 --- a/shared/src/models/ipfs/mod.rs +++ b/shared/src/models/ipfs/mod.rs @@ -2,6 +2,9 @@ use lazy_static::lazy_static; use reqwest::header::HeaderValue; use thiserror::Error; +use crate::models::lookup::image::IPFS_REGEX; +use crate::models::lookup::LookupState; + use super::erc721::metadata::NFTMetadata; #[derive(Debug, PartialEq)] @@ -26,10 +29,6 @@ lazy_static! { static ref RAW_IPFS_REGEX: regex::Regex = regex::Regex::new(r"^Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,}$") .expect("should be a valid regex"); - - static ref IPFS_REGEX: regex::Regex = - regex::Regex::new(r"^ipfs://(ip[fn]s/)?([0-9a-zA-Z]+(/.*)?)") - .expect("should be a valid regex"); } impl IPFSURLUnparsed { @@ -49,26 +48,24 @@ impl IPFSURLUnparsed { IPFSURLUnparsed::URL(value) } - pub fn from_ipfs(value: String) -> Self { - Self::from_unparsed(value) - } - // This function turns the unparsed - pub fn to_url_or_gateway(&self) -> String { + pub fn to_url_or_gateway(&self, state: &LookupState) -> String { match self { IPFSURLUnparsed::URL(url) => url.to_string(), - IPFSURLUnparsed::IPFS(hash) => format!("https://ipfs.io/ipfs/{}", hash), + IPFSURLUnparsed::IPFS(hash) => { + format!("{gateway}/{hash}", gateway = state.ipfs_gateway) + } } } - pub async fn fetch(&self, opensea_api_key: &str) -> Result { - let url = self.to_url_or_gateway(); + pub async fn fetch(&self, state: &LookupState) -> Result { + let url = self.to_url_or_gateway(state); let mut client_headers = reqwest::header::HeaderMap::new(); if url.starts_with(OPENSEA_BASE_PREFIX) { client_headers.insert( "X-API-KEY", - HeaderValue::from_str(opensea_api_key) + HeaderValue::from_str(&state.opensea_api_key) .unwrap_or_else(|_| HeaderValue::from_static("")), ); } diff --git a/shared/src/models/lookup/image.rs b/shared/src/models/lookup/image.rs index 09880a1..979a907 100644 --- a/shared/src/models/lookup/image.rs +++ b/shared/src/models/lookup/image.rs @@ -14,14 +14,13 @@ use crate::models::multicoin::cointype::evm::ChainId; use super::{abi_decode_universal_ccip, ENSLookupError, LookupState}; lazy_static! { - static ref IPFS_REGEX: regex::Regex = - regex::Regex::new(r"ipfs://([0-9a-zA-Z]+)").expect("should be a valid regex"); + pub static ref IPFS_REGEX: regex::Regex = + regex::Regex::new(r"^ipfs://(ip[fn]s/)?([0-9a-zA-Z]+(/.*)?)") + .expect("should be a valid regex"); static ref EIP155_REGEX: regex::Regex = regex::Regex::new(r"eip155:([0-9]+)/(erc1155|erc721):0x([0-9a-fA-F]{40})/([0-9]+)") .expect("should be a valid regex"); } -const IPFS_GATEWAY: &str = "https://ipfs.io/ipfs/"; - #[derive(Error, Debug)] enum ImageLookupError { #[error("Format error: {0}")] @@ -54,12 +53,10 @@ pub async fn decode(data: &[u8], state: &LookupState) -> Result Result Result, pub opensea_api_key: String, + pub ipfs_gateway: String, } lazy_static! { diff --git a/shared/src/models/multicoin/decoding/p2pkh.rs b/shared/src/models/multicoin/decoding/p2pkh.rs index 122659e..898bcce 100644 --- a/shared/src/models/multicoin/decoding/p2pkh.rs +++ b/shared/src/models/multicoin/decoding/p2pkh.rs @@ -12,11 +12,15 @@ impl MulticoinDecoder for P2PKHDecoder { fn decode(&self, data: &[u8]) -> Result { let bytes_len = data.len(); if bytes_len < 3 { - return Err(MulticoinDecoderError::InvalidStructure("len < 3".to_string())); + return Err(MulticoinDecoderError::InvalidStructure( + "len < 3".to_string(), + )); } if data[..2] != [0x76, 0xa9] { - return Err(MulticoinDecoderError::InvalidStructure("invalid header".to_string())); + return Err(MulticoinDecoderError::InvalidStructure( + "invalid header".to_string(), + )); } let len = data[2] as usize; @@ -29,7 +33,9 @@ impl MulticoinDecoder for P2PKHDecoder { } if data[bytes_len - 2..bytes_len] != [0x88, 0xac] { - return Err(MulticoinDecoderError::InvalidStructure("invalid end".to_string())); + return Err(MulticoinDecoderError::InvalidStructure( + "invalid end".to_string(), + )); } let pub_key_hash = &data[3..3 + len]; diff --git a/shared/src/models/multicoin/decoding/p2sh.rs b/shared/src/models/multicoin/decoding/p2sh.rs index 4a4455e..c26801d 100644 --- a/shared/src/models/multicoin/decoding/p2sh.rs +++ b/shared/src/models/multicoin/decoding/p2sh.rs @@ -12,37 +12,45 @@ impl MulticoinDecoder for P2SHDecoder { fn decode(&self, data: &[u8]) -> Result { let bytes_len = data.len(); if bytes_len < 2 { - return Err(MulticoinDecoderError::InvalidStructure("len < 2".to_string())); + return Err(MulticoinDecoderError::InvalidStructure( + "len < 2".to_string(), + )); } - + if data[0] != 0xa9 { - return Err(MulticoinDecoderError::InvalidStructure("invalid header".to_string())); + return Err(MulticoinDecoderError::InvalidStructure( + "invalid header".to_string(), + )); } - + let len = data[1] as usize; let expected_len = 2 + len + 1; - + if bytes_len != expected_len { - return Err(MulticoinDecoderError::InvalidStructure(format!("invalid length ({bytes_len:?} != {expected_len:?})"))); + return Err(MulticoinDecoderError::InvalidStructure(format!( + "invalid length ({bytes_len:?} != {expected_len:?})" + ))); } - + if data[bytes_len - 1] != 0x87 { - return Err(MulticoinDecoderError::InvalidStructure("invalid end".to_string())); + return Err(MulticoinDecoderError::InvalidStructure( + "invalid end".to_string(), + )); } - + let script_hash = &data[2..2 + len]; - + let mut full = script_hash.to_vec(); full.insert(0, self.version); - + let full_checksum = utils::sha256::hash(utils::sha256::hash(full.clone())); - + full.extend_from_slice(&full_checksum[..4]); - + let value = bs58::encode(full) .with_alphabet(Alphabet::BITCOIN) .into_string(); - + Ok(value) } } diff --git a/shared/src/utils/sha256.rs b/shared/src/utils/sha256.rs index 58b5228..3addcd1 100644 --- a/shared/src/utils/sha256.rs +++ b/shared/src/utils/sha256.rs @@ -5,4 +5,4 @@ pub fn hash>(data: T) -> Vec { hasher.update(data); hasher.finalize().as_slice().into() -} \ No newline at end of file +} diff --git a/worker/src/http_util.rs b/worker/src/http_util.rs index 0712233..b297b52 100644 --- a/worker/src/http_util.rs +++ b/worker/src/http_util.rs @@ -32,7 +32,9 @@ pub fn parse_query(req: &Request) -> worker::Result { let url = req.url()?; let query = url.query().unwrap_or(""); - SERDE_QS_CONFIG.deserialize_str::(query).map_err(|_| http_simple_status_error(StatusCode::BAD_REQUEST)) + SERDE_QS_CONFIG + .deserialize_str::(query) + .map_err(|_| http_simple_status_error(StatusCode::BAD_REQUEST)) } #[derive(Error, Debug)] diff --git a/worker/src/lib.rs b/worker/src/lib.rs index 0fa18d0..7343983 100644 --- a/worker/src/lib.rs +++ b/worker/src/lib.rs @@ -33,6 +33,11 @@ async fn main(req: Request, env: Env, _ctx: Context) -> worker::Result .expect("OPENSEA_API_KEY should've been set") .to_string(); + let ipfs_gateway = env + .var("IPFS_GATEWAY") + .map(|it| it.to_string()) + .unwrap_or_else(|_| "https://ipfs.io/ipfs/".to_string()); + let cache: Box = Box::new(CloudflareKVCache { env: Env::from(env.clone()), }); @@ -57,7 +62,8 @@ async fn main(req: Request, env: Env, _ctx: Context) -> worker::Result let service = ENSService { cache, rpc: Box::new(SimpleFactory::from(Arc::new(rpc))), - opensea_api_key: opensea_api_key.to_string(), + opensea_api_key, + ipfs_gateway, profile_records: Arc::from(profile_records), profile_chains: Arc::from(profile_chains), universal_resolver,