From 4a75b6818e5d9166f4ed615fe6e8138c0491bf8b Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 5 Jul 2024 10:24:19 +0100 Subject: [PATCH 01/46] feat: accumulator v2 (wip 1) --- runtime/src/bank.rs | 28 +++++++++++++++++-- runtime/src/bank/pyth_accumulator.rs | 42 ++++++++++++++++++++++++++++ sdk/program/src/account_info.rs | 1 + 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 runtime/src/bank/pyth_accumulator.rs diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 330f09fb80..425a834c22 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -199,6 +199,8 @@ mod builtin_programs; mod sysvar_cache; mod transaction_account_state_info; +mod pyth_accumulator; + pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0; pub const MAX_LEADER_SCHEDULE_STAKES: Epoch = 5; @@ -2242,9 +2244,31 @@ impl Bank { } info!("Accumulator: Updating accumulator. Slot: {}", self.slot()); - if let Err(e) = self.update_accumulator_impl() { - error!("Error updating accumulator: {:?}", e); + + lazy_static! { + static ref ACCUMULATOR_V2_SLOT: Option = + match std::env::var("PYTH_ACCUMULATOR_V2_FROM_SLOT") { + Ok(value) => Some( + value + .parse() + .expect("invalid value of PYTH_ACCUMULATOR_V2_FROM_SLOT env var") + ), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(err)) => { + panic!("invalid value of PYTH_ACCUMULATOR_V2_FROM_SLOT env var: {err:?}"); + } + }; } + + if ACCUMULATOR_V2_SLOT.map_or(false, |v2_slot| self.slot >= v2_slot) { + if let Err(e) = pyth_accumulator::update_v2(&self) { + error!("Error updating accumulator: {:?}", e); + } + } else { + if let Err(e) = self.update_accumulator_impl() { + error!("Error updating accumulator: {:?}", e); + } + }; } fn update_accumulator_impl(&self) -> std::result::Result<(), AccumulatorUpdateError> { diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs new file mode 100644 index 0000000000..de58e15206 --- /dev/null +++ b/runtime/src/bank/pyth_accumulator.rs @@ -0,0 +1,42 @@ +use std::env::{self, VarError}; + +use solana_sdk::pubkey::Pubkey; + +use crate::accounts_index::{ScanConfig, ScanError}; + +use super::Bank; + +#[derive(Debug, thiserror::Error)] +pub enum AccumulatorUpdateV2Error { + #[error("no oracle pubkey")] + NoOraclePubkey, + #[error("get_program_accounts failed to return accounts: {0}")] + GetProgramAccounts(#[from] ScanError), +} + +lazy_static! { + static ref ORACLE_PUBKEY: Option = match env::var("PYTH_ORACLE_PUBKEY") { + Ok(value) => Some( + value + .parse() + .expect("invalid value of PYTH_ORACLE_PUBKEY env var") + ), + Err(VarError::NotPresent) => None, + Err(VarError::NotUnicode(err)) => { + panic!("invalid value of PYTH_ORACLE_PUBKEY env var: {err:?}"); + } + }; +} + +pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateV2Error> { + let Some(oracle_pubkey) = &*ORACLE_PUBKEY else { + return Err(AccumulatorUpdateV2Error::NoOraclePubkey); + }; + + let accounts = bank + .get_program_accounts(oracle_pubkey, &ScanConfig::new(true)) + .map_err(AccumulatorUpdateV2Error::GetProgramAccounts)?; + + //... + Ok(()) +} diff --git a/sdk/program/src/account_info.rs b/sdk/program/src/account_info.rs index 3652e2251e..b78fdbcb47 100644 --- a/sdk/program/src/account_info.rs +++ b/sdk/program/src/account_info.rs @@ -184,6 +184,7 @@ impl<'a> AccountInfo<'a> { pub fn assign(&self, new_owner: &Pubkey) { // Set the non-mut owner field unsafe { + #[allow(invalid_reference_casting)] std::ptr::write_volatile( self.owner as *const Pubkey as *mut [u8; 32], new_owner.to_bytes(), From e54872f4fee1aed9579c1d03d2737a7a9fbbccd8 Mon Sep 17 00:00:00 2001 From: Reisen Date: Fri, 5 Jul 2024 09:43:04 +0000 Subject: [PATCH 02/46] feat: accumulator v2 (wip 2) --- Cargo.lock | 346 +++++++++++++---------- runtime/Cargo.toml | 5 +- runtime/src/bank.rs | 359 +----------------------- runtime/src/bank/pyth_accumulator.rs | 395 ++++++++++++++++++++++++++- sdk/program/src/account_info.rs | 1 - 5 files changed, 589 insertions(+), 517 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8e993688e..5ac1d3c533 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,8 +156,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "synstructure", ] @@ -168,8 +168,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -232,8 +232,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -243,8 +243,8 @@ version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -369,14 +369,18 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", + "clap 3.1.8", + "env_logger", "lazy_static", "lazycell", + "log", "peeking_take_while", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "regex", "rustc-hash", "shlex", + "which", ] [[package]] @@ -488,7 +492,7 @@ dependencies = [ "borsh-derive-internal", "borsh-schema-derive-internal", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.52", + "proc-macro2 1.0.86", "syn 1.0.109", ] @@ -498,8 +502,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -509,8 +513,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -617,8 +621,8 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aca418a974d83d40a0c1f0c5cba6ff4bc28d8df099109ca459a2118d40b6322" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -835,8 +839,8 @@ checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" dependencies = [ "heck 0.4.0", "proc-macro-error", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -909,8 +913,8 @@ version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef196d5d972878a48da7decb7686eded338b4858fbabeed513d63a7c98b2b82d" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "unicode-xid 0.2.2", ] @@ -1163,8 +1167,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df" dependencies = [ "convert_case", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustc_version 0.3.3", "syn 1.0.109", ] @@ -1251,8 +1255,8 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1339,8 +1343,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86b50932a01e7ec5c06160492ab660fb19b6bb2a7878030dd6cd68d21df9d4d" dependencies = [ "enum-ordinalize", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1380,8 +1384,8 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8958699f9359f0b04e691a13850d48b7de329138023876d07cbd024c2c820598" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1393,8 +1397,8 @@ checksum = "0b166c9e378360dd5a6666a9604bb4f54ae0cac39023ffbac425e917a2a04fef" dependencies = [ "num-bigint 0.4.3", "num-traits", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1405,8 +1409,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eb359f1476bf611266ac1f5355bc14aeca37b299d0ebccc038ee7058891c9cb" dependencies = [ "once_cell", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1649,8 +1653,8 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2305,8 +2309,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" dependencies = [ "proc-macro-crate 0.1.5", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2678,8 +2682,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2802,8 +2806,8 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2875,8 +2879,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2be1598bf1c313dcdd12092e3f1920f463462525a21b7b4e11b4168353d0123e" dependencies = [ "proc-macro-crate 1.1.0", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2943,8 +2947,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3024,8 +3028,8 @@ checksum = "084fd65d5dd8b3772edccb5ffd1e4b7eba43897ecd0f9401e330e8c542959408" dependencies = [ "Inflector", "proc-macro-error", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3172,8 +3176,8 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3223,8 +3227,8 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3320,7 +3324,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b83ec2d0af5c5c556257ff52c9f98934e243b9fd39604bfb2a9b75ec2e97f18" dependencies = [ - "proc-macro2 1.0.52", + "proc-macro2 1.0.86", "syn 1.0.109", ] @@ -3350,8 +3354,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "version_check", ] @@ -3362,8 +3366,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "version_check", ] @@ -3378,9 +3382,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.52" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -3473,8 +3477,8 @@ checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" dependencies = [ "anyhow", "itertools", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3486,8 +3490,8 @@ checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" dependencies = [ "anyhow", "itertools", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3520,6 +3524,38 @@ dependencies = [ "autotools", ] +[[package]] +name = "pyth-oracle" +version = "2.26.0" +dependencies = [ + "bindgen", + "bytemuck", + "byteorder", + "num-derive", + "num-traits", + "pythnet-sdk 1.13.6 (git+https://github.com/pyth-network/pyth-crosschain?rev=60144002053a93f424be70decd8a8ccb8d618d81)", + "solana-program 1.14.17", + "thiserror", +] + +[[package]] +name = "pythnet-sdk" +version = "1.13.6" +source = "git+https://github.com/pyth-network/pyth-crosschain?rev=60144002053a93f424be70decd8a8ccb8d618d81#60144002053a93f424be70decd8a8ccb8d618d81" +dependencies = [ + "bincode", + "borsh", + "bytemuck", + "byteorder", + "fast-math", + "hex", + "rustc_version 0.4.0", + "serde", + "sha3 0.10.8", + "slow_primes", + "thiserror", +] + [[package]] name = "pythnet-sdk" version = "1.13.6" @@ -3624,11 +3660,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.18" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ - "proc-macro2 1.0.52", + "proc-macro2 1.0.86", ] [[package]] @@ -4129,8 +4165,8 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "serde_derive_internals", "syn 1.0.109", ] @@ -4156,8 +4192,8 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -4255,8 +4291,8 @@ version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -4266,8 +4302,8 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -4338,8 +4374,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1b95bb2f4f624565e8fe8140c789af7e2082c0e0561b5a82a1b678baa9703dc" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustversion", "syn 1.0.109", ] @@ -5243,9 +5279,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.14.16" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b4953578272ac0fadec245e85e83ae86454611f0c0a7fff7d906835124bdcf" +checksum = "f53e63c8f2aac07bc21167e7ede9b9d010ae25523fff5c01b171d9bab9a5a394" dependencies = [ "ahash", "blake3", @@ -5270,7 +5306,7 @@ dependencies = [ "serde_derive", "serde_json", "sha2 0.10.2", - "solana-frozen-abi-macro 1.14.16", + "solana-frozen-abi-macro 1.14.17", "subtle", "thiserror", ] @@ -5310,12 +5346,12 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.14.16" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57892538250428ad3dc3cbe05f6cd75ad14f4f16734fcb91bc7cd5fbb63d6315" +checksum = "daeaaa2713c06a2fe4bcdcfe7e1af55ee8a89c4d6693860b4041997af667207a" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -5324,8 +5360,8 @@ dependencies = [ name = "solana-frozen-abi-macro" version = "1.14.173" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -5628,9 +5664,9 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.14.16" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06aa701c49493e93085dd1e800c05475baca15a9d4d527b59794f2ed0b66e055" +checksum = "2b502866be84a799633c0744e1d72b819a256337149e9fb6c7eee4db84ec63f5" dependencies = [ "env_logger", "lazy_static", @@ -5801,9 +5837,9 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.14.16" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f99052873619df68913cb8e92e28ff251a5483828925e87fa97ba15a9cbad51" +checksum = "d66c02ad6002fbe7903ec96edd16352fe7964d3ee43b02053112f5304529849f" dependencies = [ "base64 0.13.0", "bincode", @@ -5839,9 +5875,9 @@ dependencies = [ "serde_json", "sha2 0.10.2", "sha3 0.10.8", - "solana-frozen-abi 1.14.16", - "solana-frozen-abi-macro 1.14.16", - "solana-sdk-macro 1.14.16", + "solana-frozen-abi 1.14.17", + "solana-frozen-abi-macro 1.14.17", + "solana-sdk-macro 1.14.17", "thiserror", "tiny-bip39", "wasm-bindgen", @@ -6085,7 +6121,8 @@ dependencies = [ "num_cpus", "once_cell", "ouroboros", - "pythnet-sdk", + "pyth-oracle", + "pythnet-sdk 1.13.6 (git+https://github.com/pyth-network/pyth-crosschain?rev=e670f57f89b05398ca352e4accb1e32724a8e1b4)", "rand 0.7.3", "rand_chacha 0.2.2", "rayon", @@ -6112,17 +6149,19 @@ dependencies = [ "strum", "strum_macros", "symlink", + "syn 2.0.15", "tar", "tempfile", "thiserror", + "thiserror-impl", "zstd", ] [[package]] name = "solana-sdk" -version = "1.14.16" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb47da3e18cb669f6ace0b40cee0610e278903783e0c9f7fce1e1beb881a1b7" +checksum = "60cbad77fa09d23fa5e05029dec6c88e4b784be76cf6ae390f82cc04b8089e73" dependencies = [ "assert_matches", "base64 0.13.0", @@ -6159,11 +6198,11 @@ dependencies = [ "serde_json", "sha2 0.10.2", "sha3 0.10.8", - "solana-frozen-abi 1.14.16", - "solana-frozen-abi-macro 1.14.16", - "solana-logger 1.14.16", - "solana-program 1.14.16", - "solana-sdk-macro 1.14.16", + "solana-frozen-abi 1.14.17", + "solana-frozen-abi-macro 1.14.17", + "solana-logger 1.14.17", + "solana-program 1.14.17", + "solana-sdk-macro 1.14.17", "thiserror", "uriparse", "wasm-bindgen", @@ -6225,13 +6264,13 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.14.16" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d41a09b9cecd0a4df63c78a192adee99ebf2d3757c19713a68246e1d9789c7c" +checksum = "b73f54502e7d537472bf393ffce0c252c55b534f16797029a1614d79ec0209c9" dependencies = [ "bs58", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustversion", "syn 1.0.109", ] @@ -6241,8 +6280,8 @@ name = "solana-sdk-macro" version = "1.14.173" dependencies = [ "bs58", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustversion", "syn 1.0.109", ] @@ -6639,9 +6678,9 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.14.16" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab38abd096769f79fd8e3fe8465070f04742395db724606a5263c8ebc215567" +checksum = "b28c5ec36aa1393174f7ea18c0cb809af82c10977bc5b2e1240a6b4b048b8f24" dependencies = [ "aes-gcm-siv", "arrayref", @@ -6661,8 +6700,8 @@ dependencies = [ "serde", "serde_json", "sha3 0.9.1", - "solana-program 1.14.16", - "solana-sdk 1.14.16", + "solana-program 1.14.17", + "solana-sdk 1.14.17", "subtle", "thiserror", "zeroize", @@ -6747,7 +6786,7 @@ dependencies = [ "borsh", "num-derive", "num-traits", - "solana-program 1.14.16", + "solana-program 1.14.17", "spl-token", "spl-token-2022 0.5.0", "thiserror", @@ -6759,7 +6798,7 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd0dc6f70db6bacea7ff25870b016a65ba1d1b6013536f08e4fd79a8f9005325" dependencies = [ - "solana-program 1.14.16", + "solana-program 1.14.17", ] [[package]] @@ -6773,7 +6812,7 @@ dependencies = [ "num-derive", "num-traits", "num_enum", - "solana-program 1.14.16", + "solana-program 1.14.17", "thiserror", ] @@ -6788,8 +6827,8 @@ dependencies = [ "num-derive", "num-traits", "num_enum", - "solana-program 1.14.16", - "solana-zk-token-sdk 1.14.16", + "solana-program 1.14.17", + "solana-zk-token-sdk 1.14.17", "spl-memo", "spl-token", "thiserror", @@ -6806,8 +6845,8 @@ dependencies = [ "num-derive", "num-traits", "num_enum", - "solana-program 1.14.16", - "solana-zk-token-sdk 1.14.16", + "solana-program 1.14.17", + "solana-zk-token-sdk 1.14.17", "spl-memo", "spl-token", "thiserror", @@ -6864,8 +6903,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" dependencies = [ "heck 0.4.0", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustversion", "syn 1.0.109", ] @@ -6899,8 +6938,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2 1.0.86", + "quote 1.0.36", "unicode-ident", ] @@ -6916,8 +6966,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "unicode-xid 0.2.2", ] @@ -7000,8 +7050,8 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee42b4e559f17bce0385ebf511a7beb67d5cc33c12c96b7f4e9789919d9c10f" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -7061,8 +7111,8 @@ checksum = "8dd461f47ade621665c9f4e44b20449341769911c253275dc5cb03726cbb852c" dependencies = [ "cfg-if 1.0.0", "proc-macro-error", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -7083,22 +7133,22 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.31" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.31" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", - "syn 1.0.109", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 2.0.15", ] [[package]] @@ -7235,8 +7285,8 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -7426,9 +7476,9 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" dependencies = [ - "proc-macro2 1.0.52", + "proc-macro2 1.0.86", "prost-build 0.9.0", - "quote 1.0.18", + "quote 1.0.36", "syn 1.0.109", ] @@ -7439,9 +7489,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fbcd2800e34e743b9ae795867d5f77b535d3a3be69fd731e39145719752df8c" dependencies = [ "prettyplease", - "proc-macro2 1.0.52", + "proc-macro2 1.0.86", "prost-build 0.11.0", - "quote 1.0.18", + "quote 1.0.36", "syn 1.0.109", ] @@ -7515,8 +7565,8 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -7836,8 +7886,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "wasm-bindgen-shared", ] @@ -7860,7 +7910,7 @@ version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" dependencies = [ - "quote 1.0.18", + "quote 1.0.36", "wasm-bindgen-macro-support", ] @@ -7870,8 +7920,8 @@ version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -8145,8 +8195,8 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdff2024a851a322b08f179173ae2ba620445aef1e838f0c196820eade4ae0c7" dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.18", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "synstructure", ] diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 61f3f0193d..283e6e9d18 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -35,6 +35,7 @@ num-traits = { version = "0.2" } num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" +pyth-oracle = { path = "../../pyth-client/program/rust" } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "e670f57f89b05398ca352e4accb1e32724a8e1b4" } rand = "0.7.0" rayon = "1.5.3" @@ -59,9 +60,11 @@ solana-zk-token-sdk = { path = "../zk-token-sdk", version = "=1.14.173" } strum = { version = "0.24", features = ["derive"] } strum_macros = "0.24" symlink = "0.1.0" +syn = "=2.0.15" tar = "0.4.38" tempfile = "3.3.0" -thiserror = "1.0" +thiserror-impl = "=1.0.40" +thiserror = "=1.0.40" zstd = "0.11.2" [lib] diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 425a834c22..ce98dae1b9 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -205,8 +205,6 @@ pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0; pub const MAX_LEADER_SCHEDULE_STAKES: Epoch = 5; -pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; - pub type Rewrites = RwLock>; #[derive(Default)] @@ -444,26 +442,6 @@ pub struct LoadAndExecuteTransactionsOutput { pub error_counters: TransactionErrorMetrics, } -/// Accumulator specific error type. It would be nice to use `transaction::Error` but it does -/// not include any `Custom` style variant we can leverage, so we introduce our own. -#[derive(Debug, thiserror::Error)] -pub enum AccumulatorUpdateError { - #[error("get_program_accounts failed to return accounts")] - GetProgramAccounts, - - #[error("failed to serialize sequence account")] - FailedSequenceSerialization, - - #[error("failed to serialize message account")] - FailedMessageSerialization, - - #[error("io error")] - Io(#[from] std::io::Error), - - #[error("could not parse Pubkey from environment")] - InvalidEnvPubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), -} - #[derive(Debug, Clone)] pub enum DurableNonceFee { Valid(u64), @@ -1430,7 +1408,7 @@ impl Bank { // state before the accumulator is used. bank is in a fully // updated state before the accumulator is used. if !accumulator_moved_to_end_of_block { - bank.update_accumulator(); + pyth_accumulator::update_accumulator(&bank); } bank @@ -1816,7 +1794,7 @@ impl Bank { // the accumulator sysvar updates. sysvars are in a fully updated // state before the accumulator sysvar updates. if !accumulator_moved_to_end_of_block { - new.update_accumulator(); + pyth_accumulator::update_accumulator(&new); } }); @@ -2224,337 +2202,6 @@ impl Bank { ) } - /// Updates the Accumulator Sysvar at the start of a new slot. See `update_clock` to see a similar - /// sysvar this is based on. - /// - /// Note: - /// - Library imports are placed within this function to keep the diff against upstream small. - /// - This update will incur a performance hit on each slot, so must be kept efficient. - /// - Focused on Merkle for initial release but will generalise to more accumulators in future. - fn update_accumulator(&self) { - if !self - .feature_set - .is_active(&feature_set::enable_accumulator_sysvar::id()) - { - info!( - "Accumulator: Skipping because the feature is disabled. Slot: {}", - self.slot() - ); - return; - } - - info!("Accumulator: Updating accumulator. Slot: {}", self.slot()); - - lazy_static! { - static ref ACCUMULATOR_V2_SLOT: Option = - match std::env::var("PYTH_ACCUMULATOR_V2_FROM_SLOT") { - Ok(value) => Some( - value - .parse() - .expect("invalid value of PYTH_ACCUMULATOR_V2_FROM_SLOT env var") - ), - Err(std::env::VarError::NotPresent) => None, - Err(std::env::VarError::NotUnicode(err)) => { - panic!("invalid value of PYTH_ACCUMULATOR_V2_FROM_SLOT env var: {err:?}"); - } - }; - } - - if ACCUMULATOR_V2_SLOT.map_or(false, |v2_slot| self.slot >= v2_slot) { - if let Err(e) = pyth_accumulator::update_v2(&self) { - error!("Error updating accumulator: {:?}", e); - } - } else { - if let Err(e) = self.update_accumulator_impl() { - error!("Error updating accumulator: {:?}", e); - } - }; - } - - fn update_accumulator_impl(&self) -> std::result::Result<(), AccumulatorUpdateError> { - use { - byteorder::ReadBytesExt, - pythnet_sdk::{ - accumulators::{merkle::MerkleAccumulator, Accumulator}, - hashers::keccak256_160::Keccak160, - MESSAGE_BUFFER_PID, - }, - solana_sdk::borsh, - }; - - // Use the current Clock to determine the index into the accumulator ring buffer. - let ring_index = (self.slot() % 10_000) as u32; - - // Find all accounts owned by the Message Buffer program using get_program_accounts, and - // extract the account data. - let message_buffer_pid = self.env_pubkey_or( - "MESSAGE_BUFFER_PID", - Pubkey::new_from_array(MESSAGE_BUFFER_PID), - )?; - - let accounts = self - .get_program_accounts(&message_buffer_pid, &ScanConfig::new(true)) - .map_err(|_| AccumulatorUpdateError::GetProgramAccounts)?; - - let preimage = b"account:MessageBuffer"; - let mut expected_sighash = [0u8; 8]; - expected_sighash.copy_from_slice(&hashv(&[preimage]).to_bytes()[..8]); - - // Filter accounts that don't match the Anchor sighash. - let accounts = accounts.iter().filter(|(_, account)| { - // Remove accounts that do not start with the expected Anchor sighash. - let mut sighash = [0u8; 8]; - sighash.copy_from_slice(&account.data()[..8]); - sighash == expected_sighash - }); - - // This code, using the offsets in each Account, extracts the various data versions from - // the account. We deduplicate this result because the accumulator expects a set. - let accounts = accounts - .map(|(_, account)| { - let data = account.data(); - let mut cursor = std::io::Cursor::new(&data); - let _sighash = cursor.read_u64::()?; - let _bump = cursor.read_u8()?; - let _version = cursor.read_u8()?; - let header_len = cursor.read_u16::()?; - let mut header_begin = header_len; - let mut inputs = Vec::new(); - let mut cur_end_offsets_idx: usize = 0; - while let Some(end) = cursor.read_u16::().ok() { - if end == 0 || cur_end_offsets_idx == (u8::MAX as usize) { - break; - } - - let end_offset = header_len + end; - if end_offset as usize > data.len() { - break; - } - let accumulator_input_data = &data[header_begin as usize..end_offset as usize]; - inputs.push(accumulator_input_data); - header_begin = end_offset; - cur_end_offsets_idx += 1; - } - - Ok(inputs) - }) - .collect::, std::io::Error>>()? - .into_iter() - .flatten() - .sorted_unstable() - .dedup(); - - // We now generate a Proof PDA (Owned by the System Program) to store the resulting Proof - // Set. The derivation includes the ring buffer index to simulate a ring buffer in order - // for RPC users to select the correct proof for an associated VAA. - let (accumulator_account, _) = Pubkey::find_program_address( - &[b"AccumulatorState", &ring_index.to_be_bytes()], - &solana_sdk::system_program::id(), - ); - - let accumulator_data = { - let mut data = vec![]; - let acc_state_magic = &mut b"PAS1".to_vec(); - let accounts_data = - &mut borsh::BorshSerialize::try_to_vec(&accounts.clone().collect::>())?; - data.append(acc_state_magic); - data.append(&mut borsh::BorshSerialize::try_to_vec(&self.slot())?); - data.append(&mut borsh::BorshSerialize::try_to_vec( - &ACCUMULATOR_RING_SIZE, - )?); - data.append(accounts_data); - let owner = solana_sdk::system_program::id(); - let balance = self.get_minimum_balance_for_rent_exemption(data.len()); - let mut account = AccountSharedData::new(balance, data.len(), &owner); - account.set_data(data); - account - }; - - // Generate a Message owned by Wormhole to be sent cross-chain. This short-circuits the - // Wormhole message generation code that would normally be called, but the Guardian - // set filters our messages so this does not pose a security risk. - if let Some(accumulator) = MerkleAccumulator::::from_set(accounts) { - self.post_accumulator_attestation(accumulator, ring_index)?; - } - - // Write the Account Set into `accumulator_state` so that the hermes application can - // request historical data to prove. - info!( - "Accumulator: Writing accumulator state to {:?}", - accumulator_account - ); - self.store_account_and_update_capitalization(&accumulator_account, &accumulator_data); - - Ok(()) - } - - /// TODO: Safe integer conversion checks if any are missed. - fn post_accumulator_attestation( - &self, - acc: pythnet_sdk::accumulators::merkle::MerkleAccumulator< - pythnet_sdk::hashers::keccak256_160::Keccak160, - >, - ring_index: u32, - ) -> std::result::Result<(), AccumulatorUpdateError> { - use { - pythnet_sdk::{ - pythnet, - wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, - ACCUMULATOR_EMITTER_ADDRESS, - }, - solana_sdk::borsh::try_from_slice_unchecked, - }; - - let accumulator_sequence_addr = self.env_pubkey_or( - "ACCUMULATOR_SEQUENCE_ADDR", - Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), - )?; - - let accumulator_emitter_addr = self.env_pubkey_or( - "ACCUMULATOR_EMITTER_ADDR", - Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), - )?; - - // Wormhole uses a Sequence account that is incremented each time a message is posted. As - // we aren't calling Wormhole we need to bump this ourselves. If it doesn't exist, we just - // create it instead. - let mut sequence: AccumulatorSequenceTracker = { - let data = self - .get_account_with_fixed_root(&accumulator_sequence_addr) - .unwrap_or_default(); - let data = data.data(); - solana_sdk::borsh::try_from_slice_unchecked(data) - .unwrap_or(AccumulatorSequenceTracker { sequence: 0 }) - }; - - debug!("Accumulator: accumulator sequence: {:?}", sequence.sequence); - - // Generate the Message to emit via Wormhole. - let message = PostedMessageUnreliableData { - message: if !self - .feature_set - .is_active(&feature_set::zero_wormhole_message_timestamps::id()) - { - MessageData { - vaa_version: 1, - consistency_level: 1, - vaa_time: 1u32, - vaa_signature_account: Pubkey::default().to_bytes(), - submission_time: self.clock().unix_timestamp as u32, - nonce: 0, - sequence: sequence.sequence, - emitter_chain: 26, - emitter_address: accumulator_emitter_addr.to_bytes(), - payload: acc.serialize(self.slot(), ACCUMULATOR_RING_SIZE), - } - } else { - // Use Default::default() to ensure zeroed VAA fields. - MessageData { - vaa_version: 1, - consistency_level: 1, - submission_time: self.clock().unix_timestamp as u32, - sequence: sequence.sequence, - emitter_chain: 26, - emitter_address: accumulator_emitter_addr.to_bytes(), - payload: acc.serialize(self.slot(), ACCUMULATOR_RING_SIZE), - ..Default::default() - } - }, - }; - - debug!("Accumulator: Wormhole message data: {:?}", message.message); - let wormhole_pid = self.env_pubkey_or( - "WORMHOLE_PID", - Pubkey::new_from_array(pythnet::WORMHOLE_PID), - )?; - - // Now we can bump and write the Sequence account. - sequence.sequence += 1; - let sequence = solana_sdk::borsh::BorshSerialize::try_to_vec(&sequence) - .map_err(|_| AccumulatorUpdateError::FailedSequenceSerialization)?; - let sequence_balance = self.get_minimum_balance_for_rent_exemption(sequence.len()); - let sequence_account = { - let owner = &wormhole_pid; - let mut account = AccountSharedData::new(sequence_balance, sequence.len(), owner); - account.set_data(sequence); - account - }; - - // Serialize into (and create if necessary) the message account. - let message = solana_sdk::borsh::BorshSerialize::try_to_vec(&message) - .map_err(|_| AccumulatorUpdateError::FailedMessageSerialization)?; - let message_balance = self.get_minimum_balance_for_rent_exemption(message.len()); - let message_account = { - let owner = &wormhole_pid; - let mut account = AccountSharedData::new(message_balance, message.len(), owner); - account.set_data(message); - account - }; - - // The message_pda derivation includes the ring buffer index to simulate a ring buffer in order - // for RPC users to select the message for an associated VAA. - let (message_pda, _) = Pubkey::find_program_address( - &[b"AccumulatorMessage", &ring_index.to_be_bytes()], - &wormhole_pid, - ); - - self.store_account_and_update_capitalization(&accumulator_sequence_addr, &sequence_account); - - info!("Accumulator: Writing wormhole message to {:?}", message_pda); - self.store_account_and_update_capitalization(&message_pda, &message_account); - - Ok(()) - } - - /// Read the pubkey from the environment variable `var` or return `default` - /// if the variable is not set. - fn env_pubkey_or( - &self, - var: &str, - default: Pubkey, - ) -> std::result::Result { - Ok(std::env::var(var) - .as_deref() - .map(Pubkey::from_str) - .ok() - .transpose()? - .unwrap_or(default)) - } - - /// Get all accumulator related pubkeys from environment variables - /// or return default if the variable is not set. - pub fn get_accumulator_keys( - &self, - ) -> Vec<(&str, std::result::Result)> { - use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; - let accumulator_keys = vec![ - ( - "MESSAGE_BUFFER_PID", - Pubkey::new_from_array(MESSAGE_BUFFER_PID), - ), - // accumulator emitter address should always be the same regardless - // of environment but checking here for completeness - ( - "ACCUMULATOR_EMITTER_ADDR", - Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), - ), - ( - "ACCUMULATOR_SEQUENCE_ADDR", - Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), - ), - ( - "WORMHOLE_PID", - Pubkey::new_from_array(pythnet::WORMHOLE_PID), - ), - ]; - let accumulator_pubkeys: Vec<(&str, std::result::Result)> = - accumulator_keys - .iter() - .map(|(k, d)| (*k, self.env_pubkey_or(k, *d))) - .collect(); - accumulator_pubkeys - } - pub fn clock(&self) -> sysvar::clock::Clock { from_account(&self.get_account(&sysvar::clock::id()).unwrap_or_default()) .unwrap_or_default() @@ -3588,7 +3235,7 @@ impl Bank { // If accumulator move to end of block is active update the accumulator before doing // other tasks when freezing to avoid any conflicts. if accumulator_moved_to_end_of_block { - self.update_accumulator(); + pyth_accumulator::update_accumulator(self); } else { info!( "Accumulator: Skipping accumulating end of block because the feature is disabled. Slot: {}", diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index de58e15206..4125d50362 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -1,10 +1,23 @@ -use std::env::{self, VarError}; +use { + super::Bank, + crate::accounts_index::{ScanConfig, ScanError}, + byteorder::LittleEndian, + itertools::Itertools, + log::*, + solana_sdk::{ + account::{AccountSharedData, ReadableAccount}, + clock::Slot, + feature_set, + hash::hashv, + pubkey::Pubkey, + }, + std::{ + env::{self, VarError}, + str::FromStr, + }, +}; -use solana_sdk::pubkey::Pubkey; - -use crate::accounts_index::{ScanConfig, ScanError}; - -use super::Bank; +pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; #[derive(Debug, thiserror::Error)] pub enum AccumulatorUpdateV2Error { @@ -28,15 +41,375 @@ lazy_static! { }; } -pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateV2Error> { - let Some(oracle_pubkey) = &*ORACLE_PUBKEY else { - return Err(AccumulatorUpdateV2Error::NoOraclePubkey); +/// Accumulator specific error type. It would be nice to use `transaction::Error` but it does +/// not include any `Custom` style variant we can leverage, so we introduce our own. +#[derive(Debug, thiserror::Error)] +pub enum AccumulatorUpdateErrorV1 { + #[error("get_program_accounts failed to return accounts")] + GetProgramAccounts, + + #[error("failed to serialize sequence account")] + FailedSequenceSerialization, + + #[error("failed to serialize message account")] + FailedMessageSerialization, + + #[error("io error")] + Io(#[from] std::io::Error), + + #[error("could not parse Pubkey from environment")] + InvalidEnvPubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), +} + +/// Updates the Accumulator Sysvar at the start of a new slot. See `update_clock` to see a similar +/// sysvar this is based on. +/// +/// Note: +/// - Library imports are placed within this function to keep the diff against upstream small. +/// - This update will incur a performance hit on each slot, so must be kept efficient. +/// - Focused on Merkle for initial release but will generalise to more accumulators in future. +pub fn update_accumulator(bank: &Bank) { + if !bank + .feature_set + .is_active(&feature_set::enable_accumulator_sysvar::id()) + { + info!( + "Accumulator: Skipping because the feature is disabled. Slot: {}", + bank.slot() + ); + return; + } + + info!("Accumulator: Updating accumulator. Slot: {}", bank.slot()); + + lazy_static! { + static ref ACCUMULATOR_V2_SLOT: Option = + match std::env::var("PYTH_ACCUMULATOR_V2_FROM_SLOT") { + Ok(value) => Some( + value + .parse() + .expect("invalid value of PYTH_ACCUMULATOR_V2_FROM_SLOT env var") + ), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(err)) => { + panic!("invalid value of PYTH_ACCUMULATOR_V2_FROM_SLOT env var: {err:?}"); + } + }; + } + + if (*ACCUMULATOR_V2_SLOT).map_or(false, |v2_slot| bank.slot() >= v2_slot) { + if let Err(e) = update_v2(bank) { + error!("Error updating accumulator: {:?}", e); + } + } else { + if let Err(e) = update_v1(bank) { + error!("Error updating accumulator: {:?}", e); + } + }; +} + +/// Read the pubkey from the environment variable `var` or return `default` +/// if the variable is not set. +fn env_pubkey_or( + var: &str, + default: Pubkey, +) -> std::result::Result { + Ok(std::env::var(var) + .as_deref() + .map(Pubkey::from_str) + .ok() + .transpose()? + .unwrap_or(default)) +} + +/// Get all accumulator related pubkeys from environment variables +/// or return default if the variable is not set. +pub fn get_accumulator_keys() -> Vec<( + &'static str, + std::result::Result, +)> { + use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; + let accumulator_keys = vec![ + ( + "MESSAGE_BUFFER_PID", + Pubkey::new_from_array(MESSAGE_BUFFER_PID), + ), + // accumulator emitter address should always be the same regardless + // of environment but checking here for completeness + ( + "ACCUMULATOR_EMITTER_ADDR", + Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), + ), + ( + "ACCUMULATOR_SEQUENCE_ADDR", + Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), + ), + ( + "WORMHOLE_PID", + Pubkey::new_from_array(pythnet::WORMHOLE_PID), + ), + ]; + let accumulator_pubkeys: Vec<(&str, std::result::Result)> = + accumulator_keys + .iter() + .map(|(k, d)| (*k, env_pubkey_or(k, *d))) + .collect(); + accumulator_pubkeys +} + +pub fn update_v1(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { + use { + byteorder::ReadBytesExt, + pythnet_sdk::{ + accumulators::{merkle::MerkleAccumulator, Accumulator}, + hashers::keccak256_160::Keccak160, + MESSAGE_BUFFER_PID, + }, + solana_sdk::borsh, }; + // Use the current Clock to determine the index into the accumulator ring buffer. + let ring_index = (bank.slot() % 10_000) as u32; + + // Find all accounts owned by the Message Buffer program using get_program_accounts, and + // extract the account data. + let message_buffer_pid = env_pubkey_or( + "MESSAGE_BUFFER_PID", + Pubkey::new_from_array(MESSAGE_BUFFER_PID), + )?; + let accounts = bank - .get_program_accounts(oracle_pubkey, &ScanConfig::new(true)) + .get_program_accounts(&message_buffer_pid, &ScanConfig::new(true)) + .map_err(|_| AccumulatorUpdateErrorV1::GetProgramAccounts)?; + + let preimage = b"account:MessageBuffer"; + let mut expected_sighash = [0u8; 8]; + expected_sighash.copy_from_slice(&hashv(&[preimage]).to_bytes()[..8]); + + // Filter accounts that don't match the Anchor sighash. + let accounts = accounts.iter().filter(|(_, account)| { + // Remove accounts that do not start with the expected Anchor sighash. + let mut sighash = [0u8; 8]; + sighash.copy_from_slice(&account.data()[..8]); + sighash == expected_sighash + }); + + // This code, using the offsets in each Account, extracts the various data versions from + // the account. We deduplicate this result because the accumulator expects a set. + let accounts = accounts + .map(|(_, account)| { + let data = account.data(); + let mut cursor = std::io::Cursor::new(&data); + let _sighash = cursor.read_u64::()?; + let _bump = cursor.read_u8()?; + let _version = cursor.read_u8()?; + let header_len = cursor.read_u16::()?; + let mut header_begin = header_len; + let mut inputs = Vec::new(); + let mut cur_end_offsets_idx: usize = 0; + while let Some(end) = cursor.read_u16::().ok() { + if end == 0 || cur_end_offsets_idx == (u8::MAX as usize) { + break; + } + + let end_offset = header_len + end; + if end_offset as usize > data.len() { + break; + } + let accumulator_input_data = &data[header_begin as usize..end_offset as usize]; + inputs.push(accumulator_input_data); + header_begin = end_offset; + cur_end_offsets_idx += 1; + } + + Ok(inputs) + }) + .collect::, std::io::Error>>()? + .into_iter() + .flatten() + .sorted_unstable() + .dedup(); + + // We now generate a Proof PDA (Owned by the System Program) to store the resulting Proof + // Set. The derivation includes the ring buffer index to simulate a ring buffer in order + // for RPC users to select the correct proof for an associated VAA. + let (accumulator_account, _) = Pubkey::find_program_address( + &[b"AccumulatorState", &ring_index.to_be_bytes()], + &solana_sdk::system_program::id(), + ); + + let accumulator_data = { + let mut data = vec![]; + let acc_state_magic = &mut b"PAS1".to_vec(); + let accounts_data = + &mut borsh::BorshSerialize::try_to_vec(&accounts.clone().collect::>())?; + data.append(acc_state_magic); + data.append(&mut borsh::BorshSerialize::try_to_vec(&bank.slot())?); + data.append(&mut borsh::BorshSerialize::try_to_vec( + &ACCUMULATOR_RING_SIZE, + )?); + data.append(accounts_data); + let owner = solana_sdk::system_program::id(); + let balance = bank.get_minimum_balance_for_rent_exemption(data.len()); + let mut account = AccountSharedData::new(balance, data.len(), &owner); + account.set_data(data); + account + }; + + // Generate a Message owned by Wormhole to be sent cross-chain. This short-circuits the + // Wormhole message generation code that would normally be called, but the Guardian + // set filters our messages so this does not pose a security risk. + if let Some(accumulator) = MerkleAccumulator::::from_set(accounts) { + post_accumulator_attestation(bank, accumulator, ring_index)?; + } + + // Write the Account Set into `accumulator_state` so that the hermes application can + // request historical data to prove. + info!( + "Accumulator: Writing accumulator state to {:?}", + accumulator_account + ); + bank.store_account_and_update_capitalization(&accumulator_account, &accumulator_data); + + Ok(()) +} + +/// TODO: Safe integer conversion checks if any are missed. +fn post_accumulator_attestation( + bank: &Bank, + acc: pythnet_sdk::accumulators::merkle::MerkleAccumulator< + pythnet_sdk::hashers::keccak256_160::Keccak160, + >, + ring_index: u32, +) -> std::result::Result<(), AccumulatorUpdateErrorV1> { + use { + pythnet_sdk::{ + pythnet, + wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, + ACCUMULATOR_EMITTER_ADDRESS, + }, + solana_sdk::borsh::try_from_slice_unchecked, + }; + + let accumulator_sequence_addr = env_pubkey_or( + "ACCUMULATOR_SEQUENCE_ADDR", + Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), + )?; + + let accumulator_emitter_addr = env_pubkey_or( + "ACCUMULATOR_EMITTER_ADDR", + Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), + )?; + + // Wormhole uses a Sequence account that is incremented each time a message is posted. As + // we aren't calling Wormhole we need to bump this ourselves. If it doesn't exist, we just + // create it instead. + let mut sequence: AccumulatorSequenceTracker = { + let data = bank + .get_account_with_fixed_root(&accumulator_sequence_addr) + .unwrap_or_default(); + let data = data.data(); + solana_sdk::borsh::try_from_slice_unchecked(data) + .unwrap_or(AccumulatorSequenceTracker { sequence: 0 }) + }; + + debug!("Accumulator: accumulator sequence: {:?}", sequence.sequence); + + // Generate the Message to emit via Wormhole. + let message = PostedMessageUnreliableData { + message: if !bank + .feature_set + .is_active(&feature_set::zero_wormhole_message_timestamps::id()) + { + MessageData { + vaa_version: 1, + consistency_level: 1, + vaa_time: 1u32, + vaa_signature_account: Pubkey::default().to_bytes(), + submission_time: bank.clock().unix_timestamp as u32, + nonce: 0, + sequence: sequence.sequence, + emitter_chain: 26, + emitter_address: accumulator_emitter_addr.to_bytes(), + payload: acc.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), + } + } else { + // Use Default::default() to ensure zeroed VAA fields. + MessageData { + vaa_version: 1, + consistency_level: 1, + submission_time: bank.clock().unix_timestamp as u32, + sequence: sequence.sequence, + emitter_chain: 26, + emitter_address: accumulator_emitter_addr.to_bytes(), + payload: acc.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), + ..Default::default() + } + }, + }; + + debug!("Accumulator: Wormhole message data: {:?}", message.message); + let wormhole_pid = env_pubkey_or( + "WORMHOLE_PID", + Pubkey::new_from_array(pythnet::WORMHOLE_PID), + )?; + + // Now we can bump and write the Sequence account. + sequence.sequence += 1; + let sequence = solana_sdk::borsh::BorshSerialize::try_to_vec(&sequence) + .map_err(|_| AccumulatorUpdateErrorV1::FailedSequenceSerialization)?; + let sequence_balance = bank.get_minimum_balance_for_rent_exemption(sequence.len()); + let sequence_account = { + let owner = &wormhole_pid; + let mut account = AccountSharedData::new(sequence_balance, sequence.len(), owner); + account.set_data(sequence); + account + }; + + // Serialize into (and create if necessary) the message account. + let message = solana_sdk::borsh::BorshSerialize::try_to_vec(&message) + .map_err(|_| AccumulatorUpdateErrorV1::FailedMessageSerialization)?; + let message_balance = bank.get_minimum_balance_for_rent_exemption(message.len()); + let message_account = { + let owner = &wormhole_pid; + let mut account = AccountSharedData::new(message_balance, message.len(), owner); + account.set_data(message); + account + }; + + // The message_pda derivation includes the ring buffer index to simulate a ring buffer in order + // for RPC users to select the message for an associated VAA. + let (message_pda, _) = Pubkey::find_program_address( + &[b"AccumulatorMessage", &ring_index.to_be_bytes()], + &wormhole_pid, + ); + + bank.store_account_and_update_capitalization(&accumulator_sequence_addr, &sequence_account); + + info!("Accumulator: Writing wormhole message to {:?}", message_pda); + bank.store_account_and_update_capitalization(&message_pda, &message_account); + + Ok(()) +} + +pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateV2Error> { + // 1. Find Oracle + let oracle_pubkey = ORACLE_PUBKEY.ok_or(AccumulatorUpdateV2Error::NoOraclePubkey)?; + + // 2. Find Oracle Accounts + let accounts = bank + .get_program_accounts(&oracle_pubkey, &ScanConfig::new(true)) .map_err(AccumulatorUpdateV2Error::GetProgramAccounts)?; - //... + // 3. Filter for Price Accounts + let accounts = accounts.iter().filter(|(_, account)| true); + + // 4. Pass the PriceAccounts to the Oracle Code + // - Change Oracle to not run update code. + // - Change Oracle to have the aggregation code itself as a pure function. + // - Call aggregation with PriceAccount as input. + // 5. Merkleize the results. + // 6. Create Wormhole Message Account + Ok(()) } diff --git a/sdk/program/src/account_info.rs b/sdk/program/src/account_info.rs index b78fdbcb47..3652e2251e 100644 --- a/sdk/program/src/account_info.rs +++ b/sdk/program/src/account_info.rs @@ -184,7 +184,6 @@ impl<'a> AccountInfo<'a> { pub fn assign(&self, new_owner: &Pubkey) { // Set the non-mut owner field unsafe { - #[allow(invalid_reference_casting)] std::ptr::write_volatile( self.owner as *const Pubkey as *mut [u8; 32], new_owner.to_bytes(), From 0bb6502304928816fb3620745240e63e58615245 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 5 Jul 2024 18:27:20 +0100 Subject: [PATCH 03/46] chore: fix some warnings and fix validator build --- Cargo.lock | 74 +++++++++++++++++++--------- core/src/validator.rs | 16 +++--- ledger/src/blockstore.rs | 2 + runtime/src/bank.rs | 3 +- runtime/src/bank/pyth_accumulator.rs | 19 +++---- 5 files changed, 70 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ac1d3c533..f49ea6b650 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -282,7 +282,7 @@ checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25" dependencies = [ "async-trait", "axum-core", - "bitflags", + "bitflags 1.3.2", "bytes", "futures-util", "http", @@ -366,7 +366,7 @@ version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cexpr", "clang-sys", "clap 3.1.8", @@ -383,6 +383,26 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2 1.0.86", + "quote 1.0.36", + "regex", + "rustc-hash", + "shlex", + "syn 1.0.109", +] + [[package]] name = "bit-set" version = "0.5.2" @@ -404,6 +424,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "bytemuck", +] + [[package]] name = "bitmaps" version = "2.1.0" @@ -807,7 +836,7 @@ checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" dependencies = [ "ansi_term", "atty", - "bitflags", + "bitflags 1.3.2", "strsim 0.8.0", "textwrap 0.11.0", "unicode-width", @@ -821,7 +850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c" dependencies = [ "atty", - "bitflags", + "bitflags 1.3.2", "clap_derive", "indexmap", "lazy_static", @@ -1881,7 +1910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cff78e5788be1e0ab65b04d306b2ed5092c815ec97ec70f4ebd5aee158aa55d" dependencies = [ "base64 0.13.0", - "bitflags", + "bitflags 1.3.2", "bytes", "headers-core", "http", @@ -2433,11 +2462,11 @@ checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" [[package]] name = "librocksdb-sys" -version = "0.8.0+7.4.4" +version = "0.8.3+7.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d" +checksum = "557b255ff04123fcc176162f56ed0c9cd42d8f357cf55b3fabeb60f7413741b3" dependencies = [ - "bindgen", + "bindgen 0.64.0", "bzip2-sys", "cc", "glob", @@ -2728,7 +2757,7 @@ version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "libc", "memoffset", @@ -2932,7 +2961,7 @@ version = "0.10.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "foreign-types", "libc", @@ -3396,7 +3425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0d9cc07f18492d879586c92b485def06bc850da3118075cd45d50e9c95b0e5" dependencies = [ "bit-set", - "bitflags", + "bitflags 1.3.2", "byteorder", "lazy_static", "num-traits", @@ -3528,7 +3557,8 @@ dependencies = [ name = "pyth-oracle" version = "2.26.0" dependencies = [ - "bindgen", + "bindgen 0.60.1", + "bitflags 2.6.0", "bytemuck", "byteorder", "num-derive", @@ -3855,7 +3885,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -4041,7 +4071,7 @@ version = "0.34.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb617eb09c4ef1536405e357e3b63f39e3ab4cc2159db05395278ad5c352bb16" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -4223,7 +4253,7 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -5528,7 +5558,7 @@ version = "1.14.173" dependencies = [ "assert_matches", "bincode", - "bitflags", + "bitflags 1.3.2", "bs58", "byteorder", "chrono", @@ -5843,7 +5873,7 @@ checksum = "d66c02ad6002fbe7903ec96edd16352fe7964d3ee43b02053112f5304529849f" dependencies = [ "base64 0.13.0", "bincode", - "bitflags", + "bitflags 1.3.2", "blake3", "borsh", "borsh-derive", @@ -5892,7 +5922,7 @@ dependencies = [ "assert_matches", "base64 0.13.0", "bincode", - "bitflags", + "bitflags 1.3.2", "blake3", "borsh", "borsh-derive", @@ -6166,7 +6196,7 @@ dependencies = [ "assert_matches", "base64 0.13.0", "bincode", - "bitflags", + "bitflags 1.3.2", "borsh", "bs58", "bytemuck", @@ -6216,7 +6246,7 @@ dependencies = [ "assert_matches", "base64 0.13.0", "bincode", - "bitflags", + "bitflags 1.3.2", "borsh", "bs58", "bytemuck", @@ -6988,7 +7018,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1123645dfaf2b5eac6b6c88addafc359c789b8ef2a770ecaef758c1ddf363ea4" dependencies = [ - "bitflags", + "bitflags 1.3.2", "byteorder", "libc", "thiserror", @@ -7521,7 +7551,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bytes", "futures-core", "futures-util", diff --git a/core/src/validator.rs b/core/src/validator.rs index 79a0162f22..4c7fef4e21 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -1,6 +1,7 @@ //! The `validator` module hosts all the validator microservices. pub use solana_perf::report_target_features; +use solana_runtime::bank::pyth_accumulator; use { crate::{ accounts_hash_verifier::AccountsHashVerifier, @@ -1521,15 +1522,12 @@ fn load_blockstore( } } - { - let bank = bank_forks.write().unwrap().working_bank(); - for (key_name, pk_res) in bank.get_accumulator_keys() { - match pk_res { - Ok(pk) => info!("Accumulator {}: {}", key_name, pk), - Err(err) => { - error!("Failed to get Accumulator {}: {:?}", key_name, err); - std::process::abort(); - } + for (key_name, pk_res) in pyth_accumulator::get_accumulator_keys() { + match pk_res { + Ok(pk) => info!("Accumulator {}: {}", key_name, pk), + Err(err) => { + error!("Failed to get Accumulator {}: {:?}", key_name, err); + std::process::abort(); } } } diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 2f0f799aa0..d7bcd29bdb 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -4323,11 +4323,13 @@ pub fn make_chaining_slot_entries( } #[cfg(not(unix))] +#[allow(dead_code)] // No longer called. fn adjust_ulimit_nofile(_enforce_ulimit_nofile: bool) -> Result<()> { Ok(()) } #[cfg(unix)] +#[allow(dead_code)] // No longer called. fn adjust_ulimit_nofile(enforce_ulimit_nofile: bool) -> Result<()> { // Rocks DB likes to have many open files. The default open file descriptor limit is // usually not enough diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index ce98dae1b9..9254625cff 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -158,7 +158,6 @@ use { ops::{Deref, RangeInclusive}, path::PathBuf, rc::Rc, - str::FromStr, sync::{ atomic::{ AtomicBool, AtomicI64, AtomicU64, AtomicUsize, @@ -199,7 +198,7 @@ mod builtin_programs; mod sysvar_cache; mod transaction_account_state_info; -mod pyth_accumulator; +pub mod pyth_accumulator; pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0; diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 4125d50362..dcf4ef0477 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -20,7 +20,7 @@ use { pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; #[derive(Debug, thiserror::Error)] -pub enum AccumulatorUpdateV2Error { +pub enum AccumulatorUpdateErrorV2 { #[error("no oracle pubkey")] NoOraclePubkey, #[error("get_program_accounts failed to return accounts: {0}")] @@ -282,13 +282,10 @@ fn post_accumulator_attestation( >, ring_index: u32, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - use { - pythnet_sdk::{ - pythnet, - wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, - ACCUMULATOR_EMITTER_ADDRESS, - }, - solana_sdk::borsh::try_from_slice_unchecked, + use pythnet_sdk::{ + pythnet, + wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, + ACCUMULATOR_EMITTER_ADDRESS, }; let accumulator_sequence_addr = env_pubkey_or( @@ -392,14 +389,14 @@ fn post_accumulator_attestation( Ok(()) } -pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateV2Error> { +pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV2> { // 1. Find Oracle - let oracle_pubkey = ORACLE_PUBKEY.ok_or(AccumulatorUpdateV2Error::NoOraclePubkey)?; + let oracle_pubkey = ORACLE_PUBKEY.ok_or(AccumulatorUpdateErrorV2::NoOraclePubkey)?; // 2. Find Oracle Accounts let accounts = bank .get_program_accounts(&oracle_pubkey, &ScanConfig::new(true)) - .map_err(AccumulatorUpdateV2Error::GetProgramAccounts)?; + .map_err(AccumulatorUpdateErrorV2::GetProgramAccounts)?; // 3. Filter for Price Accounts let accounts = accounts.iter().filter(|(_, account)| true); From a4cafa9bba2cd4e606014bd2c547523ea3c2bbff Mon Sep 17 00:00:00 2001 From: Reisen Date: Mon, 8 Jul 2024 08:24:56 +0000 Subject: [PATCH 04/46] feat: accumulator v2 (wip 3) --- Cargo.lock | 1 + runtime/Cargo.toml | 2 +- runtime/src/bank/pyth_accumulator.rs | 27 +++++++++++++++++++++------ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f49ea6b650..455a14336e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3565,6 +3565,7 @@ dependencies = [ "num-traits", "pythnet-sdk 1.13.6 (git+https://github.com/pyth-network/pyth-crosschain?rev=60144002053a93f424be70decd8a8ccb8d618d81)", "solana-program 1.14.17", + "solana-sdk 1.14.17", "thiserror", ] diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 283e6e9d18..39852a6361 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -35,7 +35,7 @@ num-traits = { version = "0.2" } num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" -pyth-oracle = { path = "../../pyth-client/program/rust" } +pyth-oracle = { path = "../../pyth-client/program/rust", features = ["library"] } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "e670f57f89b05398ca352e4accb1e32724a8e1b4" } rand = "0.7.0" rayon = "1.5.3" diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index dcf4ef0477..2b37f4ac0a 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -97,6 +97,7 @@ pub fn update_accumulator(bank: &Bank) { }; } + // TODO: No longer a slot or feature flag, based on price account flag. if (*ACCUMULATOR_V2_SLOT).map_or(false, |v2_slot| bank.slot() >= v2_slot) { if let Err(e) = update_v2(bank) { error!("Error updating accumulator: {:?}", e); @@ -398,14 +399,28 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV .get_program_accounts(&oracle_pubkey, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV2::GetProgramAccounts)?; - // 3. Filter for Price Accounts - let accounts = accounts.iter().filter(|(_, account)| true); + // 3. Call Aggregation on Price Accounts. + for (pubkey, mut account) in accounts { + let mut price_account_data = account.data().to_owned(); + + // Perform Accumulation + match pyth_oracle::validator::aggregate_price( + bank.slot(), + bank.clock().unix_timestamp, + &mut price_account_data, + ) { + Ok(success) => { + if success { + account.set_data(price_account_data); + bank.store_account_and_update_capitalization(&pubkey, &account); + } + } + Err(err) => trace!("Aggregation: failed to update_price_cumulative, {:?}", err), + } + } - // 4. Pass the PriceAccounts to the Oracle Code - // - Change Oracle to not run update code. - // - Change Oracle to have the aggregation code itself as a pure function. - // - Call aggregation with PriceAccount as input. // 5. Merkleize the results. + // 6. Create Wormhole Message Account Ok(()) From 48d01f7a983230e8f48d8b6fa5037c17bf983b69 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Mon, 8 Jul 2024 13:47:20 +0100 Subject: [PATCH 05/46] feat: skip message buffer aggregation if all price feeds use v2 aggregation --- runtime/src/bank/pyth_accumulator.rs | 188 +++++++++++++-------------- 1 file changed, 92 insertions(+), 96 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 2b37f4ac0a..d5d31ea56d 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -4,9 +4,9 @@ use { byteorder::LittleEndian, itertools::Itertools, log::*, + pyth_oracle::validator::AggregationError, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, - clock::Slot, feature_set, hash::hashv, pubkey::Pubkey, @@ -19,14 +19,6 @@ use { pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; -#[derive(Debug, thiserror::Error)] -pub enum AccumulatorUpdateErrorV2 { - #[error("no oracle pubkey")] - NoOraclePubkey, - #[error("get_program_accounts failed to return accounts: {0}")] - GetProgramAccounts(#[from] ScanError), -} - lazy_static! { static ref ORACLE_PUBKEY: Option = match env::var("PYTH_ORACLE_PUBKEY") { Ok(value) => Some( @@ -45,8 +37,8 @@ lazy_static! { /// not include any `Custom` style variant we can leverage, so we introduce our own. #[derive(Debug, thiserror::Error)] pub enum AccumulatorUpdateErrorV1 { - #[error("get_program_accounts failed to return accounts")] - GetProgramAccounts, + #[error("get_program_accounts failed to return accounts: {0}")] + GetProgramAccounts(#[from] ScanError), #[error("failed to serialize sequence account")] FailedSequenceSerialization, @@ -59,6 +51,9 @@ pub enum AccumulatorUpdateErrorV1 { #[error("could not parse Pubkey from environment")] InvalidEnvPubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), + + #[error("no oracle pubkey")] + NoOraclePubkey, } /// Updates the Accumulator Sysvar at the start of a new slot. See `update_clock` to see a similar @@ -82,31 +77,9 @@ pub fn update_accumulator(bank: &Bank) { info!("Accumulator: Updating accumulator. Slot: {}", bank.slot()); - lazy_static! { - static ref ACCUMULATOR_V2_SLOT: Option = - match std::env::var("PYTH_ACCUMULATOR_V2_FROM_SLOT") { - Ok(value) => Some( - value - .parse() - .expect("invalid value of PYTH_ACCUMULATOR_V2_FROM_SLOT env var") - ), - Err(std::env::VarError::NotPresent) => None, - Err(std::env::VarError::NotUnicode(err)) => { - panic!("invalid value of PYTH_ACCUMULATOR_V2_FROM_SLOT env var: {err:?}"); - } - }; + if let Err(e) = update_v2(bank) { + error!("Error updating accumulator v2: {:?}", e); } - - // TODO: No longer a slot or feature flag, based on price account flag. - if (*ACCUMULATOR_V2_SLOT).map_or(false, |v2_slot| bank.slot() >= v2_slot) { - if let Err(e) = update_v2(bank) { - error!("Error updating accumulator: {:?}", e); - } - } else { - if let Err(e) = update_v1(bank) { - error!("Error updating accumulator: {:?}", e); - } - }; } /// Read the pubkey from the environment variable `var` or return `default` @@ -158,7 +131,11 @@ pub fn get_accumulator_keys() -> Vec<( accumulator_pubkeys } -pub fn update_v1(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { +pub fn update_v1( + bank: &Bank, + v2_messages: Vec<&[u8]>, + use_message_buffers: bool, +) -> std::result::Result<(), AccumulatorUpdateErrorV1> { use { byteorder::ReadBytesExt, pythnet_sdk::{ @@ -179,57 +156,66 @@ pub fn update_v1(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV Pubkey::new_from_array(MESSAGE_BUFFER_PID), )?; - let accounts = bank - .get_program_accounts(&message_buffer_pid, &ScanConfig::new(true)) - .map_err(|_| AccumulatorUpdateErrorV1::GetProgramAccounts)?; - - let preimage = b"account:MessageBuffer"; - let mut expected_sighash = [0u8; 8]; - expected_sighash.copy_from_slice(&hashv(&[preimage]).to_bytes()[..8]); - - // Filter accounts that don't match the Anchor sighash. - let accounts = accounts.iter().filter(|(_, account)| { - // Remove accounts that do not start with the expected Anchor sighash. - let mut sighash = [0u8; 8]; - sighash.copy_from_slice(&account.data()[..8]); - sighash == expected_sighash - }); - - // This code, using the offsets in each Account, extracts the various data versions from - // the account. We deduplicate this result because the accumulator expects a set. - let accounts = accounts - .map(|(_, account)| { - let data = account.data(); - let mut cursor = std::io::Cursor::new(&data); - let _sighash = cursor.read_u64::()?; - let _bump = cursor.read_u8()?; - let _version = cursor.read_u8()?; - let header_len = cursor.read_u16::()?; - let mut header_begin = header_len; - let mut inputs = Vec::new(); - let mut cur_end_offsets_idx: usize = 0; - while let Some(end) = cursor.read_u16::().ok() { - if end == 0 || cur_end_offsets_idx == (u8::MAX as usize) { - break; + let message_buffer_accounts; + let v1_messages = if use_message_buffers { + message_buffer_accounts = bank + .get_program_accounts(&message_buffer_pid, &ScanConfig::new(true)) + .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; + + let preimage = b"account:MessageBuffer"; + let mut expected_sighash = [0u8; 8]; + expected_sighash.copy_from_slice(&hashv(&[preimage]).to_bytes()[..8]); + + // Filter accounts that don't match the Anchor sighash. + let message_buffer_accounts = message_buffer_accounts.iter().filter(|(_, account)| { + // Remove accounts that do not start with the expected Anchor sighash. + let mut sighash = [0u8; 8]; + sighash.copy_from_slice(&account.data()[..8]); + sighash == expected_sighash + }); + + // This code, using the offsets in each Account, extracts the various data versions from + // the account. We deduplicate this result because the accumulator expects a set. + message_buffer_accounts + .map(|(_, account)| { + let data = account.data(); + let mut cursor = std::io::Cursor::new(&data); + let _sighash = cursor.read_u64::()?; + let _bump = cursor.read_u8()?; + let _version = cursor.read_u8()?; + let header_len = cursor.read_u16::()?; + let mut header_begin = header_len; + let mut inputs = Vec::new(); + let mut cur_end_offsets_idx: usize = 0; + while let Some(end) = cursor.read_u16::().ok() { + if end == 0 || cur_end_offsets_idx == (u8::MAX as usize) { + break; + } + + let end_offset = header_len + end; + if end_offset as usize > data.len() { + break; + } + let accumulator_input_data = &data[header_begin as usize..end_offset as usize]; + inputs.push(accumulator_input_data); + header_begin = end_offset; + cur_end_offsets_idx += 1; } - let end_offset = header_len + end; - if end_offset as usize > data.len() { - break; - } - let accumulator_input_data = &data[header_begin as usize..end_offset as usize]; - inputs.push(accumulator_input_data); - header_begin = end_offset; - cur_end_offsets_idx += 1; - } + Ok(inputs) + }) + .collect::, std::io::Error>>()? + .into_iter() + .flatten() + .sorted_unstable() + .dedup() + .collect() + } else { + Vec::new() + }; - Ok(inputs) - }) - .collect::, std::io::Error>>()? - .into_iter() - .flatten() - .sorted_unstable() - .dedup(); + let mut messages = v1_messages; + messages.extend(v2_messages); // We now generate a Proof PDA (Owned by the System Program) to store the resulting Proof // Set. The derivation includes the ring buffer index to simulate a ring buffer in order @@ -242,8 +228,7 @@ pub fn update_v1(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV let accumulator_data = { let mut data = vec![]; let acc_state_magic = &mut b"PAS1".to_vec(); - let accounts_data = - &mut borsh::BorshSerialize::try_to_vec(&accounts.clone().collect::>())?; + let accounts_data = &mut borsh::BorshSerialize::try_to_vec(&messages)?; data.append(acc_state_magic); data.append(&mut borsh::BorshSerialize::try_to_vec(&bank.slot())?); data.append(&mut borsh::BorshSerialize::try_to_vec( @@ -260,7 +245,7 @@ pub fn update_v1(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV // Generate a Message owned by Wormhole to be sent cross-chain. This short-circuits the // Wormhole message generation code that would normally be called, but the Guardian // set filters our messages so this does not pose a security risk. - if let Some(accumulator) = MerkleAccumulator::::from_set(accounts) { + if let Some(accumulator) = MerkleAccumulator::::from_set(messages.into_iter()) { post_accumulator_attestation(bank, accumulator, ring_index)?; } @@ -390,14 +375,16 @@ fn post_accumulator_attestation( Ok(()) } -pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV2> { +pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { // 1. Find Oracle - let oracle_pubkey = ORACLE_PUBKEY.ok_or(AccumulatorUpdateErrorV2::NoOraclePubkey)?; + let oracle_pubkey = ORACLE_PUBKEY.ok_or(AccumulatorUpdateErrorV1::NoOraclePubkey)?; // 2. Find Oracle Accounts let accounts = bank .get_program_accounts(&oracle_pubkey, &ScanConfig::new(true)) - .map_err(AccumulatorUpdateErrorV2::GetProgramAccounts)?; + .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; + + let mut any_legacy_mode = false; // 3. Call Aggregation on Price Accounts. for (pubkey, mut account) in accounts { @@ -409,16 +396,25 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV bank.clock().unix_timestamp, &mut price_account_data, ) { - Ok(success) => { - if success { - account.set_data(price_account_data); - bank.store_account_and_update_capitalization(&pubkey, &account); - } + Ok(()) => { + account.set_data(price_account_data); + bank.store_account_and_update_capitalization(&pubkey, &account); } - Err(err) => trace!("Aggregation: failed to update_price_cumulative, {:?}", err), + Err(err) => match err { + AggregationError::NotPriceFeedAccount => {} + AggregationError::LegacyAggregationMode => { + any_legacy_mode = true; + } + AggregationError::NotTradingStatus => { + trace!("Aggregation: failed to update_price_cumulative, {:?}", err); + } + }, } } + // TODO: make new messages + update_v1(bank, Vec::new(), any_legacy_mode)?; + // 5. Merkleize the results. // 6. Create Wormhole Message Account From 281e9d4bc9734fe9294f35175829bd48ece6dc7a Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Mon, 8 Jul 2024 14:40:02 +0100 Subject: [PATCH 06/46] feat: combine v1 and v2 aggregation price feed messages in the merkle tree --- runtime/src/bank/pyth_accumulator.rs | 34 ++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index d5d31ea56d..3167f5cc17 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -131,9 +131,9 @@ pub fn get_accumulator_keys() -> Vec<( accumulator_pubkeys } -pub fn update_v1( +pub fn update_v1<'a>( bank: &Bank, - v2_messages: Vec<&[u8]>, + v2_messages: Vec<&'a [u8]>, use_message_buffers: bool, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { use { @@ -214,8 +214,8 @@ pub fn update_v1( Vec::new() }; - let mut messages = v1_messages; - messages.extend(v2_messages); + let mut messages = v2_messages; + messages.extend(v1_messages); // We now generate a Proof PDA (Owned by the System Program) to store the resulting Proof // Set. The derivation includes the ring buffer index to simulate a ring buffer in order @@ -384,7 +384,8 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV .get_program_accounts(&oracle_pubkey, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; - let mut any_legacy_mode = false; + let mut any_v1_aggregations = false; + let mut v2_messages = Vec::new(); // 3. Call Aggregation on Price Accounts. for (pubkey, mut account) in accounts { @@ -394,26 +395,31 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV match pyth_oracle::validator::aggregate_price( bank.slot(), bank.clock().unix_timestamp, + &pubkey.to_bytes().into(), &mut price_account_data, ) { - Ok(()) => { - account.set_data(price_account_data); - bank.store_account_and_update_capitalization(&pubkey, &account); + Ok(outcome) => { + if outcome.commit { + account.set_data(price_account_data); + bank.store_account_and_update_capitalization(&pubkey, &account); + } + v2_messages.extend(outcome.messages); } Err(err) => match err { AggregationError::NotPriceFeedAccount => {} - AggregationError::LegacyAggregationMode => { - any_legacy_mode = true; - } - AggregationError::NotTradingStatus => { - trace!("Aggregation: failed to update_price_cumulative, {:?}", err); + AggregationError::V1AggregationMode => { + any_v1_aggregations = true; } }, } } // TODO: make new messages - update_v1(bank, Vec::new(), any_legacy_mode)?; + update_v1( + bank, + v2_messages.iter().map(|x| &**x).collect(), + any_v1_aggregations, + )?; // 5. Merkleize the results. From db835e4e49a7044a5002d41a36aecb0366e3ff19 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Tue, 9 Jul 2024 10:07:28 +0100 Subject: [PATCH 07/46] refactor: clean up and avoid extra vector --- Cargo.lock | 1 + runtime/Cargo.toml | 2 +- runtime/src/bank/pyth_accumulator.rs | 22 ++++------------------ 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 455a14336e..bc9554def7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3556,6 +3556,7 @@ dependencies = [ [[package]] name = "pyth-oracle" version = "2.26.0" +source = "git+https://github.com/pyth-network/pyth-client?branch=accumulator-v2#27212eeda219b5d8d5d7c1e128fc2dd407861a8a" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 39852a6361..b83f212d9f 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -35,7 +35,7 @@ num-traits = { version = "0.2" } num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" -pyth-oracle = { path = "../../pyth-client/program/rust", features = ["library"] } +pyth-oracle = { git = "https://github.com/pyth-network/pyth-client", branch = "accumulator-v2", features = ["library"] } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "e670f57f89b05398ca352e4accb1e32724a8e1b4" } rand = "0.7.0" rayon = "1.5.3" diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 3167f5cc17..146e71a2cf 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -133,7 +133,7 @@ pub fn get_accumulator_keys() -> Vec<( pub fn update_v1<'a>( bank: &Bank, - v2_messages: Vec<&'a [u8]>, + v2_messages: &[Vec], use_message_buffers: bool, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { use { @@ -214,8 +214,8 @@ pub fn update_v1<'a>( Vec::new() }; - let mut messages = v2_messages; - messages.extend(v1_messages); + let mut messages = v1_messages; + messages.extend(v2_messages.iter().map(|x| &**x)); // We now generate a Proof PDA (Owned by the System Program) to store the resulting Proof // Set. The derivation includes the ring buffer index to simulate a ring buffer in order @@ -376,10 +376,8 @@ fn post_accumulator_attestation( } pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - // 1. Find Oracle let oracle_pubkey = ORACLE_PUBKEY.ok_or(AccumulatorUpdateErrorV1::NoOraclePubkey)?; - // 2. Find Oracle Accounts let accounts = bank .get_program_accounts(&oracle_pubkey, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; @@ -387,7 +385,6 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV let mut any_v1_aggregations = false; let mut v2_messages = Vec::new(); - // 3. Call Aggregation on Price Accounts. for (pubkey, mut account) in accounts { let mut price_account_data = account.data().to_owned(); @@ -414,16 +411,5 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV } } - // TODO: make new messages - update_v1( - bank, - v2_messages.iter().map(|x| &**x).collect(), - any_v1_aggregations, - )?; - - // 5. Merkleize the results. - - // 6. Create Wormhole Message Account - - Ok(()) + update_v1(bank, &v2_messages, any_v1_aggregations) } From 5df9d534151fc79c7cf9baf03a253284e7bec531 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Wed, 10 Jul 2024 17:23:23 +0100 Subject: [PATCH 08/46] fix: don't aggregate if already aggregated in the program --- Cargo.lock | 2 +- runtime/src/bank/pyth_accumulator.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc9554def7..f747cac27e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3556,7 +3556,7 @@ dependencies = [ [[package]] name = "pyth-oracle" version = "2.26.0" -source = "git+https://github.com/pyth-network/pyth-client?branch=accumulator-v2#27212eeda219b5d8d5d7c1e128fc2dd407861a8a" +source = "git+https://github.com/pyth-network/pyth-client?branch=accumulator-v2#f0ac2678d738d659ea4ef94f1db81e7d7e439183" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 146e71a2cf..17ecb155fd 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -404,7 +404,7 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV } Err(err) => match err { AggregationError::NotPriceFeedAccount => {} - AggregationError::V1AggregationMode => { + AggregationError::V1AggregationMode | AggregationError::AlreadyAggregated => { any_v1_aggregations = true; } }, From cd58b050bd2dd4d24e5800e2caa3a1c88bbb67c7 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Wed, 10 Jul 2024 18:02:11 +0100 Subject: [PATCH 09/46] test: bring back old pyth tests (wip) --- runtime/src/bank.rs | 3 + runtime/src/bank/pyth_accumulator_tests.rs | 646 +++++++++++++++++++++ 2 files changed, 649 insertions(+) create mode 100644 runtime/src/bank/pyth_accumulator_tests.rs diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 9254625cff..f81ab66a35 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -200,6 +200,9 @@ mod transaction_account_state_info; pub mod pyth_accumulator; +#[cfg(test)] +mod pyth_accumulator_tests; + pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0; pub const MAX_LEADER_SCHEDULE_STAKES: Epoch = 5; diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs new file mode 100644 index 0000000000..53af505c12 --- /dev/null +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -0,0 +1,646 @@ +use crate::{ + bank::{ + pyth_accumulator::{get_accumulator_keys, ACCUMULATOR_RING_SIZE}, + Bank, + }, + genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, +}; +use byteorder::ByteOrder; +use byteorder::{LittleEndian, ReadBytesExt}; +use itertools::Itertools; +use pythnet_sdk::{ + accumulators::{merkle::MerkleAccumulator, Accumulator}, + hashers::{keccak256_160::Keccak160, Hasher}, + wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, + ACCUMULATOR_EMITTER_ADDRESS, +}; +use solana_sdk::{ + account::{AccountSharedData, ReadableAccount, WritableAccount}, + borsh::{BorshDeserialize, BorshSerialize}, + epoch_schedule::EpochSchedule, + feature::{self, Feature}, + feature_set, + hash::hashv, + pubkey::Pubkey, + signature::keypair_from_seed, + signer::Signer, +}; +use std::{io::Read, sync::Arc}; + +// Create Message Account Bytes +// +// NOTE: This was serialized by hand, but should be replaced with the pythnet-sdk +// serializer once implemented. +fn create_message_buffer_bytes(msgs: Vec>) -> Vec { + let mut buffer = Vec::new(); + let preimage = b"account:MessageBuffer"; + buffer.extend_from_slice(&hashv(&[preimage]).to_bytes()[..8]); + buffer.extend_from_slice(&[0, 1, 10, 2]); + let mut sums: Vec = msgs.iter().map(|m| m.len() as u16).collect(); + sums.resize(255, 0u16); + buffer.extend( + sums.into_iter() + .scan(0, |acc, v| { + *acc += v; + Some(if v == 0 { v } else { *acc }.to_le_bytes()) + }) + .flatten(), + ); + buffer.extend(msgs.into_iter().flatten()); + buffer +} + +fn get_acc_sequence_tracker(bank: &Bank) -> AccumulatorSequenceTracker { + let account = bank + .get_account(&Pubkey::new_from_array( + pythnet_sdk::pythnet::ACCUMULATOR_SEQUENCE_ADDR, + )) + .unwrap(); + AccumulatorSequenceTracker::try_from_slice(&mut account.data()).unwrap() +} + +fn get_wormhole_message_account(bank: &Bank, ring_index: u32) -> AccountSharedData { + let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( + &[b"AccumulatorMessage", &ring_index.to_be_bytes()], + &Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), + ); + bank.get_account(&wormhole_message_pubkey) + .unwrap_or_default() +} + +fn get_accumulator_state(bank: &Bank, ring_index: u32) -> Vec { + let (accumulator_state_pubkey, _) = Pubkey::find_program_address( + &[b"AccumulatorState", &ring_index.to_be_bytes()], + &solana_sdk::system_program::id(), + ); + + let account = bank.get_account(&accumulator_state_pubkey).unwrap(); + account.data().to_vec() +} + +#[test] +fn test_update_accumulator_sysvar() { + let leader_pubkey = solana_sdk::pubkey::new_rand(); + let GenesisConfigInfo { + mut genesis_config, .. + } = create_genesis_config_with_leader(5, &leader_pubkey, 3); + + // The genesis create function uses `Develompent` mode which enables all feature flags, so + // we need to remove the accumulator sysvar in order to test the validator behaves + // correctly when the feature is disabled. We will re-enable it further into this test. + genesis_config + .accounts + .remove(&feature_set::enable_accumulator_sysvar::id()) + .unwrap(); + genesis_config + .accounts + .remove(&feature_set::move_accumulator_to_end_of_block::id()) + .unwrap(); + + // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here + // due to slot 0 having special handling. + let slots_in_epoch = 32; + genesis_config.epoch_schedule = EpochSchedule::new(slots_in_epoch); + let mut bank = Bank::new_for_tests(&genesis_config); + bank = new_from_parent(&Arc::new(bank)); + bank = new_from_parent(&Arc::new(bank)); + + let message_0 = vec![1u8; 127]; + let message_1 = vec![2u8; 127]; + // insert into message buffer in reverse order to test that accumulator + // sorts first + let messages = vec![message_1, message_0]; + + let message_buffer_bytes = create_message_buffer_bytes(messages.clone()); + + // Create a Message account. + let price_message_key = keypair_from_seed(&[1u8; 32]).unwrap(); + let mut price_message_account = bank + .get_account(&price_message_key.pubkey()) + .unwrap_or_default(); + price_message_account.set_lamports(1_000_000_000); + price_message_account.set_owner(Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID)); + price_message_account.set_data(message_buffer_bytes); + + // Store Message account so the accumulator sysvar updater can find it. + bank.store_account(&price_message_key.pubkey(), &price_message_account); + + // Derive the Wormhole Message Account that will be generated by the sysvar updater. + let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( + &[b"AccumulatorMessage", &(bank.slot() as u32).to_be_bytes()], + &Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), + ); + + // Account Data should be empty at this point. Check account data is []. + let wormhole_message_account = bank + .get_account(&wormhole_message_pubkey) + .unwrap_or_default(); + assert_eq!(wormhole_message_account.data().len(), 0); + + // Run accumulator by creating a new bank from parent,the feature is + // disabled so account data should still be empty. Check account data is + // still []. + bank = new_from_parent(&Arc::new(bank)); + + let wormhole_message_account = bank + .get_account(&wormhole_message_pubkey) + .unwrap_or_default(); + assert_eq!( + bank.feature_set + .is_active(&feature_set::enable_accumulator_sysvar::id()), + false + ); + assert_eq!(wormhole_message_account.data().len(), 0); + + // Enable Accumulator Feature (42 = random lamport balance, and the meaning of the universe). + let feature_id = feature_set::enable_accumulator_sysvar::id(); + let feature = Feature { + activated_at: Some(30), + }; + bank.store_account(&feature_id, &feature::create_account(&feature, 42)); + bank.compute_active_feature_set(true); + for _ in 0..slots_in_epoch { + bank = new_from_parent(&Arc::new(bank)); + } + + // Feature should now be enabled on the new bank as the epoch has changed. + assert_eq!( + bank.feature_set + .is_active(&feature_set::enable_accumulator_sysvar::id()), + true + ); + + // The current sequence value will be used in the message when the bank advances, so we snapshot + // it here before advancing the slot so we can assert the correct sequence is present in the message. + let sequence_tracker_before_bank_advance = get_acc_sequence_tracker(&bank); + bank = new_from_parent(&Arc::new(bank)); + + // get the timestamp & slot for the message + let ring_index = (bank.slot() % ACCUMULATOR_RING_SIZE as u64) as u32; + let wormhole_message_account = get_wormhole_message_account(&bank, ring_index); + + assert_ne!(wormhole_message_account.data().len(), 0); + + let wormhole_message = + PostedMessageUnreliableData::deserialize(&mut wormhole_message_account.data()).unwrap(); + + let messages = messages.iter().map(|m| m.as_slice()).collect::>(); + let accumulator_elements = messages.clone().into_iter().sorted_unstable().dedup(); + let expected_accumulator = + MerkleAccumulator::::from_set(accumulator_elements).unwrap(); + let expected_wormhole_message_payload = + expected_accumulator.serialize(bank.slot(), ACCUMULATOR_RING_SIZE); + assert_eq!( + wormhole_message.message.payload, + expected_wormhole_message_payload + ); + + let expected_wormhole_message = PostedMessageUnreliableData { + message: MessageData { + vaa_version: 1, + consistency_level: 1, + submission_time: bank.clock().unix_timestamp as u32, + sequence: sequence_tracker_before_bank_advance.sequence, // sequence is incremented after the message is processed + emitter_chain: 26, + emitter_address: ACCUMULATOR_EMITTER_ADDRESS, + payload: expected_wormhole_message_payload, + ..Default::default() + }, + }; + + assert_eq!( + wormhole_message_account.data().to_vec(), + expected_wormhole_message.try_to_vec().unwrap() + ); + + // verify hashes verify in accumulator + for msg in messages { + let msg_hash = Keccak160::hashv(&[[0u8].as_ref(), msg]); + let msg_proof = expected_accumulator.prove(msg).unwrap(); + + assert!(expected_accumulator.nodes.contains(&msg_hash)); + assert!(expected_accumulator.check(msg_proof, msg)); + } + + // verify accumulator state account + let accumulator_state = get_accumulator_state(&bank, ring_index); + let acc_state_magic = &accumulator_state[..4]; + let acc_state_slot = LittleEndian::read_u64(&accumulator_state[4..12]); + let acc_state_ring_size = LittleEndian::read_u32(&accumulator_state[12..16]); + + assert_eq!(acc_state_magic, b"PAS1"); + assert_eq!(acc_state_slot, bank.slot()); + assert_eq!(acc_state_ring_size, ACCUMULATOR_RING_SIZE); + + let mut cursor = std::io::Cursor::new(&accumulator_state[16..]); + let num_elems = cursor.read_u32::().unwrap(); + for _ in 0..(num_elems as usize) { + let element_len = cursor.read_u32::().unwrap(); + let mut element_data = vec![0u8; element_len as usize]; + cursor.read_exact(&mut element_data).unwrap(); + + let elem_hash = Keccak160::hashv(&[[0u8].as_ref(), element_data.as_slice()]); + let elem_proof = expected_accumulator.prove(element_data.as_slice()).unwrap(); + + assert!(expected_accumulator.nodes.contains(&elem_hash)); + assert!(expected_accumulator.check(elem_proof, element_data.as_slice())); + } + + // verify sequence_tracker increments + assert_eq!( + get_acc_sequence_tracker(&bank).sequence, + sequence_tracker_before_bank_advance.sequence + 1 + ); + + // verify ring buffer cycles + let ring_index_before_buffer_cycle = (bank.slot() % ACCUMULATOR_RING_SIZE as u64) as u32; + let target_slot = bank.slot() + ACCUMULATOR_RING_SIZE as u64; + // advance ACCUMULATOR_RING_SIZE slots using warp_from_parent since doing large loops + // with new_from_parent takes a long time. warp_from_parent results in a bank that is frozen. + bank = Bank::warp_from_parent(&Arc::new(bank), &Pubkey::default(), target_slot); + + // accumulator messages should still be the same before looping around + let ring_index_after_buffer_cycle = (bank.slot() % ACCUMULATOR_RING_SIZE as u64) as u32; + assert_eq!( + ring_index_before_buffer_cycle, + ring_index_after_buffer_cycle + ); + + let accumulator_state_after_skip = get_accumulator_state(&bank, ring_index_after_buffer_cycle); + assert_eq!( + &accumulator_state[16..], + &accumulator_state_after_skip[16..] + ); + + // insert new message to make sure the update is written in the right position + // in the ring buffer and overwrites the existing message + + // advance the bank to unfreeze it (to be able to store accounts). see the comment on warp_from_parent above. + bank = new_from_parent(&Arc::new(bank)); + + let wh_sequence_before_acc_update = get_acc_sequence_tracker(&bank).sequence; + + let message_0 = vec![1u8; 127]; + let message_1 = vec![2u8; 127]; + let message_2 = vec![3u8; 254]; + + let updated_messages = vec![message_1.clone(), message_2.clone(), message_0.clone()]; + + let updated_message_buffer_bytes = create_message_buffer_bytes(updated_messages.clone()); + price_message_account.set_data(updated_message_buffer_bytes); + + // Store Message account so the accumulator sysvar updater can find it. + bank.store_account(&price_message_key.pubkey(), &price_message_account); + + // Run accumulator, update clock & other sysvars etc + bank = new_from_parent(&Arc::new(bank)); + + let ring_index = (bank.slot() % ACCUMULATOR_RING_SIZE as u64) as u32; + let updated_wormhole_message_account = get_wormhole_message_account(&bank, ring_index); + + let updated_wormhole_message = + PostedMessageUnreliableData::deserialize(&mut updated_wormhole_message_account.data()) + .unwrap(); + + let updated_messages = updated_messages + .iter() + .map(|m| m.as_slice()) + .collect::>(); + let updated_accumulator_elements = updated_messages + .clone() + .into_iter() + .sorted_unstable() + .dedup(); + + let expected_accumulator = + MerkleAccumulator::::from_set(updated_accumulator_elements).unwrap(); + assert_eq!( + updated_wormhole_message.message.payload, + expected_accumulator.serialize(bank.slot(), ACCUMULATOR_RING_SIZE) + ); + + let expected_wormhole_message = PostedMessageUnreliableData { + message: MessageData { + vaa_version: 1, + consistency_level: 1, + submission_time: bank.clock().unix_timestamp as u32, + sequence: wh_sequence_before_acc_update, + emitter_chain: 26, + emitter_address: ACCUMULATOR_EMITTER_ADDRESS, + payload: expected_accumulator.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), + ..Default::default() + }, + }; + + assert_eq!( + updated_wormhole_message_account.data(), + expected_wormhole_message.try_to_vec().unwrap() + ); + + // TODO: Should be done as additional tests. + // + // 1. Verify the accumulator state stays intact after the bank is advanced. + // done in this test but can be moved to a separate test. + // 2. Intentionally add corrupted accounts that do not appear in the accumulator. + // 3. Check if message offset is > message size to prevent validator crash. +} + +fn new_from_parent(parent: &Arc) -> Bank { + Bank::new_from_parent(parent, &Pubkey::default(), parent.slot() + 1) +} + +#[test] +fn test_update_accumulator_end_of_block() { + let leader_pubkey = solana_sdk::pubkey::new_rand(); + let GenesisConfigInfo { + mut genesis_config, .. + } = create_genesis_config_with_leader(5, &leader_pubkey, 3); + + // The genesis create function uses `Develompent` mode which enables all feature flags, so + // we need to remove the accumulator sysvar in order to test the validator behaves + // correctly when the feature is disabled. We will re-enable it further into this test. + genesis_config + .accounts + .remove(&feature_set::enable_accumulator_sysvar::id()) + .unwrap(); + genesis_config + .accounts + .remove(&feature_set::move_accumulator_to_end_of_block::id()) + .unwrap(); + + // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here + // due to slot 0 having special handling. + let slots_in_epoch = 32; + genesis_config.epoch_schedule = EpochSchedule::new(slots_in_epoch); + let mut bank = Bank::new_for_tests(&genesis_config); + bank = new_from_parent(&Arc::new(bank)); + bank = new_from_parent(&Arc::new(bank)); + + let message_0 = vec![1u8; 127]; + let message_1 = vec![2u8; 127]; + // insert into message buffer in reverse order to test that accumulator + // sorts first + let messages = vec![message_1, message_0]; + + let message_buffer_bytes = create_message_buffer_bytes(messages.clone()); + + // Create a Message account. + let price_message_key = keypair_from_seed(&[1u8; 32]).unwrap(); + let mut price_message_account = bank + .get_account(&price_message_key.pubkey()) + .unwrap_or_default(); + price_message_account.set_lamports(1_000_000_000); + price_message_account.set_owner(Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID)); + price_message_account.set_data(message_buffer_bytes); + + // Store Message account so the accumulator sysvar updater can find it. + bank.store_account(&price_message_key.pubkey(), &price_message_account); + + // Derive the Wormhole Message Account that will be generated by the sysvar updater. + let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( + &[b"AccumulatorMessage", &(bank.slot() as u32).to_be_bytes()], + &Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), + ); + + // Account Data should be empty at this point. Check account data is []. + let wormhole_message_account = bank + .get_account(&wormhole_message_pubkey) + .unwrap_or_default(); + assert_eq!(wormhole_message_account.data().len(), 0); + + // Run accumulator by creating a new bank from parent, the feature is + // disabled so account data should still be empty. Check account data is + // still []. + bank = new_from_parent(&Arc::new(bank)); + + assert_eq!( + bank.feature_set + .is_active(&feature_set::enable_accumulator_sysvar::id()), + false + ); + assert_eq!( + bank.feature_set + .is_active(&feature_set::move_accumulator_to_end_of_block::id()), + false + ); + + let wormhole_message_account = bank + .get_account(&wormhole_message_pubkey) + .unwrap_or_default(); + assert_eq!(wormhole_message_account.data().len(), 0); + + // Enable Accumulator Features (42 = random lamport balance, and the meaning of the universe). + let feature_id = feature_set::enable_accumulator_sysvar::id(); + let feature = Feature { + activated_at: Some(30), + }; + bank.store_account(&feature_id, &feature::create_account(&feature, 42)); + + let feature_id = feature_set::move_accumulator_to_end_of_block::id(); + let feature = Feature { + activated_at: Some(30), + }; + bank.store_account(&feature_id, &feature::create_account(&feature, 42)); + + bank.compute_active_feature_set(true); + for _ in 0..slots_in_epoch { + bank = new_from_parent(&Arc::new(bank)); + } + + // Features should now be enabled on the new bank as the epoch has changed. + assert_eq!( + bank.feature_set + .is_active(&feature_set::enable_accumulator_sysvar::id()), + true + ); + assert_eq!( + bank.feature_set + .is_active(&feature_set::move_accumulator_to_end_of_block::id()), + true + ); + + // The current sequence value will be used in the message when the bank advances, so we snapshot + // it here before freezing the bank so we can assert the correct sequence is present in the message. + let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); + // Freeze the bank to make sure accumulator is updated + bank.freeze(); + + // get the timestamp & slot for the message + let ring_index = (bank.slot() % ACCUMULATOR_RING_SIZE as u64) as u32; + let wormhole_message_account = get_wormhole_message_account(&bank, ring_index); + + assert_ne!(wormhole_message_account.data().len(), 0); + + let wormhole_message = + PostedMessageUnreliableData::deserialize(&mut wormhole_message_account.data()).unwrap(); + + let messages = messages.iter().map(|m| m.as_slice()).collect::>(); + let accumulator_elements = messages.clone().into_iter().sorted_unstable().dedup(); + let expected_accumulator = + MerkleAccumulator::::from_set(accumulator_elements).unwrap(); + let expected_wormhole_message_payload = + expected_accumulator.serialize(bank.slot(), ACCUMULATOR_RING_SIZE); + assert_eq!( + wormhole_message.message.payload, + expected_wormhole_message_payload + ); + + let expected_wormhole_message = PostedMessageUnreliableData { + message: MessageData { + vaa_version: 1, + consistency_level: 1, + submission_time: bank.clock().unix_timestamp as u32, + sequence: sequence_tracker_before_bank_freeze.sequence, // sequence is incremented after the message is processed + emitter_chain: 26, + emitter_address: ACCUMULATOR_EMITTER_ADDRESS, + payload: expected_wormhole_message_payload, + ..Default::default() + }, + }; + + assert_eq!( + wormhole_message_account.data().to_vec(), + expected_wormhole_message.try_to_vec().unwrap() + ); + + // verify hashes verify in accumulator + for msg in messages { + let msg_hash = Keccak160::hashv(&[[0u8].as_ref(), msg]); + let msg_proof = expected_accumulator.prove(msg).unwrap(); + + assert!(expected_accumulator.nodes.contains(&msg_hash)); + assert!(expected_accumulator.check(msg_proof, msg)); + } + + // verify accumulator state account + let accumulator_state = get_accumulator_state(&bank, ring_index); + let acc_state_magic = &accumulator_state[..4]; + let acc_state_slot = LittleEndian::read_u64(&accumulator_state[4..12]); + let acc_state_ring_size = LittleEndian::read_u32(&accumulator_state[12..16]); + + assert_eq!(acc_state_magic, b"PAS1"); + assert_eq!(acc_state_slot, bank.slot()); + assert_eq!(acc_state_ring_size, ACCUMULATOR_RING_SIZE); + + let mut cursor = std::io::Cursor::new(&accumulator_state[16..]); + let num_elems = cursor.read_u32::().unwrap(); + for _ in 0..(num_elems as usize) { + let element_len = cursor.read_u32::().unwrap(); + let mut element_data = vec![0u8; element_len as usize]; + cursor.read_exact(&mut element_data).unwrap(); + + let elem_hash = Keccak160::hashv(&[[0u8].as_ref(), element_data.as_slice()]); + let elem_proof = expected_accumulator.prove(element_data.as_slice()).unwrap(); + + assert!(expected_accumulator.nodes.contains(&elem_hash)); + assert!(expected_accumulator.check(elem_proof, element_data.as_slice())); + } + + // verify sequence_tracker increments + assert_eq!( + get_acc_sequence_tracker(&bank).sequence, + sequence_tracker_before_bank_freeze.sequence + 1 + ); + + // verify ring buffer cycles + let ring_index_before_buffer_cycle = (bank.slot() % ACCUMULATOR_RING_SIZE as u64) as u32; + let target_slot = bank.slot() + ACCUMULATOR_RING_SIZE as u64; + // advance ACCUMULATOR_RING_SIZE slots using warp_from_parent since doing large loops + // with new_from_parent takes a long time. warp_from_parent results in a bank that is frozen. + bank = Bank::warp_from_parent(&Arc::new(bank), &Pubkey::default(), target_slot); + + // accumulator messages should still be the same before looping around + let ring_index_after_buffer_cycle = (bank.slot() % ACCUMULATOR_RING_SIZE as u64) as u32; + assert_eq!( + ring_index_before_buffer_cycle, + ring_index_after_buffer_cycle + ); + + let accumulator_state_after_skip = get_accumulator_state(&bank, ring_index_after_buffer_cycle); + assert_eq!( + &accumulator_state[16..], + &accumulator_state_after_skip[16..] + ); + + // insert new message to make sure the update is written in the right position + // in the ring buffer and overwrites the existing message + + // advance the bank to unfreeze it (to be able to store accounts). see the comment on warp_from_parent above. + bank = new_from_parent(&Arc::new(bank)); + + let wh_sequence_before_acc_update = get_acc_sequence_tracker(&bank).sequence; + + let message_0 = vec![1u8; 127]; + let message_1 = vec![2u8; 127]; + let message_2 = vec![3u8; 254]; + + let updated_messages = vec![message_1.clone(), message_2.clone(), message_0.clone()]; + + let updated_message_buffer_bytes = create_message_buffer_bytes(updated_messages.clone()); + price_message_account.set_data(updated_message_buffer_bytes); + + // Store Message account so the accumulator sysvar updater can find it. + bank.store_account(&price_message_key.pubkey(), &price_message_account); + + // Freeze the bank to run accumulator + bank.freeze(); + + let ring_index = (bank.slot() % ACCUMULATOR_RING_SIZE as u64) as u32; + let updated_wormhole_message_account = get_wormhole_message_account(&bank, ring_index); + + let updated_wormhole_message = + PostedMessageUnreliableData::deserialize(&mut updated_wormhole_message_account.data()) + .unwrap(); + + let updated_messages = updated_messages + .iter() + .map(|m| m.as_slice()) + .collect::>(); + let updated_accumulator_elements = updated_messages + .clone() + .into_iter() + .sorted_unstable() + .dedup(); + + let expected_accumulator = + MerkleAccumulator::::from_set(updated_accumulator_elements).unwrap(); + assert_eq!( + updated_wormhole_message.message.payload, + expected_accumulator.serialize(bank.slot(), ACCUMULATOR_RING_SIZE) + ); + + let expected_wormhole_message = PostedMessageUnreliableData { + message: MessageData { + vaa_version: 1, + consistency_level: 1, + submission_time: bank.clock().unix_timestamp as u32, + sequence: wh_sequence_before_acc_update, + emitter_chain: 26, + emitter_address: ACCUMULATOR_EMITTER_ADDRESS, + payload: expected_accumulator.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), + ..Default::default() + }, + }; + + assert_eq!( + updated_wormhole_message_account.data(), + expected_wormhole_message.try_to_vec().unwrap() + ); +} + +#[test] +fn test_get_accumulator_keys() { + use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; + let leader_pubkey = solana_sdk::pubkey::new_rand(); + let accumulator_keys: Vec = get_accumulator_keys() + .iter() + .map(|(_, pk_res)| *pk_res.as_ref().unwrap()) + .collect(); + let expected_pyth_keys = vec![ + Pubkey::new_from_array(MESSAGE_BUFFER_PID), + Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), + Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), + Pubkey::new_from_array(pythnet::WORMHOLE_PID), + ]; + assert_eq!(accumulator_keys, expected_pyth_keys); +} From da4eb87c7e73a7a360989ff3d5c67d9e1f95f913 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 12 Jul 2024 12:20:16 +0100 Subject: [PATCH 10/46] test: fix old pyth tests --- Cargo.lock | 2 +- runtime/src/bank/pyth_accumulator.rs | 32 ++++++--------- runtime/src/bank/pyth_accumulator_tests.rs | 46 ++++++++++++++++++++-- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f747cac27e..566276a61a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3556,7 +3556,7 @@ dependencies = [ [[package]] name = "pyth-oracle" version = "2.26.0" -source = "git+https://github.com/pyth-network/pyth-client?branch=accumulator-v2#f0ac2678d738d659ea4ef94f1db81e7d7e439183" +source = "git+https://github.com/pyth-network/pyth-client?branch=accumulator-v2#a59effe7d77cfe88488fd3c4c34f3d5e33719f96" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 17ecb155fd..b4ba93ccde 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -20,13 +20,14 @@ use { pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; lazy_static! { - static ref ORACLE_PUBKEY: Option = match env::var("PYTH_ORACLE_PUBKEY") { - Ok(value) => Some( - value - .parse() - .expect("invalid value of PYTH_ORACLE_PUBKEY env var") - ), - Err(VarError::NotPresent) => None, + pub static ref ORACLE_PUBKEY: Pubkey = match env::var("PYTH_ORACLE_PUBKEY") { + Ok(value) => value + .parse() + .expect("invalid value of PYTH_ORACLE_PUBKEY env var"), + Err(VarError::NotPresent) => { + // Pythnet oracle program address + "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH".parse().unwrap() + } Err(VarError::NotUnicode(err)) => { panic!("invalid value of PYTH_ORACLE_PUBKEY env var: {err:?}"); } @@ -51,9 +52,6 @@ pub enum AccumulatorUpdateErrorV1 { #[error("could not parse Pubkey from environment")] InvalidEnvPubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), - - #[error("no oracle pubkey")] - NoOraclePubkey, } /// Updates the Accumulator Sysvar at the start of a new slot. See `update_clock` to see a similar @@ -376,10 +374,8 @@ fn post_accumulator_attestation( } pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - let oracle_pubkey = ORACLE_PUBKEY.ok_or(AccumulatorUpdateErrorV1::NoOraclePubkey)?; - let accounts = bank - .get_program_accounts(&oracle_pubkey, &ScanConfig::new(true)) + .get_program_accounts(&ORACLE_PUBKEY, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; let mut any_v1_aggregations = false; @@ -395,12 +391,10 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV &pubkey.to_bytes().into(), &mut price_account_data, ) { - Ok(outcome) => { - if outcome.commit { - account.set_data(price_account_data); - bank.store_account_and_update_capitalization(&pubkey, &account); - } - v2_messages.extend(outcome.messages); + Ok(messages) => { + account.set_data(price_account_data); + bank.store_account_and_update_capitalization(&pubkey, &account); + v2_messages.extend(messages); } Err(err) => match err { AggregationError::NotPriceFeedAccount => {} diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 53af505c12..8a03a0d833 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -1,6 +1,6 @@ use crate::{ bank::{ - pyth_accumulator::{get_accumulator_keys, ACCUMULATOR_RING_SIZE}, + pyth_accumulator::{get_accumulator_keys, ACCUMULATOR_RING_SIZE, ORACLE_PUBKEY}, Bank, }, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, @@ -8,6 +8,8 @@ use crate::{ use byteorder::ByteOrder; use byteorder::{LittleEndian, ReadBytesExt}; use itertools::Itertools; +use pyth_oracle::solana_program::account_info::AccountInfo; +use pyth_oracle::{PriceAccount, PythAccount}; use pythnet_sdk::{ accumulators::{merkle::MerkleAccumulator, Accumulator}, hashers::{keccak256_160::Keccak160, Hasher}, @@ -17,6 +19,7 @@ use pythnet_sdk::{ use solana_sdk::{ account::{AccountSharedData, ReadableAccount, WritableAccount}, borsh::{BorshDeserialize, BorshSerialize}, + clock::Epoch, epoch_schedule::EpochSchedule, feature::{self, Feature}, feature_set, @@ -25,7 +28,7 @@ use solana_sdk::{ signature::keypair_from_seed, signer::Signer, }; -use std::{io::Read, sync::Arc}; +use std::{io::Read, mem::size_of, sync::Arc}; // Create Message Account Bytes // @@ -125,6 +128,25 @@ fn test_update_accumulator_sysvar() { // Store Message account so the accumulator sysvar updater can find it. bank.store_account(&price_message_key.pubkey(), &price_message_account); + let (price_feed_key, _bump) = Pubkey::find_program_address(&[b"123"], &ORACLE_PUBKEY); + let mut price_feed_account = + AccountSharedData::new(42, size_of::(), &ORACLE_PUBKEY); + PriceAccount::initialize( + &AccountInfo::new( + &price_feed_key.to_bytes().into(), + false, + true, + &mut 0, + &mut price_feed_account.data_mut(), + &ORACLE_PUBKEY.to_bytes().into(), + false, + Epoch::default(), + ), + 0, + ) + .unwrap(); + bank.store_account(&price_feed_key, &price_feed_account); + // Derive the Wormhole Message Account that will be generated by the sysvar updater. let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( &[b"AccumulatorMessage", &(bank.slot() as u32).to_be_bytes()], @@ -396,6 +418,25 @@ fn test_update_accumulator_end_of_block() { // Store Message account so the accumulator sysvar updater can find it. bank.store_account(&price_message_key.pubkey(), &price_message_account); + let (price_feed_key, _bump) = Pubkey::find_program_address(&[b"123"], &ORACLE_PUBKEY); + let mut price_feed_account = + AccountSharedData::new(42, size_of::(), &ORACLE_PUBKEY); + PriceAccount::initialize( + &AccountInfo::new( + &price_feed_key.to_bytes().into(), + false, + true, + &mut 0, + &mut price_feed_account.data_mut(), + &ORACLE_PUBKEY.to_bytes().into(), + false, + Epoch::default(), + ), + 0, + ) + .unwrap(); + bank.store_account(&price_feed_key, &price_feed_account); + // Derive the Wormhole Message Account that will be generated by the sysvar updater. let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( &[b"AccumulatorMessage", &(bank.slot() as u32).to_be_bytes()], @@ -631,7 +672,6 @@ fn test_update_accumulator_end_of_block() { #[test] fn test_get_accumulator_keys() { use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; - let leader_pubkey = solana_sdk::pubkey::new_rand(); let accumulator_keys: Vec = get_accumulator_keys() .iter() .map(|(_, pk_res)| *pk_res.as_ref().unwrap()) From f3deb0f6e4f78bee8df7411f2ebecca363f6bf46 Mon Sep 17 00:00:00 2001 From: Reisen Date: Tue, 16 Jul 2024 15:53:52 +0000 Subject: [PATCH 11/46] test: add initial v2 tests to validator --- Cargo.lock | 1 - runtime/Cargo.toml | 2 +- runtime/src/bank/pyth_accumulator_tests.rs | 201 ++++++++++++++++++++- 3 files changed, 200 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 566276a61a..455a14336e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3556,7 +3556,6 @@ dependencies = [ [[package]] name = "pyth-oracle" version = "2.26.0" -source = "git+https://github.com/pyth-network/pyth-client?branch=accumulator-v2#a59effe7d77cfe88488fd3c4c34f3d5e33719f96" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index b83f212d9f..450f6c1188 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -35,7 +35,7 @@ num-traits = { version = "0.2" } num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" -pyth-oracle = { git = "https://github.com/pyth-network/pyth-client", branch = "accumulator-v2", features = ["library"] } +pyth-oracle = { path = "/code/pyth-network/pyth-client/program/rust", features = ["library"] } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "e670f57f89b05398ca352e4accb1e32724a8e1b4" } rand = "0.7.0" rayon = "1.5.3" diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 8a03a0d833..d24d4f37a4 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -8,7 +8,8 @@ use crate::{ use byteorder::ByteOrder; use byteorder::{LittleEndian, ReadBytesExt}; use itertools::Itertools; -use pyth_oracle::solana_program::account_info::AccountInfo; +use pyth_oracle::PythOracleSerialize; +use pyth_oracle::{solana_program::account_info::AccountInfo, PriceAccountFlags}; use pyth_oracle::{PriceAccount, PythAccount}; use pythnet_sdk::{ accumulators::{merkle::MerkleAccumulator, Accumulator}, @@ -131,7 +132,7 @@ fn test_update_accumulator_sysvar() { let (price_feed_key, _bump) = Pubkey::find_program_address(&[b"123"], &ORACLE_PUBKEY); let mut price_feed_account = AccountSharedData::new(42, size_of::(), &ORACLE_PUBKEY); - PriceAccount::initialize( + let _ = PriceAccount::initialize( &AccountInfo::new( &price_feed_key.to_bytes().into(), false, @@ -669,6 +670,202 @@ fn test_update_accumulator_end_of_block() { ); } +// This test will +#[test] +fn test_accumulator_v2() { + let leader_pubkey = solana_sdk::pubkey::new_rand(); + let GenesisConfigInfo { + mut genesis_config, .. + } = create_genesis_config_with_leader(5, &leader_pubkey, 3); + + // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here + // due to slot 0 having special handling. + let slots_in_epoch = 32; + genesis_config.epoch_schedule = EpochSchedule::new(slots_in_epoch); + let mut bank = Bank::new_for_tests(&genesis_config); + + bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. + bank = new_from_parent(&Arc::new(bank)); // Advance slot 2. + + let generate_price = |seeds, generate_buffers: bool| { + let (price_feed_key, _bump) = Pubkey::find_program_address(&[seeds], &ORACLE_PUBKEY); + let mut price_feed_account = + AccountSharedData::new(42, size_of::(), &ORACLE_PUBKEY); + + let messages = { + let price_feed_info_key = &price_feed_key.to_bytes().into(); + let price_feed_info_lamports = &mut 0; + let price_feed_info_owner = &ORACLE_PUBKEY.to_bytes().into(); + let price_feed_info_data = price_feed_account.data_mut(); + let price_feed_info = AccountInfo::new( + price_feed_info_key, + false, + true, + price_feed_info_lamports, + price_feed_info_data, + price_feed_info_owner, + false, + Epoch::default(), + ); + + let mut price_account = PriceAccount::initialize(&price_feed_info, 0).unwrap(); + if !generate_buffers { + price_account.flags.insert( + PriceAccountFlags::ACCUMULATOR_V2 | PriceAccountFlags::MESSAGE_BUFFER_CLEARED, + ); + } + + vec![ + price_account + .as_price_feed_message(&price_feed_key.to_bytes().into()) + .to_bytes(), + price_account + .as_twap_message(&price_feed_key.to_bytes().into()) + .to_bytes(), + ] + }; + + bank.store_account(&price_feed_key, &price_feed_account); + + if generate_buffers { + // Insert into message buffer in reverse order to test that accumulator + // sorts first. + let message_buffer_bytes = create_message_buffer_bytes(messages.clone()); + + // Create a Message account. + let price_message_key = keypair_from_seed(&[1u8; 32]).unwrap(); + let mut price_message_account = bank + .get_account(&price_message_key.pubkey()) + .unwrap_or_default(); + + price_message_account.set_lamports(1_000_000_000); + price_message_account + .set_owner(Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID)); + price_message_account.set_data(message_buffer_bytes); + + // Store Message account so the accumulator sysvar updater can find it. + bank.store_account(&price_message_key.pubkey(), &price_message_account); + } + + (price_feed_key, messages) + }; + + // TODO: New test functionality here. + // 1. Create Price Feed Accounts owned by ORACLE_PUBKEY + // 2. Populate Price Feed Accounts + // 3. Call update_v2() + // - Cases: + // - No V1 Messages, Only Price Accounts with no V2 + // - No V1 Messages, Some Price Accounts with no V2 + // - Some V1 Messages, No Price Accounts with no V2 + // - Some V1 Messages, Some Price Accounts with no V2 + // - Simulate PriceUpdate that WOULD trigger a real V1 aggregate before End of Slot + // - Simulate PriceUpdate that doesn't trigger a real V1 aggregate, only V2. + + assert!(bank + .feature_set + .is_active(&feature_set::enable_accumulator_sysvar::id())); + assert!(bank + .feature_set + .is_active(&feature_set::move_accumulator_to_end_of_block::id())); + assert!(bank + .feature_set + .is_active(&feature_set::undo_move_accumulator_to_end_of_block::id())); + assert!(bank + .feature_set + .is_active(&feature_set::redo_move_accumulator_to_end_of_block::id())); + + let prices_with_messages = [ + generate_price(b"seeds_1", false), + generate_price(b"seeds_2", false), + generate_price(b"seeds_3", false), + generate_price(b"seeds_4", false), + ]; + + let messages = prices_with_messages + .iter() + .map(|(_, messages)| messages) + .flatten() + .map(|message| &message[..]); + + // Trigger Aggregation. We freeze instead of new_from_parent so + // we can keep access to the bank. + let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); + bank.freeze(); + + // Get the wormhole message generated by freezed. We don't need + // to offset the ring index as our test is always below 10K slots. + let wormhole_message_account = get_wormhole_message_account(&bank, bank.slot() as u32); + assert_ne!(wormhole_message_account.data().len(), 0); + PostedMessageUnreliableData::deserialize(&mut wormhole_message_account.data()).unwrap(); + + // Create MerkleAccumulator by hand to verify that the Wormhole message + // contents are correctg. + let expected_accumulator = + MerkleAccumulator::::from_set(messages.clone().sorted_unstable().dedup()) + .unwrap(); + + let expected_wormhole_message_payload = + expected_accumulator.serialize(bank.slot(), ACCUMULATOR_RING_SIZE); + + let expected_wormhole_message = PostedMessageUnreliableData { + message: MessageData { + vaa_version: 1, + consistency_level: 1, + submission_time: bank.clock().unix_timestamp as u32, + sequence: sequence_tracker_before_bank_freeze.sequence, // sequence is incremented after the message is processed + emitter_chain: 26, + emitter_address: ACCUMULATOR_EMITTER_ADDRESS, + payload: expected_wormhole_message_payload, + ..Default::default() + }, + }; + + assert_eq!( + wormhole_message_account.data().to_vec(), + expected_wormhole_message.try_to_vec().unwrap() + ); + + // Verify hashes in accumulator. + for msg in messages { + let msg_hash = Keccak160::hashv(&[[0u8].as_ref(), msg]); + let msg_proof = expected_accumulator.prove(msg).unwrap(); + assert!(expected_accumulator.nodes.contains(&msg_hash)); + assert!(expected_accumulator.check(msg_proof, msg)); + } + + // Verify accumulator state account. + let accumulator_state = get_accumulator_state(&bank, bank.slot() as u32); + let acc_state_magic = &accumulator_state[..4]; + let acc_state_slot = LittleEndian::read_u64(&accumulator_state[4..12]); + let acc_state_ring_size = LittleEndian::read_u32(&accumulator_state[12..16]); + + assert_eq!(acc_state_magic, b"PAS1"); + assert_eq!(acc_state_slot, bank.slot()); + assert_eq!(acc_state_ring_size, ACCUMULATOR_RING_SIZE); + + // Verify the messages within the accumulator state account + // were in the accumulator as well. + let mut cursor = std::io::Cursor::new(&accumulator_state[16..]); + let num_elems = cursor.read_u32::().unwrap(); + for _ in 0..(num_elems as usize) { + let element_len = cursor.read_u32::().unwrap(); + let mut element_data = vec![0u8; element_len as usize]; + cursor.read_exact(&mut element_data).unwrap(); + + let elem_hash = Keccak160::hashv(&[[0u8].as_ref(), element_data.as_slice()]); + let elem_proof = expected_accumulator.prove(element_data.as_slice()).unwrap(); + + assert!(expected_accumulator.nodes.contains(&elem_hash)); + assert!(expected_accumulator.check(elem_proof, element_data.as_slice())); + } + + // Verify sequence_tracker increments for wormhole to accept it. + assert_eq!( + get_acc_sequence_tracker(&bank).sequence, + sequence_tracker_before_bank_freeze.sequence + 1 + ); +} #[test] fn test_get_accumulator_keys() { use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; From 5f9e29f2a444ad5b6d828139b2f418a7ceeb54e7 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Thu, 18 Jul 2024 09:54:06 +0100 Subject: [PATCH 12/46] refactor: unify env var handling --- runtime/src/bank/pyth_accumulator.rs | 155 ++++++++------------- runtime/src/bank/pyth_accumulator_tests.rs | 22 ++- 2 files changed, 68 insertions(+), 109 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index b4ba93ccde..4bfd8eac14 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -2,36 +2,49 @@ use { super::Bank, crate::accounts_index::{ScanConfig, ScanError}, byteorder::LittleEndian, + byteorder::ReadBytesExt, itertools::Itertools, log::*, pyth_oracle::validator::AggregationError, + pythnet_sdk::{ + accumulators::{merkle::MerkleAccumulator, Accumulator}, + hashers::keccak256_160::Keccak160, + wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, + }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, - feature_set, + borsh, feature_set, hash::hashv, pubkey::Pubkey, }, - std::{ - env::{self, VarError}, - str::FromStr, - }, + std::env::{self, VarError}, }; pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; lazy_static! { - pub static ref ORACLE_PUBKEY: Pubkey = match env::var("PYTH_ORACLE_PUBKEY") { - Ok(value) => value + pub static ref MESSAGE_BUFFER_PID: Pubkey = env_pubkey_or( + "MESSAGE_BUFFER_PID", + Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID), + ); + pub static ref ACCUMULATOR_EMITTER_ADDR: Pubkey = env_pubkey_or( + "ACCUMULATOR_EMITTER_ADDR", + Pubkey::new_from_array(pythnet_sdk::ACCUMULATOR_EMITTER_ADDRESS), + ); + pub static ref ACCUMULATOR_SEQUENCE_ADDR: Pubkey = env_pubkey_or( + "ACCUMULATOR_SEQUENCE_ADDR", + Pubkey::new_from_array(pythnet_sdk::pythnet::ACCUMULATOR_SEQUENCE_ADDR), + ); + pub static ref WORMHOLE_PID: Pubkey = env_pubkey_or( + "WORMHOLE_PID", + Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), + ); + pub static ref ORACLE_PID: Pubkey = env_pubkey_or( + "ORACLE_PID", + "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH" .parse() - .expect("invalid value of PYTH_ORACLE_PUBKEY env var"), - Err(VarError::NotPresent) => { - // Pythnet oracle program address - "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH".parse().unwrap() - } - Err(VarError::NotUnicode(err)) => { - panic!("invalid value of PYTH_ORACLE_PUBKEY env var: {err:?}"); - } - }; + .unwrap(), + ); } /// Accumulator specific error type. It would be nice to use `transaction::Error` but it does @@ -82,16 +95,19 @@ pub fn update_accumulator(bank: &Bank) { /// Read the pubkey from the environment variable `var` or return `default` /// if the variable is not set. -fn env_pubkey_or( - var: &str, - default: Pubkey, -) -> std::result::Result { - Ok(std::env::var(var) - .as_deref() - .map(Pubkey::from_str) - .ok() - .transpose()? - .unwrap_or(default)) +fn env_pubkey_or(var: &str, default: Pubkey) -> Pubkey { + match env::var(var) { + Ok(value) => value.parse().unwrap_or_else(|err| { + panic!( + "failed to parse env var {}={:?} as pubkey: {}", + var, value, err + ) + }), + Err(VarError::NotPresent) => default, + Err(VarError::NotUnicode(value)) => { + panic!("invalid value of env var {}={:?}: not unicode", var, value); + } + } } /// Get all accumulator related pubkeys from environment variables @@ -100,33 +116,13 @@ pub fn get_accumulator_keys() -> Vec<( &'static str, std::result::Result, )> { - use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; - let accumulator_keys = vec![ - ( - "MESSAGE_BUFFER_PID", - Pubkey::new_from_array(MESSAGE_BUFFER_PID), - ), - // accumulator emitter address should always be the same regardless - // of environment but checking here for completeness - ( - "ACCUMULATOR_EMITTER_ADDR", - Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), - ), - ( - "ACCUMULATOR_SEQUENCE_ADDR", - Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), - ), - ( - "WORMHOLE_PID", - Pubkey::new_from_array(pythnet::WORMHOLE_PID), - ), - ]; - let accumulator_pubkeys: Vec<(&str, std::result::Result)> = - accumulator_keys - .iter() - .map(|(k, d)| (*k, env_pubkey_or(k, *d))) - .collect(); - accumulator_pubkeys + vec![ + ("MESSAGE_BUFFER_PID", Ok(*MESSAGE_BUFFER_PID)), + ("ACCUMULATOR_EMITTER_ADDR", Ok(*ACCUMULATOR_EMITTER_ADDR)), + ("ACCUMULATOR_SEQUENCE_ADDR", Ok(*ACCUMULATOR_SEQUENCE_ADDR)), + ("WORMHOLE_PID", Ok(*WORMHOLE_PID)), + ("ORACLE_PID", Ok(*ORACLE_PID)), + ] } pub fn update_v1<'a>( @@ -134,30 +130,16 @@ pub fn update_v1<'a>( v2_messages: &[Vec], use_message_buffers: bool, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - use { - byteorder::ReadBytesExt, - pythnet_sdk::{ - accumulators::{merkle::MerkleAccumulator, Accumulator}, - hashers::keccak256_160::Keccak160, - MESSAGE_BUFFER_PID, - }, - solana_sdk::borsh, - }; - // Use the current Clock to determine the index into the accumulator ring buffer. let ring_index = (bank.slot() % 10_000) as u32; // Find all accounts owned by the Message Buffer program using get_program_accounts, and // extract the account data. - let message_buffer_pid = env_pubkey_or( - "MESSAGE_BUFFER_PID", - Pubkey::new_from_array(MESSAGE_BUFFER_PID), - )?; let message_buffer_accounts; let v1_messages = if use_message_buffers { message_buffer_accounts = bank - .get_program_accounts(&message_buffer_pid, &ScanConfig::new(true)) + .get_program_accounts(&MESSAGE_BUFFER_PID, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; let preimage = b"account:MessageBuffer"; @@ -266,28 +248,12 @@ fn post_accumulator_attestation( >, ring_index: u32, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - use pythnet_sdk::{ - pythnet, - wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, - ACCUMULATOR_EMITTER_ADDRESS, - }; - - let accumulator_sequence_addr = env_pubkey_or( - "ACCUMULATOR_SEQUENCE_ADDR", - Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), - )?; - - let accumulator_emitter_addr = env_pubkey_or( - "ACCUMULATOR_EMITTER_ADDR", - Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), - )?; - // Wormhole uses a Sequence account that is incremented each time a message is posted. As // we aren't calling Wormhole we need to bump this ourselves. If it doesn't exist, we just // create it instead. let mut sequence: AccumulatorSequenceTracker = { let data = bank - .get_account_with_fixed_root(&accumulator_sequence_addr) + .get_account_with_fixed_root(&ACCUMULATOR_SEQUENCE_ADDR) .unwrap_or_default(); let data = data.data(); solana_sdk::borsh::try_from_slice_unchecked(data) @@ -311,7 +277,7 @@ fn post_accumulator_attestation( nonce: 0, sequence: sequence.sequence, emitter_chain: 26, - emitter_address: accumulator_emitter_addr.to_bytes(), + emitter_address: ACCUMULATOR_EMITTER_ADDR.to_bytes(), payload: acc.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), } } else { @@ -322,7 +288,7 @@ fn post_accumulator_attestation( submission_time: bank.clock().unix_timestamp as u32, sequence: sequence.sequence, emitter_chain: 26, - emitter_address: accumulator_emitter_addr.to_bytes(), + emitter_address: ACCUMULATOR_EMITTER_ADDR.to_bytes(), payload: acc.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), ..Default::default() } @@ -330,18 +296,13 @@ fn post_accumulator_attestation( }; debug!("Accumulator: Wormhole message data: {:?}", message.message); - let wormhole_pid = env_pubkey_or( - "WORMHOLE_PID", - Pubkey::new_from_array(pythnet::WORMHOLE_PID), - )?; - // Now we can bump and write the Sequence account. sequence.sequence += 1; let sequence = solana_sdk::borsh::BorshSerialize::try_to_vec(&sequence) .map_err(|_| AccumulatorUpdateErrorV1::FailedSequenceSerialization)?; let sequence_balance = bank.get_minimum_balance_for_rent_exemption(sequence.len()); let sequence_account = { - let owner = &wormhole_pid; + let owner = &WORMHOLE_PID; let mut account = AccountSharedData::new(sequence_balance, sequence.len(), owner); account.set_data(sequence); account @@ -352,7 +313,7 @@ fn post_accumulator_attestation( .map_err(|_| AccumulatorUpdateErrorV1::FailedMessageSerialization)?; let message_balance = bank.get_minimum_balance_for_rent_exemption(message.len()); let message_account = { - let owner = &wormhole_pid; + let owner = &WORMHOLE_PID; let mut account = AccountSharedData::new(message_balance, message.len(), owner); account.set_data(message); account @@ -362,10 +323,10 @@ fn post_accumulator_attestation( // for RPC users to select the message for an associated VAA. let (message_pda, _) = Pubkey::find_program_address( &[b"AccumulatorMessage", &ring_index.to_be_bytes()], - &wormhole_pid, + &WORMHOLE_PID, ); - bank.store_account_and_update_capitalization(&accumulator_sequence_addr, &sequence_account); + bank.store_account_and_update_capitalization(&ACCUMULATOR_SEQUENCE_ADDR, &sequence_account); info!("Accumulator: Writing wormhole message to {:?}", message_pda); bank.store_account_and_update_capitalization(&message_pda, &message_account); @@ -375,7 +336,7 @@ fn post_accumulator_attestation( pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { let accounts = bank - .get_program_accounts(&ORACLE_PUBKEY, &ScanConfig::new(true)) + .get_program_accounts(&ORACLE_PID, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; let mut any_v1_aggregations = false; diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index d24d4f37a4..8528a584c5 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -1,6 +1,6 @@ use crate::{ bank::{ - pyth_accumulator::{get_accumulator_keys, ACCUMULATOR_RING_SIZE, ORACLE_PUBKEY}, + pyth_accumulator::{get_accumulator_keys, ACCUMULATOR_RING_SIZE, ORACLE_PID}, Bank, }, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, @@ -129,9 +129,8 @@ fn test_update_accumulator_sysvar() { // Store Message account so the accumulator sysvar updater can find it. bank.store_account(&price_message_key.pubkey(), &price_message_account); - let (price_feed_key, _bump) = Pubkey::find_program_address(&[b"123"], &ORACLE_PUBKEY); - let mut price_feed_account = - AccountSharedData::new(42, size_of::(), &ORACLE_PUBKEY); + let (price_feed_key, _bump) = Pubkey::find_program_address(&[b"123"], &ORACLE_PID); + let mut price_feed_account = AccountSharedData::new(42, size_of::(), &ORACLE_PID); let _ = PriceAccount::initialize( &AccountInfo::new( &price_feed_key.to_bytes().into(), @@ -139,7 +138,7 @@ fn test_update_accumulator_sysvar() { true, &mut 0, &mut price_feed_account.data_mut(), - &ORACLE_PUBKEY.to_bytes().into(), + &ORACLE_PID.to_bytes().into(), false, Epoch::default(), ), @@ -419,9 +418,8 @@ fn test_update_accumulator_end_of_block() { // Store Message account so the accumulator sysvar updater can find it. bank.store_account(&price_message_key.pubkey(), &price_message_account); - let (price_feed_key, _bump) = Pubkey::find_program_address(&[b"123"], &ORACLE_PUBKEY); - let mut price_feed_account = - AccountSharedData::new(42, size_of::(), &ORACLE_PUBKEY); + let (price_feed_key, _bump) = Pubkey::find_program_address(&[b"123"], &ORACLE_PID); + let mut price_feed_account = AccountSharedData::new(42, size_of::(), &ORACLE_PID); PriceAccount::initialize( &AccountInfo::new( &price_feed_key.to_bytes().into(), @@ -429,7 +427,7 @@ fn test_update_accumulator_end_of_block() { true, &mut 0, &mut price_feed_account.data_mut(), - &ORACLE_PUBKEY.to_bytes().into(), + &ORACLE_PID.to_bytes().into(), false, Epoch::default(), ), @@ -688,14 +686,14 @@ fn test_accumulator_v2() { bank = new_from_parent(&Arc::new(bank)); // Advance slot 2. let generate_price = |seeds, generate_buffers: bool| { - let (price_feed_key, _bump) = Pubkey::find_program_address(&[seeds], &ORACLE_PUBKEY); + let (price_feed_key, _bump) = Pubkey::find_program_address(&[seeds], &ORACLE_PID); let mut price_feed_account = - AccountSharedData::new(42, size_of::(), &ORACLE_PUBKEY); + AccountSharedData::new(42, size_of::(), &ORACLE_PID); let messages = { let price_feed_info_key = &price_feed_key.to_bytes().into(); let price_feed_info_lamports = &mut 0; - let price_feed_info_owner = &ORACLE_PUBKEY.to_bytes().into(); + let price_feed_info_owner = &ORACLE_PID.to_bytes().into(); let price_feed_info_data = price_feed_account.data_mut(); let price_feed_info = AccountInfo::new( price_feed_info_key, From 3b3bd3969c9accad5aaf7983711451647c7c6d4f Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 16 Jul 2024 22:14:58 +0100 Subject: [PATCH 13/46] go --- Cargo.lock | 121 ++++++++++++------ runtime/Cargo.toml | 2 +- runtime/src/bank/pyth_accumulator.rs | 176 ++++++++++++++++++--------- 3 files changed, 205 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 455a14336e..58d218ac19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -508,18 +508,41 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15bf3650200d8bffa99015595e10f1fbd17de07abbc25bb067da79e769939bfa" dependencies = [ - "borsh-derive", + "borsh-derive 0.9.3", "hashbrown 0.11.2", ] +[[package]] +name = "borsh" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" +dependencies = [ + "borsh-derive 0.10.3", + "hashbrown 0.12.3", +] + [[package]] name = "borsh-derive" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6441c552f230375d18e3cc377677914d2ca2b0d36e52129fe15450a2dce46775" dependencies = [ - "borsh-derive-internal", - "borsh-schema-derive-internal", + "borsh-derive-internal 0.9.3", + "borsh-schema-derive-internal 0.9.3", + "proc-macro-crate 0.1.5", + "proc-macro2 1.0.86", + "syn 1.0.109", +] + +[[package]] +name = "borsh-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" +dependencies = [ + "borsh-derive-internal 0.10.3", + "borsh-schema-derive-internal 0.10.3", "proc-macro-crate 0.1.5", "proc-macro2 1.0.86", "syn 1.0.109", @@ -536,6 +559,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "borsh-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" +dependencies = [ + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + [[package]] name = "borsh-schema-derive-internal" version = "0.9.3" @@ -547,6 +581,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "borsh-schema-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" +dependencies = [ + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + [[package]] name = "brotli" version = "3.3.4" @@ -582,19 +627,18 @@ checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" dependencies = [ "lazy_static", "memchr", - "regex-automata", + "regex-automata 0.1.10", "serde", ] [[package]] name = "bstr" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" dependencies = [ "memchr", - "once_cell", - "regex-automata", + "regex-automata 0.3.9", "serde", ] @@ -1320,9 +1364,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dyn-clone" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "eager" @@ -3556,6 +3600,7 @@ dependencies = [ [[package]] name = "pyth-oracle" version = "2.26.0" +source = "git+https://github.com/pyth-network/pyth-client.git?branch=feat/publisher-caps#170afdec9f5434deb3c242765ca409b180f306f7" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", @@ -3563,7 +3608,7 @@ dependencies = [ "byteorder", "num-derive", "num-traits", - "pythnet-sdk 1.13.6 (git+https://github.com/pyth-network/pyth-crosschain?rev=60144002053a93f424be70decd8a8ccb8d618d81)", + "pythnet-sdk 2.1.0", "solana-program 1.14.17", "solana-sdk 1.14.17", "thiserror", @@ -3572,37 +3617,37 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "1.13.6" -source = "git+https://github.com/pyth-network/pyth-crosschain?rev=60144002053a93f424be70decd8a8ccb8d618d81#60144002053a93f424be70decd8a8ccb8d618d81" +source = "git+https://github.com/pyth-network/pyth-crosschain?rev=e670f57f89b05398ca352e4accb1e32724a8e1b4#e670f57f89b05398ca352e4accb1e32724a8e1b4" dependencies = [ "bincode", - "borsh", + "borsh 0.9.3", "bytemuck", - "byteorder", "fast-math", "hex", "rustc_version 0.4.0", "serde", + "serde_wormhole", "sha3 0.10.8", "slow_primes", - "thiserror", + "wormhole-sdk", ] [[package]] name = "pythnet-sdk" -version = "1.13.6" -source = "git+https://github.com/pyth-network/pyth-crosschain?rev=e670f57f89b05398ca352e4accb1e32724a8e1b4#e670f57f89b05398ca352e4accb1e32724a8e1b4" +version = "2.1.0" +source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#3522f2940522e49aa56038e1000ea78f2e489d20" dependencies = [ "bincode", - "borsh", + "borsh 0.10.3", "bytemuck", + "byteorder", "fast-math", "hex", "rustc_version 0.4.0", "serde", - "serde_wormhole", "sha3 0.10.8", "slow_primes", - "wormhole-sdk", + "thiserror", ] [[package]] @@ -3931,6 +3976,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +[[package]] +name = "regex-automata" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" + [[package]] name = "regex-syntax" version = "0.6.26" @@ -4180,9 +4231,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" +checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" dependencies = [ "dyn-clone", "schemars_derive", @@ -4192,9 +4243,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" +checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", @@ -4732,7 +4783,7 @@ dependencies = [ name = "solana-banks-client" version = "1.14.173" dependencies = [ - "borsh", + "borsh 0.9.3", "futures 0.3.21", "solana-banks-interface", "solana-banks-server", @@ -5876,8 +5927,8 @@ dependencies = [ "bincode", "bitflags 1.3.2", "blake3", - "borsh", - "borsh-derive", + "borsh 0.9.3", + "borsh-derive 0.9.3", "bs58", "bv", "bytemuck", @@ -5925,8 +5976,8 @@ dependencies = [ "bincode", "bitflags 1.3.2", "blake3", - "borsh", - "borsh-derive", + "borsh 0.9.3", + "borsh-derive 0.9.3", "bs58", "bv", "bytemuck", @@ -6153,7 +6204,7 @@ dependencies = [ "once_cell", "ouroboros", "pyth-oracle", - "pythnet-sdk 1.13.6 (git+https://github.com/pyth-network/pyth-crosschain?rev=e670f57f89b05398ca352e4accb1e32724a8e1b4)", + "pythnet-sdk 1.13.6", "rand 0.7.3", "rand_chacha 0.2.2", "rayon", @@ -6198,7 +6249,7 @@ dependencies = [ "base64 0.13.0", "bincode", "bitflags 1.3.2", - "borsh", + "borsh 0.9.3", "bs58", "bytemuck", "byteorder", @@ -6248,7 +6299,7 @@ dependencies = [ "base64 0.13.0", "bincode", "bitflags 1.3.2", - "borsh", + "borsh 0.9.3", "bs58", "bytemuck", "byteorder", @@ -6560,7 +6611,7 @@ dependencies = [ "Inflector", "base64 0.13.0", "bincode", - "borsh", + "borsh 0.9.3", "bs58", "lazy_static", "log", @@ -6814,7 +6865,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbc000f0fdf1f12f99d77d398137c1751345b18c88258ce0f99b7872cf6c9bd6" dependencies = [ "assert_matches", - "borsh", + "borsh 0.9.3", "num-derive", "num-traits", "solana-program 1.14.17", @@ -8158,7 +8209,7 @@ version = "0.1.0" source = "git+https://github.com/wormhole-foundation/wormhole?tag=v2.17.1#3e423a75180f9da69263279e9ffce47b1858ae78" dependencies = [ "anyhow", - "bstr 1.4.0", + "bstr 1.6.0", "schemars", "serde", "serde_wormhole", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 450f6c1188..84f6d5caa5 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -35,7 +35,7 @@ num-traits = { version = "0.2" } num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" -pyth-oracle = { path = "/code/pyth-network/pyth-client/program/rust", features = ["library"] } +pyth-oracle = { git = "https://github.com/pyth-network/pyth-client.git" ,branch="feat/publisher-caps" , features = ["library"] } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "e670f57f89b05398ca352e4accb1e32724a8e1b4" } rand = "0.7.0" rayon = "1.5.3" diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 4bfd8eac14..6204430da1 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -2,49 +2,35 @@ use { super::Bank, crate::accounts_index::{ScanConfig, ScanError}, byteorder::LittleEndian, - byteorder::ReadBytesExt, itertools::Itertools, log::*, pyth_oracle::validator::AggregationError, - pythnet_sdk::{ - accumulators::{merkle::MerkleAccumulator, Accumulator}, - hashers::keccak256_160::Keccak160, - wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, - }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, - borsh, feature_set, + feature_set, hash::hashv, pubkey::Pubkey, }, - std::env::{self, VarError}, + std::{ + borrow::Borrow, env::{self, VarError}, str::FromStr + }, }; pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; lazy_static! { - pub static ref MESSAGE_BUFFER_PID: Pubkey = env_pubkey_or( - "MESSAGE_BUFFER_PID", - Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID), - ); - pub static ref ACCUMULATOR_EMITTER_ADDR: Pubkey = env_pubkey_or( - "ACCUMULATOR_EMITTER_ADDR", - Pubkey::new_from_array(pythnet_sdk::ACCUMULATOR_EMITTER_ADDRESS), - ); - pub static ref ACCUMULATOR_SEQUENCE_ADDR: Pubkey = env_pubkey_or( - "ACCUMULATOR_SEQUENCE_ADDR", - Pubkey::new_from_array(pythnet_sdk::pythnet::ACCUMULATOR_SEQUENCE_ADDR), - ); - pub static ref WORMHOLE_PID: Pubkey = env_pubkey_or( - "WORMHOLE_PID", - Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), - ); - pub static ref ORACLE_PID: Pubkey = env_pubkey_or( - "ORACLE_PID", - "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH" + pub static ref ORACLE_PUBKEY: Pubkey = match env::var("PYTH_ORACLE_PUBKEY") { + Ok(value) => value .parse() - .unwrap(), - ); + .expect("invalid value of PYTH_ORACLE_PUBKEY env var"), + Err(VarError::NotPresent) => { + // Pythnet oracle program address + "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH".parse().unwrap() + } + Err(VarError::NotUnicode(err)) => { + panic!("invalid value of PYTH_ORACLE_PUBKEY env var: {err:?}"); + } + }; } /// Accumulator specific error type. It would be nice to use `transaction::Error` but it does @@ -95,19 +81,16 @@ pub fn update_accumulator(bank: &Bank) { /// Read the pubkey from the environment variable `var` or return `default` /// if the variable is not set. -fn env_pubkey_or(var: &str, default: Pubkey) -> Pubkey { - match env::var(var) { - Ok(value) => value.parse().unwrap_or_else(|err| { - panic!( - "failed to parse env var {}={:?} as pubkey: {}", - var, value, err - ) - }), - Err(VarError::NotPresent) => default, - Err(VarError::NotUnicode(value)) => { - panic!("invalid value of env var {}={:?}: not unicode", var, value); - } - } +fn env_pubkey_or( + var: &str, + default: Pubkey, +) -> std::result::Result { + Ok(std::env::var(var) + .as_deref() + .map(Pubkey::from_str) + .ok() + .transpose()? + .unwrap_or(default)) } /// Get all accumulator related pubkeys from environment variables @@ -116,13 +99,33 @@ pub fn get_accumulator_keys() -> Vec<( &'static str, std::result::Result, )> { - vec![ - ("MESSAGE_BUFFER_PID", Ok(*MESSAGE_BUFFER_PID)), - ("ACCUMULATOR_EMITTER_ADDR", Ok(*ACCUMULATOR_EMITTER_ADDR)), - ("ACCUMULATOR_SEQUENCE_ADDR", Ok(*ACCUMULATOR_SEQUENCE_ADDR)), - ("WORMHOLE_PID", Ok(*WORMHOLE_PID)), - ("ORACLE_PID", Ok(*ORACLE_PID)), - ] + use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; + let accumulator_keys = vec![ + ( + "MESSAGE_BUFFER_PID", + Pubkey::new_from_array(MESSAGE_BUFFER_PID), + ), + // accumulator emitter address should always be the same regardless + // of environment but checking here for completeness + ( + "ACCUMULATOR_EMITTER_ADDR", + Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), + ), + ( + "ACCUMULATOR_SEQUENCE_ADDR", + Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), + ), + ( + "WORMHOLE_PID", + Pubkey::new_from_array(pythnet::WORMHOLE_PID), + ), + ]; + let accumulator_pubkeys: Vec<(&str, std::result::Result)> = + accumulator_keys + .iter() + .map(|(k, d)| (*k, env_pubkey_or(k, *d))) + .collect(); + accumulator_pubkeys } pub fn update_v1<'a>( @@ -130,16 +133,30 @@ pub fn update_v1<'a>( v2_messages: &[Vec], use_message_buffers: bool, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { + use { + byteorder::ReadBytesExt, + pythnet_sdk::{ + accumulators::{merkle::MerkleAccumulator, Accumulator}, + hashers::keccak256_160::Keccak160, + MESSAGE_BUFFER_PID, + }, + solana_sdk::borsh, + }; + // Use the current Clock to determine the index into the accumulator ring buffer. let ring_index = (bank.slot() % 10_000) as u32; // Find all accounts owned by the Message Buffer program using get_program_accounts, and // extract the account data. + let message_buffer_pid = env_pubkey_or( + "MESSAGE_BUFFER_PID", + Pubkey::new_from_array(MESSAGE_BUFFER_PID), + )?; let message_buffer_accounts; let v1_messages = if use_message_buffers { message_buffer_accounts = bank - .get_program_accounts(&MESSAGE_BUFFER_PID, &ScanConfig::new(true)) + .get_program_accounts(&message_buffer_pid, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; let preimage = b"account:MessageBuffer"; @@ -248,12 +265,28 @@ fn post_accumulator_attestation( >, ring_index: u32, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { + use pythnet_sdk::{ + pythnet, + wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, + ACCUMULATOR_EMITTER_ADDRESS, + }; + + let accumulator_sequence_addr = env_pubkey_or( + "ACCUMULATOR_SEQUENCE_ADDR", + Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), + )?; + + let accumulator_emitter_addr = env_pubkey_or( + "ACCUMULATOR_EMITTER_ADDR", + Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), + )?; + // Wormhole uses a Sequence account that is incremented each time a message is posted. As // we aren't calling Wormhole we need to bump this ourselves. If it doesn't exist, we just // create it instead. let mut sequence: AccumulatorSequenceTracker = { let data = bank - .get_account_with_fixed_root(&ACCUMULATOR_SEQUENCE_ADDR) + .get_account_with_fixed_root(&accumulator_sequence_addr) .unwrap_or_default(); let data = data.data(); solana_sdk::borsh::try_from_slice_unchecked(data) @@ -277,7 +310,7 @@ fn post_accumulator_attestation( nonce: 0, sequence: sequence.sequence, emitter_chain: 26, - emitter_address: ACCUMULATOR_EMITTER_ADDR.to_bytes(), + emitter_address: accumulator_emitter_addr.to_bytes(), payload: acc.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), } } else { @@ -288,7 +321,7 @@ fn post_accumulator_attestation( submission_time: bank.clock().unix_timestamp as u32, sequence: sequence.sequence, emitter_chain: 26, - emitter_address: ACCUMULATOR_EMITTER_ADDR.to_bytes(), + emitter_address: accumulator_emitter_addr.to_bytes(), payload: acc.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), ..Default::default() } @@ -296,13 +329,18 @@ fn post_accumulator_attestation( }; debug!("Accumulator: Wormhole message data: {:?}", message.message); + let wormhole_pid = env_pubkey_or( + "WORMHOLE_PID", + Pubkey::new_from_array(pythnet::WORMHOLE_PID), + )?; + // Now we can bump and write the Sequence account. sequence.sequence += 1; let sequence = solana_sdk::borsh::BorshSerialize::try_to_vec(&sequence) .map_err(|_| AccumulatorUpdateErrorV1::FailedSequenceSerialization)?; let sequence_balance = bank.get_minimum_balance_for_rent_exemption(sequence.len()); let sequence_account = { - let owner = &WORMHOLE_PID; + let owner = &wormhole_pid; let mut account = AccountSharedData::new(sequence_balance, sequence.len(), owner); account.set_data(sequence); account @@ -313,7 +351,7 @@ fn post_accumulator_attestation( .map_err(|_| AccumulatorUpdateErrorV1::FailedMessageSerialization)?; let message_balance = bank.get_minimum_balance_for_rent_exemption(message.len()); let message_account = { - let owner = &WORMHOLE_PID; + let owner = &wormhole_pid; let mut account = AccountSharedData::new(message_balance, message.len(), owner); account.set_data(message); account @@ -323,10 +361,10 @@ fn post_accumulator_attestation( // for RPC users to select the message for an associated VAA. let (message_pda, _) = Pubkey::find_program_address( &[b"AccumulatorMessage", &ring_index.to_be_bytes()], - &WORMHOLE_PID, + &wormhole_pid, ); - bank.store_account_and_update_capitalization(&ACCUMULATOR_SEQUENCE_ADDR, &sequence_account); + bank.store_account_and_update_capitalization(&accumulator_sequence_addr, &sequence_account); info!("Accumulator: Writing wormhole message to {:?}", message_pda); bank.store_account_and_update_capitalization(&message_pda, &message_account); @@ -336,12 +374,15 @@ fn post_accumulator_attestation( pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { let accounts = bank - .get_program_accounts(&ORACLE_PID, &ScanConfig::new(true)) + .get_program_accounts(&ORACLE_PUBKEY, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; let mut any_v1_aggregations = false; let mut v2_messages = Vec::new(); + let add_publisher_stake_caps = bank.feature_set.is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()); + compute_publisher_stake_caps(&accounts, bank.clock().unix_timestamp, &mut v2_messages, add_publisher_stake_caps); + for (pubkey, mut account) in accounts { let mut price_account_data = account.data().to_owned(); @@ -368,3 +409,22 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV update_v1(bank, &v2_messages, any_v1_aggregations) } + +pub fn compute_publisher_stake_caps( + accounts : &Vec<(Pubkey, AccountSharedData)>, + timestamp : i64, + messages : &mut Vec>, + is_active : bool +) { + info!("Computing publisher stake caps"); + let account_datas : Vec<&[u8]> = accounts + .iter() + .map(|(_, account)| account.data().borrow()) + .collect(); + let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp); + + if is_active { + info!("Adding publisher stake caps to the accumulator"); + messages.push(message); + } +} From 89b6c22c7fbe1437a7d9d9df9a957c00d74a9d15 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 16:55:35 +0100 Subject: [PATCH 14/46] go --- runtime/src/bank/pyth_accumulator.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 6204430da1..ba24eaa521 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -373,7 +373,7 @@ fn post_accumulator_attestation( } pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - let accounts = bank + let accounts: Vec<(Pubkey, AccountSharedData)> = bank .get_program_accounts(&ORACLE_PUBKEY, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; @@ -421,10 +421,9 @@ pub fn compute_publisher_stake_caps( .iter() .map(|(_, account)| account.data().borrow()) .collect(); - let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp); + let message = pyth_oracle::validator::compute_publisher_caps(account_datas, timestamp); if is_active { - info!("Adding publisher stake caps to the accumulator"); messages.push(message); } } From 8e220a7865d3cbf9b211d5719d6c801040fe5f21 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 17:23:34 +0100 Subject: [PATCH 15/46] clean --- runtime/src/bank/pyth_accumulator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index ba24eaa521..9e3219d20c 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -373,7 +373,7 @@ fn post_accumulator_attestation( } pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - let accounts: Vec<(Pubkey, AccountSharedData)> = bank + let accounts = bank .get_program_accounts(&ORACLE_PUBKEY, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; From 49559457b5d691263778698bf8db2ffb018cab89 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 20:42:51 +0100 Subject: [PATCH 16/46] ready --- runtime/src/bank/pyth_accumulator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 9e3219d20c..766d492cbd 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -421,7 +421,7 @@ pub fn compute_publisher_stake_caps( .iter() .map(|(_, account)| account.data().borrow()) .collect(); - let message = pyth_oracle::validator::compute_publisher_caps(account_datas, timestamp); + let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp); if is_active { messages.push(message); From 16ef4a8db05122be2ad9f1b57f2be6d8a38c5c93 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 20:43:52 +0100 Subject: [PATCH 17/46] add info --- runtime/src/bank/pyth_accumulator.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 766d492cbd..178706f9c1 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -424,6 +424,7 @@ pub fn compute_publisher_stake_caps( let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp); if is_active { + info!("Computing publisher caps"); messages.push(message); } } From d7001d3ecb7050fadaf0629a06c9e18bb6fd864e Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 14:46:10 +0100 Subject: [PATCH 18/46] Add new feature --- runtime/src/bank/pyth_accumulator.rs | 2 +- sdk/src/feature_set.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 178706f9c1..60d20cd196 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -424,7 +424,7 @@ pub fn compute_publisher_stake_caps( let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp); if is_active { - info!("Computing publisher caps"); + info!("Send publisher stake caps"); messages.push(message); } } diff --git a/sdk/src/feature_set.rs b/sdk/src/feature_set.rs index 9e67b42764..7f081625d9 100644 --- a/sdk/src/feature_set.rs +++ b/sdk/src/feature_set.rs @@ -564,6 +564,10 @@ pub mod redo_move_accumulator_to_end_of_block { solana_sdk::declare_id!("skyhwRBbP1LoHzWy1QrwLWy3vo2uHkzVV1zpN9UsGuw"); } +pub mod add_publisher_stake_caps_to_the_accumulator { + solana_sdk::declare_id!("J5u6Vrgj7de8zLcjQVhuRAPzEzfDamrxAQMz3q6HSmi1"); +} + lazy_static! { /// Map of feature identifiers to user-visible description pub static ref FEATURE_NAMES: HashMap = [ @@ -699,6 +703,7 @@ lazy_static! { (zero_wormhole_message_timestamps::id(), "use zeroed timestamps in wormhole messages"), (undo_move_accumulator_to_end_of_block::id(), "undo accumulator end of block change"), (redo_move_accumulator_to_end_of_block::id(), "redo accumulator end of block change"), + (add_publisher_stake_caps_to_the_accumulator::id(), "add publisher stake caps to the accumulator") /*************** ADD NEW FEATURES HERE ***************/ ] .iter() From 966996582408dd9e5bcf3c74f73eb544f1cf0699 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 15:17:17 +0100 Subject: [PATCH 19/46] update logs --- runtime/src/bank/pyth_accumulator.rs | 2 +- runtime/src/bank/pyth_accumulator_tests.rs | 97 ++++++++++++---------- 2 files changed, 55 insertions(+), 44 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 60d20cd196..6204430da1 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -424,7 +424,7 @@ pub fn compute_publisher_stake_caps( let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp); if is_active { - info!("Send publisher stake caps"); + info!("Adding publisher stake caps to the accumulator"); messages.push(message); } } diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 8528a584c5..81c693a77f 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -8,9 +8,9 @@ use crate::{ use byteorder::ByteOrder; use byteorder::{LittleEndian, ReadBytesExt}; use itertools::Itertools; -use pyth_oracle::PythOracleSerialize; -use pyth_oracle::{solana_program::account_info::AccountInfo, PriceAccountFlags}; -use pyth_oracle::{PriceAccount, PythAccount}; +// use pyth_oracle::PythOracleSerialize; +// use pyth_oracle::{solana_program::account_info::AccountInfo, PriceAccountFlags}; +// use pyth_oracle::{PriceAccount, PythAccount}; use pythnet_sdk::{ accumulators::{merkle::MerkleAccumulator, Accumulator}, hashers::{keccak256_160::Keccak160, Hasher}, @@ -35,51 +35,62 @@ use std::{io::Read, mem::size_of, sync::Arc}; // // NOTE: This was serialized by hand, but should be replaced with the pythnet-sdk // serializer once implemented. -fn create_message_buffer_bytes(msgs: Vec>) -> Vec { - let mut buffer = Vec::new(); - let preimage = b"account:MessageBuffer"; - buffer.extend_from_slice(&hashv(&[preimage]).to_bytes()[..8]); - buffer.extend_from_slice(&[0, 1, 10, 2]); - let mut sums: Vec = msgs.iter().map(|m| m.len() as u16).collect(); - sums.resize(255, 0u16); - buffer.extend( - sums.into_iter() - .scan(0, |acc, v| { - *acc += v; - Some(if v == 0 { v } else { *acc }.to_le_bytes()) - }) - .flatten(), - ); - buffer.extend(msgs.into_iter().flatten()); - buffer -} - -fn get_acc_sequence_tracker(bank: &Bank) -> AccumulatorSequenceTracker { - let account = bank - .get_account(&Pubkey::new_from_array( - pythnet_sdk::pythnet::ACCUMULATOR_SEQUENCE_ADDR, - )) - .unwrap(); - AccumulatorSequenceTracker::try_from_slice(&mut account.data()).unwrap() -} +// fn create_message_buffer_bytes(msgs: Vec>) -> Vec { +// let mut buffer = Vec::new(); +// let preimage = b"account:MessageBuffer"; +// buffer.extend_from_slice(&hashv(&[preimage]).to_bytes()[..8]); +// buffer.extend_from_slice(&[0, 1, 10, 2]); +// let mut sums: Vec = msgs.iter().map(|m| m.len() as u16).collect(); +// sums.resize(255, 0u16); +// buffer.extend( +// sums.into_iter() +// .scan(0, |acc, v| { +// *acc += v; +// Some(if v == 0 { v } else { *acc }.to_le_bytes()) +// }) +// .flatten(), +// ); +// buffer.extend(msgs.into_iter().flatten()); +// buffer +// } + +// fn get_acc_sequence_tracker(bank: &Bank) -> AccumulatorSequenceTracker { +// let account = bank +// .get_account(&Pubkey::new_from_array( +// pythnet_sdk::pythnet::ACCUMULATOR_SEQUENCE_ADDR, +// )) +// .unwrap(); +// AccumulatorSequenceTracker::try_from_slice(&mut account.data()).unwrap() +// } + +// fn get_wormhole_message_account(bank: &Bank, ring_index: u32) -> AccountSharedData { +// let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( +// &[b"AccumulatorMessage", &ring_index.to_be_bytes()], +// &Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), +// ); +// bank.get_account(&wormhole_message_pubkey) +// .unwrap_or_default() +// } + +// fn get_accumulator_state(bank: &Bank, ring_index: u32) -> Vec { +// let (accumulator_state_pubkey, _) = Pubkey::find_program_address( +// &[b"AccumulatorState", &ring_index.to_be_bytes()], +// &solana_sdk::system_program::id(), +// ); + +// let account = bank.get_account(&accumulator_state_pubkey).unwrap(); +// account.data().to_vec() +// } -fn get_wormhole_message_account(bank: &Bank, ring_index: u32) -> AccountSharedData { - let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( - &[b"AccumulatorMessage", &ring_index.to_be_bytes()], - &Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), - ); - bank.get_account(&wormhole_message_pubkey) - .unwrap_or_default() -} - -fn get_accumulator_state(bank: &Bank, ring_index: u32) -> Vec { +#[test] +fn test_get_accumulator_state(){ + let index = 0; let (accumulator_state_pubkey, _) = Pubkey::find_program_address( - &[b"AccumulatorState", &ring_index.to_be_bytes()], + &[b"AccumulatorState", &(2386_u32).to_be_bytes()], &solana_sdk::system_program::id(), ); + println!("{:?}", accumulator_state_pubkey); - let account = bank.get_account(&accumulator_state_pubkey).unwrap(); - account.data().to_vec() } #[test] From aab935ea57634352ef478faea092a5996866b492 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 15:26:58 +0100 Subject: [PATCH 20/46] undo this change --- runtime/src/bank/pyth_accumulator_tests.rs | 77 ++++++---------------- 1 file changed, 19 insertions(+), 58 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 81c693a77f..3f9a55e6fa 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -8,9 +8,9 @@ use crate::{ use byteorder::ByteOrder; use byteorder::{LittleEndian, ReadBytesExt}; use itertools::Itertools; -// use pyth_oracle::PythOracleSerialize; -// use pyth_oracle::{solana_program::account_info::AccountInfo, PriceAccountFlags}; -// use pyth_oracle::{PriceAccount, PythAccount}; +use pyth_oracle::PythOracleSerialize; +use pyth_oracle::{solana_program::account_info::AccountInfo, PriceAccountFlags}; +use pyth_oracle::{PriceAccount, PythAccount}; use pythnet_sdk::{ accumulators::{merkle::MerkleAccumulator, Accumulator}, hashers::{keccak256_160::Keccak160, Hasher}, @@ -35,62 +35,23 @@ use std::{io::Read, mem::size_of, sync::Arc}; // // NOTE: This was serialized by hand, but should be replaced with the pythnet-sdk // serializer once implemented. -// fn create_message_buffer_bytes(msgs: Vec>) -> Vec { -// let mut buffer = Vec::new(); -// let preimage = b"account:MessageBuffer"; -// buffer.extend_from_slice(&hashv(&[preimage]).to_bytes()[..8]); -// buffer.extend_from_slice(&[0, 1, 10, 2]); -// let mut sums: Vec = msgs.iter().map(|m| m.len() as u16).collect(); -// sums.resize(255, 0u16); -// buffer.extend( -// sums.into_iter() -// .scan(0, |acc, v| { -// *acc += v; -// Some(if v == 0 { v } else { *acc }.to_le_bytes()) -// }) -// .flatten(), -// ); -// buffer.extend(msgs.into_iter().flatten()); -// buffer -// } - -// fn get_acc_sequence_tracker(bank: &Bank) -> AccumulatorSequenceTracker { -// let account = bank -// .get_account(&Pubkey::new_from_array( -// pythnet_sdk::pythnet::ACCUMULATOR_SEQUENCE_ADDR, -// )) -// .unwrap(); -// AccumulatorSequenceTracker::try_from_slice(&mut account.data()).unwrap() -// } - -// fn get_wormhole_message_account(bank: &Bank, ring_index: u32) -> AccountSharedData { -// let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( -// &[b"AccumulatorMessage", &ring_index.to_be_bytes()], -// &Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), -// ); -// bank.get_account(&wormhole_message_pubkey) -// .unwrap_or_default() -// } - -// fn get_accumulator_state(bank: &Bank, ring_index: u32) -> Vec { -// let (accumulator_state_pubkey, _) = Pubkey::find_program_address( -// &[b"AccumulatorState", &ring_index.to_be_bytes()], -// &solana_sdk::system_program::id(), -// ); - -// let account = bank.get_account(&accumulator_state_pubkey).unwrap(); -// account.data().to_vec() -// } - -#[test] -fn test_get_accumulator_state(){ - let index = 0; - let (accumulator_state_pubkey, _) = Pubkey::find_program_address( - &[b"AccumulatorState", &(2386_u32).to_be_bytes()], - &solana_sdk::system_program::id(), +fn create_message_buffer_bytes(msgs: Vec>) -> Vec { + let mut buffer = Vec::new(); + let preimage = b"account:MessageBuffer"; + buffer.extend_from_slice(&hashv(&[preimage]).to_bytes()[..8]); + buffer.extend_from_slice(&[0, 1, 10, 2]); + let mut sums: Vec = msgs.iter().map(|m| m.len() as u16).collect(); + sums.resize(255, 0u16); + buffer.extend( + sums.into_iter() + .scan(0, |acc, v| { + *acc += v; + Some(if v == 0 { v } else { *acc }.to_le_bytes()) + }) + .flatten(), ); - println!("{:?}", accumulator_state_pubkey); - + buffer.extend(msgs.into_iter().flatten()); + buffer } #[test] From 0dc93f648ebfa31e4644b7d0c8dec345f4f255ff Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 15:37:33 +0100 Subject: [PATCH 21/46] reset --- runtime/src/bank/pyth_accumulator.rs | 154 ++++++++------------- runtime/src/bank/pyth_accumulator_tests.rs | 28 ++++ 2 files changed, 86 insertions(+), 96 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 6204430da1..2569cbf185 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -2,35 +2,49 @@ use { super::Bank, crate::accounts_index::{ScanConfig, ScanError}, byteorder::LittleEndian, + byteorder::ReadBytesExt, itertools::Itertools, log::*, pyth_oracle::validator::AggregationError, + pythnet_sdk::{ + accumulators::{merkle::MerkleAccumulator, Accumulator}, + hashers::keccak256_160::Keccak160, + wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, + }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, - feature_set, + borsh, feature_set, hash::hashv, pubkey::Pubkey, }, - std::{ - borrow::Borrow, env::{self, VarError}, str::FromStr - }, + std::env::{self, VarError}, }; pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; lazy_static! { - pub static ref ORACLE_PUBKEY: Pubkey = match env::var("PYTH_ORACLE_PUBKEY") { - Ok(value) => value + pub static ref MESSAGE_BUFFER_PID: Pubkey = env_pubkey_or( + "MESSAGE_BUFFER_PID", + Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID), + ); + pub static ref ACCUMULATOR_EMITTER_ADDR: Pubkey = env_pubkey_or( + "ACCUMULATOR_EMITTER_ADDR", + Pubkey::new_from_array(pythnet_sdk::ACCUMULATOR_EMITTER_ADDRESS), + ); + pub static ref ACCUMULATOR_SEQUENCE_ADDR: Pubkey = env_pubkey_or( + "ACCUMULATOR_SEQUENCE_ADDR", + Pubkey::new_from_array(pythnet_sdk::pythnet::ACCUMULATOR_SEQUENCE_ADDR), + ); + pub static ref WORMHOLE_PID: Pubkey = env_pubkey_or( + "WORMHOLE_PID", + Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), + ); + pub static ref ORACLE_PID: Pubkey = env_pubkey_or( + "ORACLE_PID", + "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH" .parse() - .expect("invalid value of PYTH_ORACLE_PUBKEY env var"), - Err(VarError::NotPresent) => { - // Pythnet oracle program address - "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH".parse().unwrap() - } - Err(VarError::NotUnicode(err)) => { - panic!("invalid value of PYTH_ORACLE_PUBKEY env var: {err:?}"); - } - }; + .unwrap(), + ); } /// Accumulator specific error type. It would be nice to use `transaction::Error` but it does @@ -81,16 +95,19 @@ pub fn update_accumulator(bank: &Bank) { /// Read the pubkey from the environment variable `var` or return `default` /// if the variable is not set. -fn env_pubkey_or( - var: &str, - default: Pubkey, -) -> std::result::Result { - Ok(std::env::var(var) - .as_deref() - .map(Pubkey::from_str) - .ok() - .transpose()? - .unwrap_or(default)) +fn env_pubkey_or(var: &str, default: Pubkey) -> Pubkey { + match env::var(var) { + Ok(value) => value.parse().unwrap_or_else(|err| { + panic!( + "failed to parse env var {}={:?} as pubkey: {}", + var, value, err + ) + }), + Err(VarError::NotPresent) => default, + Err(VarError::NotUnicode(value)) => { + panic!("invalid value of env var {}={:?}: not unicode", var, value); + } + } } /// Get all accumulator related pubkeys from environment variables @@ -99,33 +116,13 @@ pub fn get_accumulator_keys() -> Vec<( &'static str, std::result::Result, )> { - use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; - let accumulator_keys = vec![ - ( - "MESSAGE_BUFFER_PID", - Pubkey::new_from_array(MESSAGE_BUFFER_PID), - ), - // accumulator emitter address should always be the same regardless - // of environment but checking here for completeness - ( - "ACCUMULATOR_EMITTER_ADDR", - Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), - ), - ( - "ACCUMULATOR_SEQUENCE_ADDR", - Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), - ), - ( - "WORMHOLE_PID", - Pubkey::new_from_array(pythnet::WORMHOLE_PID), - ), - ]; - let accumulator_pubkeys: Vec<(&str, std::result::Result)> = - accumulator_keys - .iter() - .map(|(k, d)| (*k, env_pubkey_or(k, *d))) - .collect(); - accumulator_pubkeys + vec![ + ("MESSAGE_BUFFER_PID", Ok(*MESSAGE_BUFFER_PID)), + ("ACCUMULATOR_EMITTER_ADDR", Ok(*ACCUMULATOR_EMITTER_ADDR)), + ("ACCUMULATOR_SEQUENCE_ADDR", Ok(*ACCUMULATOR_SEQUENCE_ADDR)), + ("WORMHOLE_PID", Ok(*WORMHOLE_PID)), + ("ORACLE_PID", Ok(*ORACLE_PID)), + ] } pub fn update_v1<'a>( @@ -133,30 +130,16 @@ pub fn update_v1<'a>( v2_messages: &[Vec], use_message_buffers: bool, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - use { - byteorder::ReadBytesExt, - pythnet_sdk::{ - accumulators::{merkle::MerkleAccumulator, Accumulator}, - hashers::keccak256_160::Keccak160, - MESSAGE_BUFFER_PID, - }, - solana_sdk::borsh, - }; - // Use the current Clock to determine the index into the accumulator ring buffer. let ring_index = (bank.slot() % 10_000) as u32; // Find all accounts owned by the Message Buffer program using get_program_accounts, and // extract the account data. - let message_buffer_pid = env_pubkey_or( - "MESSAGE_BUFFER_PID", - Pubkey::new_from_array(MESSAGE_BUFFER_PID), - )?; let message_buffer_accounts; let v1_messages = if use_message_buffers { message_buffer_accounts = bank - .get_program_accounts(&message_buffer_pid, &ScanConfig::new(true)) + .get_program_accounts(&MESSAGE_BUFFER_PID, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; let preimage = b"account:MessageBuffer"; @@ -265,28 +248,12 @@ fn post_accumulator_attestation( >, ring_index: u32, ) -> std::result::Result<(), AccumulatorUpdateErrorV1> { - use pythnet_sdk::{ - pythnet, - wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, - ACCUMULATOR_EMITTER_ADDRESS, - }; - - let accumulator_sequence_addr = env_pubkey_or( - "ACCUMULATOR_SEQUENCE_ADDR", - Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), - )?; - - let accumulator_emitter_addr = env_pubkey_or( - "ACCUMULATOR_EMITTER_ADDR", - Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), - )?; - // Wormhole uses a Sequence account that is incremented each time a message is posted. As // we aren't calling Wormhole we need to bump this ourselves. If it doesn't exist, we just // create it instead. let mut sequence: AccumulatorSequenceTracker = { let data = bank - .get_account_with_fixed_root(&accumulator_sequence_addr) + .get_account_with_fixed_root(&ACCUMULATOR_SEQUENCE_ADDR) .unwrap_or_default(); let data = data.data(); solana_sdk::borsh::try_from_slice_unchecked(data) @@ -310,7 +277,7 @@ fn post_accumulator_attestation( nonce: 0, sequence: sequence.sequence, emitter_chain: 26, - emitter_address: accumulator_emitter_addr.to_bytes(), + emitter_address: ACCUMULATOR_EMITTER_ADDR.to_bytes(), payload: acc.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), } } else { @@ -321,7 +288,7 @@ fn post_accumulator_attestation( submission_time: bank.clock().unix_timestamp as u32, sequence: sequence.sequence, emitter_chain: 26, - emitter_address: accumulator_emitter_addr.to_bytes(), + emitter_address: ACCUMULATOR_EMITTER_ADDR.to_bytes(), payload: acc.serialize(bank.slot(), ACCUMULATOR_RING_SIZE), ..Default::default() } @@ -329,18 +296,13 @@ fn post_accumulator_attestation( }; debug!("Accumulator: Wormhole message data: {:?}", message.message); - let wormhole_pid = env_pubkey_or( - "WORMHOLE_PID", - Pubkey::new_from_array(pythnet::WORMHOLE_PID), - )?; - // Now we can bump and write the Sequence account. sequence.sequence += 1; let sequence = solana_sdk::borsh::BorshSerialize::try_to_vec(&sequence) .map_err(|_| AccumulatorUpdateErrorV1::FailedSequenceSerialization)?; let sequence_balance = bank.get_minimum_balance_for_rent_exemption(sequence.len()); let sequence_account = { - let owner = &wormhole_pid; + let owner = &WORMHOLE_PID; let mut account = AccountSharedData::new(sequence_balance, sequence.len(), owner); account.set_data(sequence); account @@ -351,7 +313,7 @@ fn post_accumulator_attestation( .map_err(|_| AccumulatorUpdateErrorV1::FailedMessageSerialization)?; let message_balance = bank.get_minimum_balance_for_rent_exemption(message.len()); let message_account = { - let owner = &wormhole_pid; + let owner = &WORMHOLE_PID; let mut account = AccountSharedData::new(message_balance, message.len(), owner); account.set_data(message); account @@ -361,10 +323,10 @@ fn post_accumulator_attestation( // for RPC users to select the message for an associated VAA. let (message_pda, _) = Pubkey::find_program_address( &[b"AccumulatorMessage", &ring_index.to_be_bytes()], - &wormhole_pid, + &WORMHOLE_PID, ); - bank.store_account_and_update_capitalization(&accumulator_sequence_addr, &sequence_account); + bank.store_account_and_update_capitalization(&ACCUMULATOR_SEQUENCE_ADDR, &sequence_account); info!("Accumulator: Writing wormhole message to {:?}", message_pda); bank.store_account_and_update_capitalization(&message_pda, &message_account); @@ -374,7 +336,7 @@ fn post_accumulator_attestation( pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV1> { let accounts = bank - .get_program_accounts(&ORACLE_PUBKEY, &ScanConfig::new(true)) + .get_program_accounts(&ORACLE_PID, &ScanConfig::new(true)) .map_err(AccumulatorUpdateErrorV1::GetProgramAccounts)?; let mut any_v1_aggregations = false; diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 3f9a55e6fa..8528a584c5 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -54,6 +54,34 @@ fn create_message_buffer_bytes(msgs: Vec>) -> Vec { buffer } +fn get_acc_sequence_tracker(bank: &Bank) -> AccumulatorSequenceTracker { + let account = bank + .get_account(&Pubkey::new_from_array( + pythnet_sdk::pythnet::ACCUMULATOR_SEQUENCE_ADDR, + )) + .unwrap(); + AccumulatorSequenceTracker::try_from_slice(&mut account.data()).unwrap() +} + +fn get_wormhole_message_account(bank: &Bank, ring_index: u32) -> AccountSharedData { + let (wormhole_message_pubkey, _bump) = Pubkey::find_program_address( + &[b"AccumulatorMessage", &ring_index.to_be_bytes()], + &Pubkey::new_from_array(pythnet_sdk::pythnet::WORMHOLE_PID), + ); + bank.get_account(&wormhole_message_pubkey) + .unwrap_or_default() +} + +fn get_accumulator_state(bank: &Bank, ring_index: u32) -> Vec { + let (accumulator_state_pubkey, _) = Pubkey::find_program_address( + &[b"AccumulatorState", &ring_index.to_be_bytes()], + &solana_sdk::system_program::id(), + ); + + let account = bank.get_account(&accumulator_state_pubkey).unwrap(); + account.data().to_vec() +} + #[test] fn test_update_accumulator_sysvar() { let leader_pubkey = solana_sdk::pubkey::new_rand(); From 30b1057a6870292aa90a0f6cb1f7a2ab08a9e291 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 15:38:26 +0100 Subject: [PATCH 22/46] fix --- runtime/src/bank/pyth_accumulator.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 2569cbf185..3584a2a380 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -19,6 +19,7 @@ use { }, std::env::{self, VarError}, }; +use std::borrow::Borrow; pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; From 3aee40e759f9bb5b07011d77658858f1401f3e1b Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 16:35:41 +0100 Subject: [PATCH 23/46] bump --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58d218ac19..50c9927632 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3600,7 +3600,7 @@ dependencies = [ [[package]] name = "pyth-oracle" version = "2.26.0" -source = "git+https://github.com/pyth-network/pyth-client.git?branch=feat/publisher-caps#170afdec9f5434deb3c242765ca409b180f306f7" +source = "git+https://github.com/pyth-network/pyth-client.git?branch=feat/publisher-caps#706a29c55b5114af303a1dbac684c50b262f72aa" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", @@ -3635,7 +3635,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "2.1.0" -source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#3522f2940522e49aa56038e1000ea78f2e489d20" +source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#76a9a50e51eed23fa123af08db7b1f79d708544e" dependencies = [ "bincode", "borsh 0.10.3", From b48fa0f4a28e7a1bc4464ee13dcd369d858ac599 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 16:37:06 +0100 Subject: [PATCH 24/46] add z --- runtime/src/bank/pyth_accumulator.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 3584a2a380..35fe12f6de 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -373,6 +373,8 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV update_v1(bank, &v2_messages, any_v1_aggregations) } +pub const Z : u64 = 10; + pub fn compute_publisher_stake_caps( accounts : &Vec<(Pubkey, AccountSharedData)>, timestamp : i64, @@ -384,7 +386,7 @@ pub fn compute_publisher_stake_caps( .iter() .map(|(_, account)| account.data().borrow()) .collect(); - let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp); + let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp, Z); if is_active { info!("Adding publisher stake caps to the accumulator"); From 7145ab769c8c5c8c0213e950d804e8080a168e7a Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 19 Jul 2024 11:41:30 +0100 Subject: [PATCH 25/46] chore: fix build for stable Rust by ignoring lints --- programs/bpf_loader/src/syscalls/cpi.rs | 4 +++- runtime/src/append_vec.rs | 4 ++++ sdk/program/src/account_info.rs | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/programs/bpf_loader/src/syscalls/cpi.rs b/programs/bpf_loader/src/syscalls/cpi.rs index bf65d47d04..a3abf6a871 100644 --- a/programs/bpf_loader/src/syscalls/cpi.rs +++ b/programs/bpf_loader/src/syscalls/cpi.rs @@ -539,7 +539,9 @@ impl<'a, 'b> SyscallInvokeSigned<'a, 'b> for SyscallInvokeSignedC<'a, 'b> { vm_addr, size_of::() as u64, )?; - let ref_to_len_in_vm = unsafe { &mut *(addr as *mut u64) }; + // This lint was not present on Rust 1.60. + #[allow(unknown_lints, invalid_reference_casting)] + let ref_to_len_in_vm = { unsafe { &mut *(addr as *mut u64) } }; let ref_of_len_in_input_buffer = (account_info.data_addr as *mut u8 as u64).saturating_sub(8); diff --git a/runtime/src/append_vec.rs b/runtime/src/append_vec.rs index 50b7f6719b..be829e1407 100644 --- a/runtime/src/append_vec.rs +++ b/runtime/src/append_vec.rs @@ -585,6 +585,8 @@ pub mod tests { #[allow(clippy::cast_ref_to_mut)] fn set_data_len_unsafe(&self, new_data_len: u64) { // UNSAFE: cast away & (= const ref) to &mut to force to mutate append-only (=read-only) AppendVec + // This lint was not present on Rust 1.60. + #[allow(unknown_lints, invalid_reference_casting)] unsafe { *(&self.meta.data_len as *const u64 as *mut u64) = new_data_len; } @@ -600,6 +602,8 @@ pub mod tests { #[allow(clippy::cast_ref_to_mut)] fn set_executable_as_byte(&self, new_executable_byte: u8) { // UNSAFE: Force to interpret mmap-backed &bool as &u8 to write some crafted value; + // This lint was not present on Rust 1.60. + #[allow(unknown_lints, invalid_reference_casting)] unsafe { *(&self.account_meta.executable as *const bool as *mut u8) = new_executable_byte; } diff --git a/sdk/program/src/account_info.rs b/sdk/program/src/account_info.rs index 3652e2251e..5ab7d0b2d4 100644 --- a/sdk/program/src/account_info.rs +++ b/sdk/program/src/account_info.rs @@ -183,6 +183,8 @@ impl<'a> AccountInfo<'a> { pub fn assign(&self, new_owner: &Pubkey) { // Set the non-mut owner field + // This lint was not present on Rust 1.60. + #[allow(unknown_lints, invalid_reference_casting)] unsafe { std::ptr::write_volatile( self.owner as *const Pubkey as *mut [u8; 32], From 44d13c1fecb325067baeff8587f49d3d1b024488 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Fri, 19 Jul 2024 19:41:51 +0100 Subject: [PATCH 26/46] add m and z --- Cargo.lock | 4 ++-- runtime/Cargo.toml | 2 +- runtime/src/bank/pyth_accumulator.rs | 31 ++++++++++++++++++++++------ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50c9927632..d102667734 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3600,7 +3600,7 @@ dependencies = [ [[package]] name = "pyth-oracle" version = "2.26.0" -source = "git+https://github.com/pyth-network/pyth-client.git?branch=feat/publisher-caps#706a29c55b5114af303a1dbac684c50b262f72aa" +source = "git+https://github.com/pyth-network/pyth-client.git?branch=feat/publisher-caps#fe64764a92e7212fc9b737070b7d3e288bcc4730" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", @@ -3617,7 +3617,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "1.13.6" -source = "git+https://github.com/pyth-network/pyth-crosschain?rev=e670f57f89b05398ca352e4accb1e32724a8e1b4#e670f57f89b05398ca352e4accb1e32724a8e1b4" +source = "git+https://github.com/pyth-network/pyth-crosschain?rev=a4fafa5dacd076a4d0656549d9a182bfba07f415#a4fafa5dacd076a4d0656549d9a182bfba07f415" dependencies = [ "bincode", "borsh 0.9.3", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 84f6d5caa5..7acf37b91f 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -36,7 +36,7 @@ num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" pyth-oracle = { git = "https://github.com/pyth-network/pyth-client.git" ,branch="feat/publisher-caps" , features = ["library"] } -pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "e670f57f89b05398ca352e4accb1e32724a8e1b4" } +pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "a4fafa5dacd076a4d0656549d9a182bfba07f415" } rand = "0.7.0" rayon = "1.5.3" regex = "1.5.6" diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 35fe12f6de..f38100ab2c 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -10,6 +10,7 @@ use { accumulators::{merkle::MerkleAccumulator, Accumulator}, hashers::keccak256_160::Keccak160, wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, + publisher_stake_caps::StakeCapParameters }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, @@ -46,8 +47,17 @@ lazy_static! { .parse() .unwrap(), ); + + pub static ref STAKE_CAPS_PARAMETERS_ADDR: Pubkey = env_pubkey_or( + "STAKE_CAPS_PARAMETERS_ADDR", + "879ZVNagiWaAKsWDjGVf8pLq1wUBeBz7sREjUh3hrU36" + .parse() + .unwrap(), + ); } + + /// Accumulator specific error type. It would be nice to use `transaction::Error` but it does /// not include any `Custom` style variant we can leverage, so we introduce our own. #[derive(Debug, thiserror::Error)] @@ -123,6 +133,7 @@ pub fn get_accumulator_keys() -> Vec<( ("ACCUMULATOR_SEQUENCE_ADDR", Ok(*ACCUMULATOR_SEQUENCE_ADDR)), ("WORMHOLE_PID", Ok(*WORMHOLE_PID)), ("ORACLE_PID", Ok(*ORACLE_PID)), + ("STAKE_CAPS_PARAMETERS_ADDR", Ok(*STAKE_CAPS_PARAMETERS_ADDR)), ] } @@ -344,7 +355,7 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV let mut v2_messages = Vec::new(); let add_publisher_stake_caps = bank.feature_set.is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()); - compute_publisher_stake_caps(&accounts, bank.clock().unix_timestamp, &mut v2_messages, add_publisher_stake_caps); + compute_publisher_stake_caps(bank, &accounts, bank.clock().unix_timestamp, &mut v2_messages, add_publisher_stake_caps); for (pubkey, mut account) in accounts { let mut price_account_data = account.data().to_owned(); @@ -373,23 +384,31 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV update_v1(bank, &v2_messages, any_v1_aggregations) } -pub const Z : u64 = 10; - pub fn compute_publisher_stake_caps( + bank : &Bank, accounts : &Vec<(Pubkey, AccountSharedData)>, timestamp : i64, messages : &mut Vec>, is_active : bool ) { - info!("Computing publisher stake caps"); + let parameters: StakeCapParameters = { + let data = bank + .get_account_with_fixed_root(&STAKE_CAPS_PARAMETERS_ADDR) + .unwrap_or_default(); + let data = data.data(); + solana_sdk::borsh::try_from_slice_unchecked(data) + .unwrap_or(StakeCapParameters { _discriminator: 0, _current_authority : [0u8;32], z: 1, m : 1_000_000_000 }) +}; + + info!("Computing publisher stake caps with m : {} and z : {}", parameters.m, parameters.z); let account_datas : Vec<&[u8]> = accounts .iter() .map(|(_, account)| account.data().borrow()) .collect(); - let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp, Z); + let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp, parameters.m, parameters.z); if is_active { info!("Adding publisher stake caps to the accumulator"); messages.push(message); } -} +} \ No newline at end of file From 063bf58db0ba516954455a46e6b6f351ff906cdf Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Mon, 22 Jul 2024 13:02:38 +0100 Subject: [PATCH 27/46] test: fix and expand accumulator-v2 tests --- Cargo.lock | 1 + runtime/Cargo.toml | 2 +- runtime/src/bank/pyth_accumulator.rs | 5 +-- runtime/src/bank/pyth_accumulator_tests.rs | 51 ++++++++++++++++------ 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 455a14336e..7da5e2c8ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3556,6 +3556,7 @@ dependencies = [ [[package]] name = "pyth-oracle" version = "2.26.0" +source = "git+https://github.com/pyth-network/pyth-client?branch=accumulator-v2#d949c27e14010427ce654055827c1610e577ecec" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 450f6c1188..b83f212d9f 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -35,7 +35,7 @@ num-traits = { version = "0.2" } num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" -pyth-oracle = { path = "/code/pyth-network/pyth-client/program/rust", features = ["library"] } +pyth-oracle = { git = "https://github.com/pyth-network/pyth-client", branch = "accumulator-v2", features = ["library"] } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "e670f57f89b05398ca352e4accb1e32724a8e1b4" } rand = "0.7.0" rayon = "1.5.3" diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 4bfd8eac14..819ab79ea5 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -3,7 +3,6 @@ use { crate::accounts_index::{ScanConfig, ScanError}, byteorder::LittleEndian, byteorder::ReadBytesExt, - itertools::Itertools, log::*, pyth_oracle::validator::AggregationError, pythnet_sdk::{ @@ -187,8 +186,6 @@ pub fn update_v1<'a>( .collect::, std::io::Error>>()? .into_iter() .flatten() - .sorted_unstable() - .dedup() .collect() } else { Vec::new() @@ -196,6 +193,8 @@ pub fn update_v1<'a>( let mut messages = v1_messages; messages.extend(v2_messages.iter().map(|x| &**x)); + messages.sort_unstable(); + messages.dedup(); // We now generate a Proof PDA (Owned by the System Program) to store the resulting Proof // Set. The derivation includes the ring buffer index to simulate a ring buffer in order diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 8528a584c5..69032fea6d 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -668,9 +668,22 @@ fn test_update_accumulator_end_of_block() { ); } -// This test will #[test] -fn test_accumulator_v2() { +fn test_accumulator_v2_all_v2() { + test_accumulator_v2([false, false, false, false]); +} + +#[test] +fn test_accumulator_v2_all_v1() { + test_accumulator_v2([true, true, true, true]); +} + +#[test] +fn test_accumulator_v2_mixed() { + test_accumulator_v2([true, true, false, false]); +} + +fn test_accumulator_v2(generate_buffers: [bool; 4]) { let leader_pubkey = solana_sdk::pubkey::new_rand(); let GenesisConfigInfo { mut genesis_config, .. @@ -682,9 +695,6 @@ fn test_accumulator_v2() { genesis_config.epoch_schedule = EpochSchedule::new(slots_in_epoch); let mut bank = Bank::new_for_tests(&genesis_config); - bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. - bank = new_from_parent(&Arc::new(bank)); // Advance slot 2. - let generate_price = |seeds, generate_buffers: bool| { let (price_feed_key, _bump) = Pubkey::find_program_address(&[seeds], &ORACLE_PID); let mut price_feed_account = @@ -730,8 +740,10 @@ fn test_accumulator_v2() { // sorts first. let message_buffer_bytes = create_message_buffer_bytes(messages.clone()); + let mut seed = vec![1; 32]; + seed[..seeds.len()].copy_from_slice(seeds); // Create a Message account. - let price_message_key = keypair_from_seed(&[1u8; 32]).unwrap(); + let price_message_key = keypair_from_seed(&seed).unwrap(); let mut price_message_account = bank .get_account(&price_message_key.pubkey()) .unwrap_or_default(); @@ -774,17 +786,23 @@ fn test_accumulator_v2() { .is_active(&feature_set::redo_move_accumulator_to_end_of_block::id())); let prices_with_messages = [ - generate_price(b"seeds_1", false), - generate_price(b"seeds_2", false), - generate_price(b"seeds_3", false), - generate_price(b"seeds_4", false), + generate_price(b"seeds_1", generate_buffers[0]), + generate_price(b"seeds_2", generate_buffers[1]), + generate_price(b"seeds_3", generate_buffers[2]), + generate_price(b"seeds_4", generate_buffers[3]), ]; + bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. + bank = new_from_parent(&Arc::new(bank)); // Advance slot 2. + let messages = prices_with_messages .iter() .map(|(_, messages)| messages) .flatten() - .map(|message| &message[..]); + .map(|message| &message[..]) + .sorted_unstable() + .dedup() + .collect::>(); // Trigger Aggregation. We freeze instead of new_from_parent so // we can keep access to the bank. @@ -795,13 +813,13 @@ fn test_accumulator_v2() { // to offset the ring index as our test is always below 10K slots. let wormhole_message_account = get_wormhole_message_account(&bank, bank.slot() as u32); assert_ne!(wormhole_message_account.data().len(), 0); - PostedMessageUnreliableData::deserialize(&mut wormhole_message_account.data()).unwrap(); + let deserialized_wormhole_message = + PostedMessageUnreliableData::deserialize(&mut wormhole_message_account.data()).unwrap(); // Create MerkleAccumulator by hand to verify that the Wormhole message // contents are correctg. let expected_accumulator = - MerkleAccumulator::::from_set(messages.clone().sorted_unstable().dedup()) - .unwrap(); + MerkleAccumulator::::from_set(messages.iter().copied()).unwrap(); let expected_wormhole_message_payload = expected_accumulator.serialize(bank.slot(), ACCUMULATOR_RING_SIZE); @@ -818,6 +836,10 @@ fn test_accumulator_v2() { ..Default::default() }, }; + assert_eq!( + deserialized_wormhole_message.payload, + expected_wormhole_message.payload + ); assert_eq!( wormhole_message_account.data().to_vec(), @@ -876,6 +898,7 @@ fn test_get_accumulator_keys() { Pubkey::new_from_array(ACCUMULATOR_EMITTER_ADDRESS), Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), Pubkey::new_from_array(pythnet::WORMHOLE_PID), + *ORACLE_PID, ]; assert_eq!(accumulator_keys, expected_pyth_keys); } From 5f1ee90efd1c547a75b7799c0950e49ac83232bb Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 24 Jul 2024 22:57:20 +0100 Subject: [PATCH 28/46] fix: revert file --- core/src/validator.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/validator.rs b/core/src/validator.rs index 60ac4a052e..923be65d65 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -1,7 +1,6 @@ //! The `validator` module hosts all the validator microservices. pub use solana_perf::report_target_features; -use solana_runtime::bank::pyth_accumulator; use { crate::{ accounts_hash_verifier::AccountsHashVerifier, From 169899f4aca78360ae16290ade498df3007dea29 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 24 Jul 2024 23:03:18 +0100 Subject: [PATCH 29/46] cleanup --- runtime/src/bank/pyth_accumulator.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 63aff04c0c..a4c29bccd5 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -385,7 +385,7 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV pub fn compute_publisher_stake_caps( bank : &Bank, - accounts : &Vec<(Pubkey, AccountSharedData)>, + accounts : &[(Pubkey, AccountSharedData)], timestamp : i64, messages : &mut Vec>, is_active : bool @@ -402,7 +402,7 @@ pub fn compute_publisher_stake_caps( info!("Computing publisher stake caps with m : {} and z : {}", parameters.m, parameters.z); let account_datas : Vec<&[u8]> = accounts .iter() - .map(|(_, account)| account.data().borrow()) + .map(|(_, account)| account.data()) .collect(); let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp, parameters.m, parameters.z); From 0811f95c5285d337ed910748c0a6d88fa433ddde Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 19:09:20 +0100 Subject: [PATCH 30/46] fix: format --- runtime/src/bank/pyth_accumulator.rs | 74 ++++++++++++++++------------ 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index a4c29bccd5..ecd91dc1b3 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -1,3 +1,4 @@ +use std::borrow::Borrow; use { super::Bank, crate::accounts_index::{ScanConfig, ScanError}, @@ -8,8 +9,8 @@ use { pythnet_sdk::{ accumulators::{merkle::MerkleAccumulator, Accumulator}, hashers::keccak256_160::Keccak160, + publisher_stake_caps::StakeCapParameters, wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, - publisher_stake_caps::StakeCapParameters }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, @@ -19,7 +20,6 @@ use { }, std::env::{self, VarError}, }; -use std::borrow::Borrow; pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; @@ -46,17 +46,14 @@ lazy_static! { .parse() .unwrap(), ); - pub static ref STAKE_CAPS_PARAMETERS_ADDR: Pubkey = env_pubkey_or( "STAKE_CAPS_PARAMETERS_ADDR", "879ZVNagiWaAKsWDjGVf8pLq1wUBeBz7sREjUh3hrU36" - .parse() - .unwrap(), + .parse() + .unwrap(), ); } - - /// Accumulator specific error type. It would be nice to use `transaction::Error` but it does /// not include any `Custom` style variant we can leverage, so we introduce our own. #[derive(Debug, thiserror::Error)] @@ -132,7 +129,10 @@ pub fn get_accumulator_keys() -> Vec<( ("ACCUMULATOR_SEQUENCE_ADDR", Ok(*ACCUMULATOR_SEQUENCE_ADDR)), ("WORMHOLE_PID", Ok(*WORMHOLE_PID)), ("ORACLE_PID", Ok(*ORACLE_PID)), - ("STAKE_CAPS_PARAMETERS_ADDR", Ok(*STAKE_CAPS_PARAMETERS_ADDR)), + ( + "STAKE_CAPS_PARAMETERS_ADDR", + Ok(*STAKE_CAPS_PARAMETERS_ADDR), + ), ] } @@ -353,8 +353,10 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV let mut any_v1_aggregations = false; let mut v2_messages = Vec::new(); - let add_publisher_stake_caps = bank.feature_set.is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()); - compute_publisher_stake_caps(bank, &accounts, bank.clock().unix_timestamp, &mut v2_messages, add_publisher_stake_caps); + if let Some(publisher_stake_caps_message) = compute_publisher_stake_caps(bank, &accounts) { + info!("PublisherStakeCaps: Adding publisher stake caps to the accumulator"); + v2_messages.push(publisher_stake_caps_message); + } for (pubkey, mut account) in accounts { let mut price_account_data = account.data().to_owned(); @@ -384,30 +386,40 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV } pub fn compute_publisher_stake_caps( - bank : &Bank, - accounts : &[(Pubkey, AccountSharedData)], - timestamp : i64, - messages : &mut Vec>, - is_active : bool -) { + bank: &Bank, + accounts: &[(Pubkey, AccountSharedData)], +) -> Option> { let parameters: StakeCapParameters = { let data = bank - .get_account_with_fixed_root(&STAKE_CAPS_PARAMETERS_ADDR) - .unwrap_or_default(); - let data = data.data(); - solana_sdk::borsh::try_from_slice_unchecked(data) - .unwrap_or(StakeCapParameters { _discriminator: 0, _current_authority : [0u8;32], z: 1, m : 1_000_000_000 }) -}; + .get_account_with_fixed_root(&STAKE_CAPS_PARAMETERS_ADDR) + .unwrap_or_default(); + let data = data.data(); + solana_sdk::borsh::try_from_slice_unchecked(data).unwrap_or(StakeCapParameters { + _discriminator: 0, + _current_authority: [0u8; 32], + z: 1, + m: 1_000_000_000, + }) + }; - info!("Computing publisher stake caps with m : {} and z : {}", parameters.m, parameters.z); - let account_datas : Vec<&[u8]> = accounts - .iter() - .map(|(_, account)| account.data()) - .collect(); - let message = pyth_oracle::validator::compute_publisher_stake_caps(account_datas, timestamp, parameters.m, parameters.z); + info!( + "PublisherStakeCaps: Computing publisher stake caps with m : {} and z : {}", + parameters.m, parameters.z + ); + let account_datas: Vec<&[u8]> = accounts.iter().map(|(_, account)| account.data()).collect(); + let message = pyth_oracle::validator::compute_publisher_stake_caps( + account_datas, + bank.clock().unix_timestamp, + parameters.m, + parameters.z, + ); - if is_active { - info!("Adding publisher stake caps to the accumulator"); - messages.push(message); + if bank + .feature_set + .is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) + { + return Some(message); + } else { + return None; } } From c2346621077a6525b110b38aaaf8a3fc1f3ea1ff Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 19:15:32 +0100 Subject: [PATCH 31/46] cleanup --- runtime/src/bank/pyth_accumulator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index ecd91dc1b3..ef0f290f8a 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -136,7 +136,7 @@ pub fn get_accumulator_keys() -> Vec<( ] } -pub fn update_v1<'a>( +pub fn update_v1( bank: &Bank, v2_messages: &[Vec], use_message_buffers: bool, From 977fa3a6c74d1707d30c67bf1913ec273c4f0e1b Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 19:35:24 +0100 Subject: [PATCH 32/46] fix: then some --- runtime/src/bank/pyth_accumulator.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index ef0f290f8a..46e0ad437e 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -414,12 +414,7 @@ pub fn compute_publisher_stake_caps( parameters.z, ); - if bank + bank .feature_set - .is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) - { - return Some(message); - } else { - return None; - } + .is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()).then_some(message) } From 7e7e4ebd689a400d03bec9d74afbff8f79c162d2 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 19:45:29 +0100 Subject: [PATCH 33/46] fix: use default --- Cargo.lock | 8 ++++---- runtime/Cargo.toml | 2 +- runtime/src/bank/pyth_accumulator.rs | 10 ++-------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2a9d0d0f86..11fa14e45a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3599,8 +3599,8 @@ dependencies = [ [[package]] name = "pyth-oracle" -version = "2.26.0" -source = "git+https://github.com/pyth-network/pyth-client.git?branch=feat/publisher-caps#fe64764a92e7212fc9b737070b7d3e288bcc4730" +version = "2.32.0" +source = "git+https://github.com/pyth-network/pyth-client.git?branch=feat/publisher-caps#f6de9ed368a7aa0740298740c8585cf82273c9fc" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", @@ -3617,7 +3617,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "1.13.6" -source = "git+https://github.com/pyth-network/pyth-crosschain?rev=a4fafa5dacd076a4d0656549d9a182bfba07f415#a4fafa5dacd076a4d0656549d9a182bfba07f415" +source = "git+https://github.com/pyth-network/pyth-crosschain?rev=33f901aa45f4f0005aa5a84a1479b78ca9033074#33f901aa45f4f0005aa5a84a1479b78ca9033074" dependencies = [ "bincode", "borsh 0.9.3", @@ -3635,7 +3635,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "2.1.0" -source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#76a9a50e51eed23fa123af08db7b1f79d708544e" +source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#a9e93d96418101b3caa8e7ffaaef21d344184eb9" dependencies = [ "bincode", "borsh 0.10.3", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 8f415d8bb0..c9f9aa502d 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -36,7 +36,7 @@ num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" pyth-oracle = { git = "https://github.com/pyth-network/pyth-client.git" ,branch="feat/publisher-caps" , features = ["library"] } -pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "a4fafa5dacd076a4d0656549d9a182bfba07f415" } +pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "33f901aa45f4f0005aa5a84a1479b78ca9033074" } rand = "0.7.0" rayon = "1.5.3" regex = "1.5.6" diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 46e0ad437e..6c34ca83ef 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -394,21 +394,15 @@ pub fn compute_publisher_stake_caps( .get_account_with_fixed_root(&STAKE_CAPS_PARAMETERS_ADDR) .unwrap_or_default(); let data = data.data(); - solana_sdk::borsh::try_from_slice_unchecked(data).unwrap_or(StakeCapParameters { - _discriminator: 0, - _current_authority: [0u8; 32], - z: 1, - m: 1_000_000_000, - }) + solana_sdk::borsh::try_from_slice_unchecked(data).unwrap_or_default() }; info!( "PublisherStakeCaps: Computing publisher stake caps with m : {} and z : {}", parameters.m, parameters.z ); - let account_datas: Vec<&[u8]> = accounts.iter().map(|(_, account)| account.data()).collect(); let message = pyth_oracle::validator::compute_publisher_stake_caps( - account_datas, + accounts.iter().map(|(_, account)| account.data()), bank.clock().unix_timestamp, parameters.m, parameters.z, From c274161f9fd17c3246709b8469c696d821213b06 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 19:49:43 +0100 Subject: [PATCH 34/46] fix: fmt --- runtime/src/bank/pyth_accumulator.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 6c34ca83ef..95208d6c28 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -1,9 +1,7 @@ -use std::borrow::Borrow; use { super::Bank, crate::accounts_index::{ScanConfig, ScanError}, - byteorder::LittleEndian, - byteorder::ReadBytesExt, + byteorder::{LittleEndian, ReadBytesExt}, log::*, pyth_oracle::validator::AggregationError, pythnet_sdk::{ @@ -18,7 +16,10 @@ use { hash::hashv, pubkey::Pubkey, }, - std::env::{self, VarError}, + std::{ + borrow::Borrow, + env::{self, VarError}, + }, }; pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; @@ -408,7 +409,7 @@ pub fn compute_publisher_stake_caps( parameters.z, ); - bank - .feature_set - .is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()).then_some(message) + bank.feature_set + .is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) + .then_some(message) } From 3d54ed203c1432899187107355ce9b1038a44e1c Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 19:53:53 +0100 Subject: [PATCH 35/46] go --- runtime/src/bank/pyth_accumulator.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 95208d6c28..d6c8a3786a 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -16,10 +16,7 @@ use { hash::hashv, pubkey::Pubkey, }, - std::{ - borrow::Borrow, - env::{self, VarError}, - }, + std::env::{self, VarError}, }; pub const ACCUMULATOR_RING_SIZE: u32 = 10_000; @@ -409,7 +406,12 @@ pub fn compute_publisher_stake_caps( parameters.z, ); - bank.feature_set + if bank + .feature_set .is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) - .then_some(message) + { + Some(message) + } else { + None + } } From a9893258b69dd9dca08f379003dd6281feb00d62 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 20:00:45 +0100 Subject: [PATCH 36/46] fix: cargo lock --- Cargo.lock | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11fa14e45a..e2eafbc969 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -519,7 +519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" dependencies = [ "borsh-derive 0.10.3", - "hashbrown 0.12.3", + "hashbrown 0.11.2", ] [[package]] @@ -627,18 +627,19 @@ checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" dependencies = [ "lazy_static", "memchr", - "regex-automata 0.1.10", + "regex-automata", "serde", ] [[package]] name = "bstr" -version = "1.6.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" dependencies = [ "memchr", - "regex-automata 0.3.9", + "once_cell", + "regex-automata", "serde", ] @@ -1364,9 +1365,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dyn-clone" -version = "1.0.17" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" [[package]] name = "eager" @@ -3976,12 +3977,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -[[package]] -name = "regex-automata" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" - [[package]] name = "regex-syntax" version = "0.6.26" @@ -4231,9 +4226,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.16" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" +checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" dependencies = [ "dyn-clone", "schemars_derive", @@ -4243,9 +4238,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.16" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" +checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", @@ -8209,7 +8204,7 @@ version = "0.1.0" source = "git+https://github.com/wormhole-foundation/wormhole?tag=v2.17.1#3e423a75180f9da69263279e9ffce47b1858ae78" dependencies = [ "anyhow", - "bstr 1.6.0", + "bstr 1.4.0", "schemars", "serde", "serde_wormhole", From 7ba154046cecbd27ea3dc92670581f696ed2aa6e Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 16:15:58 +0100 Subject: [PATCH 37/46] checkpoint --- runtime/src/bank/pyth_accumulator_tests.rs | 342 ++++++++++++++------- 1 file changed, 233 insertions(+), 109 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index e8736af9ec..17563cf405 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -14,14 +14,11 @@ use { byteorder::{ByteOrder, LittleEndian, ReadBytesExt}, itertools::Itertools, pyth_oracle::{ - solana_program::account_info::AccountInfo, PriceAccount, PriceAccountFlags, PythAccount, + solana_program::{account_info::AccountInfo, message}, PriceAccount, PriceAccountFlags, PythAccount, PythOracleSerialize, }, pythnet_sdk::{ - accumulators::{merkle::MerkleAccumulator, Accumulator}, - hashers::{keccak256_160::Keccak160, Hasher}, - wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, - ACCUMULATOR_EMITTER_ADDRESS, + accumulators::{merkle::MerkleAccumulator, Accumulator}, hashers::{keccak256_160::Keccak160, Hasher}, publisher_stake_caps::StakeCapParameters, wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, ACCUMULATOR_EMITTER_ADDRESS }, solana_sdk::{ account::{AccountSharedData, ReadableAccount, WritableAccount}, @@ -694,119 +691,76 @@ fn test_accumulator_v2_mixed() { test_accumulator_v2([true, true, false, false]); } -fn test_accumulator_v2(generate_buffers: [bool; 4]) { - let leader_pubkey = solana_sdk::pubkey::new_rand(); - let GenesisConfigInfo { - mut genesis_config, .. - } = create_genesis_config_with_leader(5, &leader_pubkey, 3); - - // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here - // due to slot 0 having special handling. - let slots_in_epoch = 32; - genesis_config.epoch_schedule = EpochSchedule::new(slots_in_epoch); - let mut bank = create_new_bank_for_tests_with_index(&genesis_config); +fn generate_price(bank : &Bank, seeds: &[u8], generate_buffers: bool, publishers : &[Pubkey]) -> (Pubkey, Vec>) { + let (price_feed_key, _bump) = Pubkey::find_program_address(&[seeds], &ORACLE_PID); + let mut price_feed_account = + AccountSharedData::new(42, size_of::(), &ORACLE_PID); + + let messages = { + let price_feed_info_key = &price_feed_key.to_bytes().into(); + let price_feed_info_lamports = &mut 0; + let price_feed_info_owner = &ORACLE_PID.to_bytes().into(); + let price_feed_info_data = price_feed_account.data_mut(); + let price_feed_info = AccountInfo::new( + price_feed_info_key, + false, + true, + price_feed_info_lamports, + price_feed_info_data, + price_feed_info_owner, + false, + Epoch::default(), + ); - let generate_price = |seeds, generate_buffers: bool| { - let (price_feed_key, _bump) = Pubkey::find_program_address(&[seeds], &ORACLE_PID); - let mut price_feed_account = - AccountSharedData::new(42, size_of::(), &ORACLE_PID); - - let messages = { - let price_feed_info_key = &price_feed_key.to_bytes().into(); - let price_feed_info_lamports = &mut 0; - let price_feed_info_owner = &ORACLE_PID.to_bytes().into(); - let price_feed_info_data = price_feed_account.data_mut(); - let price_feed_info = AccountInfo::new( - price_feed_info_key, - false, - true, - price_feed_info_lamports, - price_feed_info_data, - price_feed_info_owner, - false, - Epoch::default(), + let mut price_account = PriceAccount::initialize(&price_feed_info, 0).unwrap(); + if !generate_buffers { + price_account.flags.insert( + PriceAccountFlags::ACCUMULATOR_V2 | PriceAccountFlags::MESSAGE_BUFFER_CLEARED, ); - - let mut price_account = PriceAccount::initialize(&price_feed_info, 0).unwrap(); - if !generate_buffers { - price_account.flags.insert( - PriceAccountFlags::ACCUMULATOR_V2 | PriceAccountFlags::MESSAGE_BUFFER_CLEARED, - ); - } - - vec![ - price_account - .as_price_feed_message(&price_feed_key.to_bytes().into()) - .to_bytes(), - price_account - .as_twap_message(&price_feed_key.to_bytes().into()) - .to_bytes(), - ] - }; - - bank.store_account(&price_feed_key, &price_feed_account); - - if generate_buffers { - let message_buffer_bytes = create_message_buffer_bytes(messages.clone()); - - let mut seed = vec![1; 32]; - seed[..seeds.len()].copy_from_slice(seeds); - // Create a Message account. - let price_message_key = keypair_from_seed(&seed).unwrap(); - let mut price_message_account = bank - .get_account(&price_message_key.pubkey()) - .unwrap_or_default(); - - price_message_account.set_lamports(1_000_000_000); - price_message_account - .set_owner(Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID)); - price_message_account.set_data(message_buffer_bytes); - - // Store Message account so the accumulator sysvar updater can find it. - bank.store_account(&price_message_key.pubkey(), &price_message_account); + } + price_account.num_ = publishers.len() as u32; + for (i, publisher) in publishers.into_iter().enumerate() { + price_account.comp_[i].pub_ = publisher.to_bytes().into(); } - (price_feed_key, messages) + vec![ + price_account + .as_price_feed_message(&price_feed_key.to_bytes().into()) + .to_bytes(), + price_account + .as_twap_message(&price_feed_key.to_bytes().into()) + .to_bytes(), + ] }; - assert!(bank - .feature_set - .is_active(&feature_set::enable_accumulator_sysvar::id())); - assert!(bank - .feature_set - .is_active(&feature_set::move_accumulator_to_end_of_block::id())); - assert!(bank - .feature_set - .is_active(&feature_set::undo_move_accumulator_to_end_of_block::id())); - assert!(bank - .feature_set - .is_active(&feature_set::redo_move_accumulator_to_end_of_block::id())); - - let prices_with_messages = [ - generate_price(b"seeds_1", generate_buffers[0]), - generate_price(b"seeds_2", generate_buffers[1]), - generate_price(b"seeds_3", generate_buffers[2]), - generate_price(b"seeds_4", generate_buffers[3]), - ]; - - bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. - bank = new_from_parent(&Arc::new(bank)); // Advance slot 2. + bank.store_account(&price_feed_key, &price_feed_account); - let messages = prices_with_messages - .iter() - .flat_map(|(_, messages)| messages) - .map(|message| &message[..]) - .sorted_unstable() - .dedup() - .collect::>(); - assert_eq!(messages.len(), 8); + if generate_buffers { + let message_buffer_bytes = create_message_buffer_bytes(messages.clone()); + + let mut seed = vec![1; 32]; + seed[..seeds.len()].copy_from_slice(seeds); + // Create a Message account. + let price_message_key = keypair_from_seed(&seed).unwrap(); + let mut price_message_account = bank + .get_account(&price_message_key.pubkey()) + .unwrap_or_default(); + + price_message_account.set_lamports(1_000_000_000); + price_message_account + .set_owner(Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID)); + price_message_account.set_data(message_buffer_bytes); + + // Store Message account so the accumulator sysvar updater can find it. + bank.store_account(&price_message_key.pubkey(), &price_message_account); + + } - // Trigger Aggregation. We freeze instead of new_from_parent so - // we can keep access to the bank. - let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); - bank.freeze(); + (price_feed_key, messages) +} - // Get the wormhole message generated by freezed. We don't need +fn check_accumulator_state_matches_messages(bank: &Bank, sequence_tracker_before_bank_freeze: &AccumulatorSequenceTracker, messages: &[&[u8]]) { + // Get the wormhole message generated by freezed. We don't need // to offset the ring index as our test is always below 10K slots. let wormhole_message_account = get_wormhole_message_account(&bank, bank.slot() as u32); assert_ne!(wormhole_message_account.data().len(), 0); @@ -883,6 +837,176 @@ fn test_accumulator_v2(generate_buffers: [bool; 4]) { sequence_tracker_before_bank_freeze.sequence + 1 ); } + +fn test_accumulator_v2(generate_buffers: [bool; 4]) { + let leader_pubkey = solana_sdk::pubkey::new_rand(); + let GenesisConfigInfo { + mut genesis_config, .. + } = create_genesis_config_with_leader(5, &leader_pubkey, 3); + + genesis_config + .accounts + .remove(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) + .unwrap(); + + // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here + // due to slot 0 having special handling. + let slots_in_epoch = 32; + genesis_config.epoch_schedule = EpochSchedule::new(slots_in_epoch); + let mut bank = create_new_bank_for_tests_with_index(&genesis_config); + + assert!(bank + .feature_set + .is_active(&feature_set::enable_accumulator_sysvar::id())); + assert!(bank + .feature_set + .is_active(&feature_set::move_accumulator_to_end_of_block::id())); + assert!(bank + .feature_set + .is_active(&feature_set::undo_move_accumulator_to_end_of_block::id())); + assert!(bank + .feature_set + .is_active(&feature_set::redo_move_accumulator_to_end_of_block::id())); + + let prices_with_messages = [ + generate_price(&bank, b"seeds_1", generate_buffers[0], &[]), + generate_price(&bank, b"seeds_2", generate_buffers[1], &[]), + generate_price(&bank,b"seeds_3", generate_buffers[2], &[]), + generate_price(&bank,b"seeds_4", generate_buffers[3], &[]) + ]; + + bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. + bank = new_from_parent(&Arc::new(bank)); // Advance slot 2. + + let messages = prices_with_messages + .iter() + .flat_map(|(_, messages)| messages) + .map(|message| &message[..]) + .sorted_unstable() + .dedup() + .collect::>(); + assert_eq!(messages.len(), 8); + + // Trigger Aggregation. We freeze instead of new_from_parent so + // we can keep access to the bank. + let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); + bank.freeze(); + + check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); +} + +#[test] +fn test_publisher_stake_caps() { + let leader_pubkey = solana_sdk::pubkey::new_rand(); + let GenesisConfigInfo { + mut genesis_config, .. + } = create_genesis_config_with_leader(5, &leader_pubkey, 3); + + genesis_config + .accounts + .remove(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) + .unwrap(); + + // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here + // due to slot 0 having special handling. + let slots_in_epoch = 32; + genesis_config.epoch_schedule = EpochSchedule::new(slots_in_epoch); + let mut bank = create_new_bank_for_tests_with_index(&genesis_config); + + assert!(bank + .feature_set + .is_active(&feature_set::enable_accumulator_sysvar::id())); + assert!(bank + .feature_set + .is_active(&feature_set::move_accumulator_to_end_of_block::id())); + assert!(bank + .feature_set + .is_active(&feature_set::undo_move_accumulator_to_end_of_block::id())); + assert!(bank + .feature_set + .is_active(&feature_set::redo_move_accumulator_to_end_of_block::id())); + + let new_m = 1_000_000_000_000; + let new_z = 3; + + let mut publishers_with_expected_caps : [(Pubkey, u64, u64) ; 4] = [ + (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 5 / 4, new_m / 3 + new_m / 4), + (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 3 / 4, new_m / 3 + new_m / 4), + (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 3 / 4, new_m / 3 + new_m / 4), + (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 5 / 4, new_m / 3 + new_m / 4) + ]; + + let expected_caps = [[ + StakeCapParameters::default().m * 5 / 4, + StakeCapParameters::default().m * 3 / 4, + StakeCapParameters::default().m * 2 / 4, + StakeCapParameters::default().m * 1 / 4, + ],[new_m / 3 + new_m / 4,new_m / 3 + new_m / 4,new_m / 3 + new_m / 4,new_m / 3 + new_m / 4 ]]; + + let prices_with_messages = [ + generate_price(&bank, b"seeds_1", false, &[publishers_with_expected_caps[0].0]), + generate_price(&bank, b"seeds_2", false, &[publishers_with_expected_caps[1].0, publishers_with_expected_caps[2].0]), + generate_price(&bank,b"seeds_3", true, &[publishers_with_expected_caps[3].0]), + generate_price(&bank,b"seeds_4", true, &[publishers_with_expected_caps[3].0, publishers_with_expected_caps[1].0, publishers_with_expected_caps[0].0, publishers_with_expected_caps[2].0]) + ]; + + publishers_with_expected_caps.sort_by_key(|(pk,_,_)| *pk); + + bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. + bank = new_from_parent(&Arc::new(bank)); // Advance slot 2. + + + let mut messages = prices_with_messages + .iter() + .flat_map(|(_, messages)| messages) + .map(|message| &message[..]) + .sorted_unstable() + .dedup() + .collect::>(); + assert_eq!(messages.len(), 8); + + let feature_id = feature_set::add_publisher_stake_caps_to_the_accumulator::id(); + let feature = Feature { + activated_at: Some(30), + }; + bank.store_account(&feature_id, &feature::create_account(&feature, 42)); + + bank.compute_active_feature_set(true); + + // Trigger Aggregation. We freeze instead of new_from_parent so + // we can keep access to the bank. + let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); + bank.freeze(); + + check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); + + // Enable Publisher Stake Caps + + for _ in 0..slots_in_epoch { + bank = new_from_parent(&Arc::new(bank)); + } + + let publisher_caps_message = { + let mut result = vec![]; + result.extend_from_slice(&bank.unix_timestamp_from_genesis().to_be_bytes()); + result.extend_from_slice(&4u16.to_be_bytes()); + for (pk, m, _) in publishers_with_expected_caps { + result.extend_from_slice(&pk.to_bytes()); + result.extend_from_slice(&m.to_be_bytes()); } + result + }; + + + messages.push(&publisher_caps_message); + + let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); + bank.freeze(); + check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); + + +} + + #[test] fn test_get_accumulator_keys() { use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; From 8e49c443ec6994446bd7d606797f6be35e7ea03e Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 16:50:47 +0100 Subject: [PATCH 38/46] test: add test --- runtime/src/bank/pyth_accumulator_tests.rs | 27 +++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 17563cf405..61c1a54770 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -6,7 +6,7 @@ use { AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude, }, bank::{ - pyth_accumulator::{get_accumulator_keys, ACCUMULATOR_RING_SIZE, ORACLE_PID}, + pyth_accumulator::{get_accumulator_keys, ACCUMULATOR_RING_SIZE, ORACLE_PID, STAKE_CAPS_PARAMETERS_ADDR}, Bank, }, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, @@ -987,8 +987,8 @@ fn test_publisher_stake_caps() { } let publisher_caps_message = { - let mut result = vec![]; - result.extend_from_slice(&bank.unix_timestamp_from_genesis().to_be_bytes()); + let mut result = vec![2]; + result.extend_from_slice(&bank.clock().unix_timestamp.to_be_bytes()); result.extend_from_slice(&4u16.to_be_bytes()); for (pk, m, _) in publishers_with_expected_caps { result.extend_from_slice(&pk.to_bytes()); @@ -1002,8 +1002,29 @@ fn test_publisher_stake_caps() { let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); + bank = new_from_parent(&Arc::new(bank)); + // Now store the stake cap parameters + let mut stake_cap_parameters_account = AccountSharedData::new(42, size_of::(), &ORACLE_PID); + stake_cap_parameters_account.set_data(StakeCapParameters{ _discriminator: 0, _current_authority: [0u8;32], m: new_m, z: new_z }.try_to_vec().unwrap()); + bank.store_account(&STAKE_CAPS_PARAMETERS_ADDR, &stake_cap_parameters_account); + let publisher_caps_message_with_new_parameters = { + let mut result = vec![2]; + result.extend_from_slice(&bank.clock().unix_timestamp.to_be_bytes()); + result.extend_from_slice(&4u16.to_be_bytes()); + for (pk, _, m) in publishers_with_expected_caps { + result.extend_from_slice(&pk.to_bytes()); + result.extend_from_slice(&m.to_be_bytes()); } + result + }; + + let last_element_index = messages.len() - 1; + messages[last_element_index] = &publisher_caps_message_with_new_parameters; + + let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); + bank.freeze(); + check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); } From 44a92ced06f825d503026240da536fa31e64d952 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 17:09:06 +0100 Subject: [PATCH 39/46] clean up --- runtime/src/bank/pyth_accumulator_tests.rs | 105 +++++++++------------ 1 file changed, 44 insertions(+), 61 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 61c1a54770..a4a0c01bae 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -102,6 +102,14 @@ fn get_accumulator_state(bank: &Bank, ring_index: u32) -> Vec { account.data().to_vec() } +fn activate_feature(bank: &mut Bank, feature_id: &Pubkey) { + let feature = Feature { + activated_at: Some(30), + }; + bank.store_account(&feature_id, &feature::create_account(&feature, 42)); + bank.compute_active_feature_set(true); +} + #[test] fn test_update_accumulator_sysvar() { let leader_pubkey = solana_sdk::pubkey::new_rand(); @@ -193,16 +201,10 @@ fn test_update_accumulator_sysvar() { assert_eq!(wormhole_message_account.data().len(), 0); // Enable Accumulator Feature (42 = random lamport balance, and the meaning of the universe). - let feature_id = feature_set::enable_accumulator_sysvar::id(); - let feature = Feature { - activated_at: Some(30), - }; - bank.store_account(&feature_id, &feature::create_account(&feature, 42)); - bank.compute_active_feature_set(true); - for _ in 0..slots_in_epoch { - bank = new_from_parent(&Arc::new(bank)); - } - + activate_feature(&mut bank, &feature_set::enable_accumulator_sysvar::id()); + for _ in 0..slots_in_epoch { + bank = new_from_parent(&Arc::new(bank)); + } // Feature should now be enabled on the new bank as the epoch has changed. assert!(bank .feature_set @@ -482,19 +484,8 @@ fn test_update_accumulator_end_of_block() { assert_eq!(wormhole_message_account.data().len(), 0); // Enable Accumulator Features (42 = random lamport balance, and the meaning of the universe). - let feature_id = feature_set::enable_accumulator_sysvar::id(); - let feature = Feature { - activated_at: Some(30), - }; - bank.store_account(&feature_id, &feature::create_account(&feature, 42)); - - let feature_id = feature_set::move_accumulator_to_end_of_block::id(); - let feature = Feature { - activated_at: Some(30), - }; - bank.store_account(&feature_id, &feature::create_account(&feature, 42)); - - bank.compute_active_feature_set(true); + activate_feature(&mut bank, & feature_set::enable_accumulator_sysvar::id()); + activate_feature(&mut bank, & feature_set::move_accumulator_to_end_of_block::id()); for _ in 0..slots_in_epoch { bank = new_from_parent(&Arc::new(bank)); } @@ -676,21 +667,6 @@ fn test_update_accumulator_end_of_block() { ); } -#[test] -fn test_accumulator_v2_all_v2() { - test_accumulator_v2([false, false, false, false]); -} - -#[test] -fn test_accumulator_v2_all_v1() { - test_accumulator_v2([true, true, true, true]); -} - -#[test] -fn test_accumulator_v2_mixed() { - test_accumulator_v2([true, true, false, false]); -} - fn generate_price(bank : &Bank, seeds: &[u8], generate_buffers: bool, publishers : &[Pubkey]) -> (Pubkey, Vec>) { let (price_feed_key, _bump) = Pubkey::find_program_address(&[seeds], &ORACLE_PID); let mut price_feed_account = @@ -838,6 +814,21 @@ fn check_accumulator_state_matches_messages(bank: &Bank, sequence_tracker_before ); } +#[test] +fn test_accumulator_v2_all_v2() { + test_accumulator_v2([false, false, false, false]); +} + +#[test] +fn test_accumulator_v2_all_v1() { + test_accumulator_v2([true, true, true, true]); +} + +#[test] +fn test_accumulator_v2_mixed() { + test_accumulator_v2([true, true, false, false]); +} + fn test_accumulator_v2(generate_buffers: [bool; 4]) { let leader_pubkey = solana_sdk::pubkey::new_rand(); let GenesisConfigInfo { @@ -926,9 +917,10 @@ fn test_publisher_stake_caps() { .feature_set .is_active(&feature_set::redo_move_accumulator_to_end_of_block::id())); - let new_m = 1_000_000_000_000; - let new_z = 3; - + // We will set the stake cap parameters to this later + let new_m = 1_000_000_000_000; + let new_z = 3; + // This is an array of tuples where each tuple contains the pubkey of the publisher, the expected cap with default values and the expected cap with the new values. let mut publishers_with_expected_caps : [(Pubkey, u64, u64) ; 4] = [ (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 5 / 4, new_m / 3 + new_m / 4), (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 3 / 4, new_m / 3 + new_m / 4), @@ -936,13 +928,6 @@ fn test_publisher_stake_caps() { (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 5 / 4, new_m / 3 + new_m / 4) ]; - let expected_caps = [[ - StakeCapParameters::default().m * 5 / 4, - StakeCapParameters::default().m * 3 / 4, - StakeCapParameters::default().m * 2 / 4, - StakeCapParameters::default().m * 1 / 4, - ],[new_m / 3 + new_m / 4,new_m / 3 + new_m / 4,new_m / 3 + new_m / 4,new_m / 3 + new_m / 4 ]]; - let prices_with_messages = [ generate_price(&bank, b"seeds_1", false, &[publishers_with_expected_caps[0].0]), generate_price(&bank, b"seeds_2", false, &[publishers_with_expected_caps[1].0, publishers_with_expected_caps[2].0]), @@ -950,6 +935,7 @@ fn test_publisher_stake_caps() { generate_price(&bank,b"seeds_4", true, &[publishers_with_expected_caps[3].0, publishers_with_expected_caps[1].0, publishers_with_expected_caps[0].0, publishers_with_expected_caps[2].0]) ]; + // Publishers are sorted in the publisher stake caps message so we sort them here too publishers_with_expected_caps.sort_by_key(|(pk,_,_)| *pk); bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. @@ -965,23 +951,15 @@ fn test_publisher_stake_caps() { .collect::>(); assert_eq!(messages.len(), 8); - let feature_id = feature_set::add_publisher_stake_caps_to_the_accumulator::id(); - let feature = Feature { - activated_at: Some(30), - }; - bank.store_account(&feature_id, &feature::create_account(&feature, 42)); + // We turn on the feature to add publisher stake caps to the accumulator but it won't be active until next epoch + activate_feature(&mut bank, &feature_set::add_publisher_stake_caps_to_the_accumulator::id()); - bank.compute_active_feature_set(true); - - // Trigger Aggregation. We freeze instead of new_from_parent so + // Trigger accumulation. We freeze instead of new_from_parent so // we can keep access to the bank. let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); - check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); - // Enable Publisher Stake Caps - for _ in 0..slots_in_epoch { bank = new_from_parent(&Arc::new(bank)); } @@ -996,19 +974,22 @@ fn test_publisher_stake_caps() { result }; - + // Now the messages contain the publisher caps message messages.push(&publisher_caps_message); + assert_eq!(messages.len(), 9); let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); + bank = new_from_parent(&Arc::new(bank)); - // Now store the stake cap parameters + // Now we update the stake cap parameters let mut stake_cap_parameters_account = AccountSharedData::new(42, size_of::(), &ORACLE_PID); stake_cap_parameters_account.set_data(StakeCapParameters{ _discriminator: 0, _current_authority: [0u8;32], m: new_m, z: new_z }.try_to_vec().unwrap()); bank.store_account(&STAKE_CAPS_PARAMETERS_ADDR, &stake_cap_parameters_account); + let publisher_caps_message_with_new_parameters = { let mut result = vec![2]; result.extend_from_slice(&bank.clock().unix_timestamp.to_be_bytes()); @@ -1019,8 +1000,10 @@ fn test_publisher_stake_caps() { result }; + // Update the publisher caps message in the message arrays to match the new parameters let last_element_index = messages.len() - 1; messages[last_element_index] = &publisher_caps_message_with_new_parameters; + assert_eq!(messages.len(), 9); let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); From ca68105ef83fab1dd088c87a1c131317b587de45 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 17:11:18 +0100 Subject: [PATCH 40/46] fmt --- runtime/src/bank/pyth_accumulator_tests.rs | 183 +++++++++++++++------ validator/src/main.rs | 14 +- 2 files changed, 145 insertions(+), 52 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index a4a0c01bae..b7452b446f 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -6,7 +6,9 @@ use { AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude, }, bank::{ - pyth_accumulator::{get_accumulator_keys, ACCUMULATOR_RING_SIZE, ORACLE_PID, STAKE_CAPS_PARAMETERS_ADDR}, + pyth_accumulator::{ + get_accumulator_keys, ACCUMULATOR_RING_SIZE, ORACLE_PID, STAKE_CAPS_PARAMETERS_ADDR, + }, Bank, }, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, @@ -14,11 +16,15 @@ use { byteorder::{ByteOrder, LittleEndian, ReadBytesExt}, itertools::Itertools, pyth_oracle::{ - solana_program::{account_info::AccountInfo, message}, PriceAccount, PriceAccountFlags, PythAccount, - PythOracleSerialize, + solana_program::{account_info::AccountInfo, message}, + PriceAccount, PriceAccountFlags, PythAccount, PythOracleSerialize, }, pythnet_sdk::{ - accumulators::{merkle::MerkleAccumulator, Accumulator}, hashers::{keccak256_160::Keccak160, Hasher}, publisher_stake_caps::StakeCapParameters, wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, ACCUMULATOR_EMITTER_ADDRESS + accumulators::{merkle::MerkleAccumulator, Accumulator}, + hashers::{keccak256_160::Keccak160, Hasher}, + publisher_stake_caps::StakeCapParameters, + wormhole::{AccumulatorSequenceTracker, MessageData, PostedMessageUnreliableData}, + ACCUMULATOR_EMITTER_ADDRESS, }, solana_sdk::{ account::{AccountSharedData, ReadableAccount, WritableAccount}, @@ -202,9 +208,9 @@ fn test_update_accumulator_sysvar() { // Enable Accumulator Feature (42 = random lamport balance, and the meaning of the universe). activate_feature(&mut bank, &feature_set::enable_accumulator_sysvar::id()); - for _ in 0..slots_in_epoch { - bank = new_from_parent(&Arc::new(bank)); - } + for _ in 0..slots_in_epoch { + bank = new_from_parent(&Arc::new(bank)); + } // Feature should now be enabled on the new bank as the epoch has changed. assert!(bank .feature_set @@ -484,8 +490,11 @@ fn test_update_accumulator_end_of_block() { assert_eq!(wormhole_message_account.data().len(), 0); // Enable Accumulator Features (42 = random lamport balance, and the meaning of the universe). - activate_feature(&mut bank, & feature_set::enable_accumulator_sysvar::id()); - activate_feature(&mut bank, & feature_set::move_accumulator_to_end_of_block::id()); + activate_feature(&mut bank, &feature_set::enable_accumulator_sysvar::id()); + activate_feature( + &mut bank, + &feature_set::move_accumulator_to_end_of_block::id(), + ); for _ in 0..slots_in_epoch { bank = new_from_parent(&Arc::new(bank)); } @@ -667,10 +676,14 @@ fn test_update_accumulator_end_of_block() { ); } -fn generate_price(bank : &Bank, seeds: &[u8], generate_buffers: bool, publishers : &[Pubkey]) -> (Pubkey, Vec>) { +fn generate_price( + bank: &Bank, + seeds: &[u8], + generate_buffers: bool, + publishers: &[Pubkey], +) -> (Pubkey, Vec>) { let (price_feed_key, _bump) = Pubkey::find_program_address(&[seeds], &ORACLE_PID); - let mut price_feed_account = - AccountSharedData::new(42, size_of::(), &ORACLE_PID); + let mut price_feed_account = AccountSharedData::new(42, size_of::(), &ORACLE_PID); let messages = { let price_feed_info_key = &price_feed_key.to_bytes().into(); @@ -723,20 +736,22 @@ fn generate_price(bank : &Bank, seeds: &[u8], generate_buffers: bool, publishers .unwrap_or_default(); price_message_account.set_lamports(1_000_000_000); - price_message_account - .set_owner(Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID)); + price_message_account.set_owner(Pubkey::new_from_array(pythnet_sdk::MESSAGE_BUFFER_PID)); price_message_account.set_data(message_buffer_bytes); // Store Message account so the accumulator sysvar updater can find it. bank.store_account(&price_message_key.pubkey(), &price_message_account); - } (price_feed_key, messages) } -fn check_accumulator_state_matches_messages(bank: &Bank, sequence_tracker_before_bank_freeze: &AccumulatorSequenceTracker, messages: &[&[u8]]) { - // Get the wormhole message generated by freezed. We don't need +fn check_accumulator_state_matches_messages( + bank: &Bank, + sequence_tracker_before_bank_freeze: &AccumulatorSequenceTracker, + messages: &[&[u8]], +) { + // Get the wormhole message generated by freezed. We don't need // to offset the ring index as our test is always below 10K slots. let wormhole_message_account = get_wormhole_message_account(&bank, bank.slot() as u32); assert_ne!(wormhole_message_account.data().len(), 0); @@ -836,9 +851,9 @@ fn test_accumulator_v2(generate_buffers: [bool; 4]) { } = create_genesis_config_with_leader(5, &leader_pubkey, 3); genesis_config - .accounts - .remove(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) - .unwrap(); + .accounts + .remove(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) + .unwrap(); // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here // due to slot 0 having special handling. @@ -862,8 +877,8 @@ fn test_accumulator_v2(generate_buffers: [bool; 4]) { let prices_with_messages = [ generate_price(&bank, b"seeds_1", generate_buffers[0], &[]), generate_price(&bank, b"seeds_2", generate_buffers[1], &[]), - generate_price(&bank,b"seeds_3", generate_buffers[2], &[]), - generate_price(&bank,b"seeds_4", generate_buffers[3], &[]) + generate_price(&bank, b"seeds_3", generate_buffers[2], &[]), + generate_price(&bank, b"seeds_4", generate_buffers[3], &[]), ]; bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. @@ -883,7 +898,11 @@ fn test_accumulator_v2(generate_buffers: [bool; 4]) { let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); - check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); + check_accumulator_state_matches_messages( + &bank, + &sequence_tracker_before_bank_freeze, + &messages, + ); } #[test] @@ -894,9 +913,9 @@ fn test_publisher_stake_caps() { } = create_genesis_config_with_leader(5, &leader_pubkey, 3); genesis_config - .accounts - .remove(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) - .unwrap(); + .accounts + .remove(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) + .unwrap(); // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here // due to slot 0 having special handling. @@ -918,30 +937,73 @@ fn test_publisher_stake_caps() { .is_active(&feature_set::redo_move_accumulator_to_end_of_block::id())); // We will set the stake cap parameters to this later - let new_m = 1_000_000_000_000; + let new_m = 1_000_000_000_000; let new_z = 3; // This is an array of tuples where each tuple contains the pubkey of the publisher, the expected cap with default values and the expected cap with the new values. - let mut publishers_with_expected_caps : [(Pubkey, u64, u64) ; 4] = [ - (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 5 / 4, new_m / 3 + new_m / 4), - (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 3 / 4, new_m / 3 + new_m / 4), - (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 3 / 4, new_m / 3 + new_m / 4), - (solana_sdk::pubkey::new_rand(),StakeCapParameters::default().m * 5 / 4, new_m / 3 + new_m / 4) + let mut publishers_with_expected_caps: [(Pubkey, u64, u64); 4] = [ + ( + solana_sdk::pubkey::new_rand(), + StakeCapParameters::default().m * 5 / 4, + new_m / 3 + new_m / 4, + ), + ( + solana_sdk::pubkey::new_rand(), + StakeCapParameters::default().m * 3 / 4, + new_m / 3 + new_m / 4, + ), + ( + solana_sdk::pubkey::new_rand(), + StakeCapParameters::default().m * 3 / 4, + new_m / 3 + new_m / 4, + ), + ( + solana_sdk::pubkey::new_rand(), + StakeCapParameters::default().m * 5 / 4, + new_m / 3 + new_m / 4, + ), ]; let prices_with_messages = [ - generate_price(&bank, b"seeds_1", false, &[publishers_with_expected_caps[0].0]), - generate_price(&bank, b"seeds_2", false, &[publishers_with_expected_caps[1].0, publishers_with_expected_caps[2].0]), - generate_price(&bank,b"seeds_3", true, &[publishers_with_expected_caps[3].0]), - generate_price(&bank,b"seeds_4", true, &[publishers_with_expected_caps[3].0, publishers_with_expected_caps[1].0, publishers_with_expected_caps[0].0, publishers_with_expected_caps[2].0]) + generate_price( + &bank, + b"seeds_1", + false, + &[publishers_with_expected_caps[0].0], + ), + generate_price( + &bank, + b"seeds_2", + false, + &[ + publishers_with_expected_caps[1].0, + publishers_with_expected_caps[2].0, + ], + ), + generate_price( + &bank, + b"seeds_3", + true, + &[publishers_with_expected_caps[3].0], + ), + generate_price( + &bank, + b"seeds_4", + true, + &[ + publishers_with_expected_caps[3].0, + publishers_with_expected_caps[1].0, + publishers_with_expected_caps[0].0, + publishers_with_expected_caps[2].0, + ], + ), ]; // Publishers are sorted in the publisher stake caps message so we sort them here too - publishers_with_expected_caps.sort_by_key(|(pk,_,_)| *pk); + publishers_with_expected_caps.sort_by_key(|(pk, _, _)| *pk); bank = new_from_parent(&Arc::new(bank)); // Advance slot 1. bank = new_from_parent(&Arc::new(bank)); // Advance slot 2. - let mut messages = prices_with_messages .iter() .flat_map(|(_, messages)| messages) @@ -952,13 +1014,20 @@ fn test_publisher_stake_caps() { assert_eq!(messages.len(), 8); // We turn on the feature to add publisher stake caps to the accumulator but it won't be active until next epoch - activate_feature(&mut bank, &feature_set::add_publisher_stake_caps_to_the_accumulator::id()); + activate_feature( + &mut bank, + &feature_set::add_publisher_stake_caps_to_the_accumulator::id(), + ); // Trigger accumulation. We freeze instead of new_from_parent so // we can keep access to the bank. let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); - check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); + check_accumulator_state_matches_messages( + &bank, + &sequence_tracker_before_bank_freeze, + &messages, + ); for _ in 0..slots_in_epoch { bank = new_from_parent(&Arc::new(bank)); @@ -970,7 +1039,8 @@ fn test_publisher_stake_caps() { result.extend_from_slice(&4u16.to_be_bytes()); for (pk, m, _) in publishers_with_expected_caps { result.extend_from_slice(&pk.to_bytes()); - result.extend_from_slice(&m.to_be_bytes()); } + result.extend_from_slice(&m.to_be_bytes()); + } result }; @@ -980,23 +1050,37 @@ fn test_publisher_stake_caps() { let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); - check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); + check_accumulator_state_matches_messages( + &bank, + &sequence_tracker_before_bank_freeze, + &messages, + ); bank = new_from_parent(&Arc::new(bank)); // Now we update the stake cap parameters - let mut stake_cap_parameters_account = AccountSharedData::new(42, size_of::(), &ORACLE_PID); - stake_cap_parameters_account.set_data(StakeCapParameters{ _discriminator: 0, _current_authority: [0u8;32], m: new_m, z: new_z }.try_to_vec().unwrap()); + let mut stake_cap_parameters_account = + AccountSharedData::new(42, size_of::(), &ORACLE_PID); + stake_cap_parameters_account.set_data( + StakeCapParameters { + _discriminator: 0, + _current_authority: [0u8; 32], + m: new_m, + z: new_z, + } + .try_to_vec() + .unwrap(), + ); bank.store_account(&STAKE_CAPS_PARAMETERS_ADDR, &stake_cap_parameters_account); - let publisher_caps_message_with_new_parameters = { let mut result = vec![2]; result.extend_from_slice(&bank.clock().unix_timestamp.to_be_bytes()); result.extend_from_slice(&4u16.to_be_bytes()); for (pk, _, m) in publishers_with_expected_caps { result.extend_from_slice(&pk.to_bytes()); - result.extend_from_slice(&m.to_be_bytes()); } + result.extend_from_slice(&m.to_be_bytes()); + } result }; @@ -1007,10 +1091,13 @@ fn test_publisher_stake_caps() { let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); - check_accumulator_state_matches_messages(&bank, &sequence_tracker_before_bank_freeze, &messages); + check_accumulator_state_matches_messages( + &bank, + &sequence_tracker_before_bank_freeze, + &messages, + ); } - #[test] fn test_get_accumulator_keys() { use pythnet_sdk::{pythnet, ACCUMULATOR_EMITTER_ADDRESS, MESSAGE_BUFFER_PID}; diff --git a/validator/src/main.rs b/validator/src/main.rs index 13d510e8d2..711c3b7806 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -3177,9 +3177,11 @@ fn process_account_indexes(matches: &ArgMatches) -> AccountSecondaryIndexes { }) .collect(); - assert!(account_indexes.contains(&AccountIndex::ProgramId), + assert!( + account_indexes.contains(&AccountIndex::ProgramId), "The indexing should be enabled for program-id accounts. Add the following flag:\n\ - --account-index program-id\n"); + --account-index program-id\n" + ); let account_indexes_include_keys: HashSet = values_t!(matches, "account_index_include_key", Pubkey) @@ -3199,7 +3201,9 @@ fn process_account_indexes(matches: &ArgMatches) -> AccountSecondaryIndexes { let include_keys = !account_indexes_include_keys.is_empty(); if include_keys { - if !account_indexes_include_keys.contains(&*ORACLE_PID) || !account_indexes_include_keys.contains(&*MESSAGE_BUFFER_PID) { + if !account_indexes_include_keys.contains(&*ORACLE_PID) + || !account_indexes_include_keys.contains(&*MESSAGE_BUFFER_PID) + { panic!( "The oracle program id and message buffer program id must be included in the account index. Add the following flags\n\ --account-index-include-key {}\n\ @@ -3210,7 +3214,9 @@ fn process_account_indexes(matches: &ArgMatches) -> AccountSecondaryIndexes { } if exclude_keys { - if account_indexes_exclude_keys.contains(&*ORACLE_PID) || account_indexes_exclude_keys.contains(&*MESSAGE_BUFFER_PID) { + if account_indexes_exclude_keys.contains(&*ORACLE_PID) + || account_indexes_exclude_keys.contains(&*MESSAGE_BUFFER_PID) + { panic!("The oracle program id and message buffer program id must *not* be excluded from the account index."); } } From 430b685da8cb641c6b7cf39459df345c2404630e Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 17:14:32 +0100 Subject: [PATCH 41/46] clippy --- runtime/src/bank/pyth_accumulator_tests.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index b7452b446f..dbf4ffd282 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -16,7 +16,7 @@ use { byteorder::{ByteOrder, LittleEndian, ReadBytesExt}, itertools::Itertools, pyth_oracle::{ - solana_program::{account_info::AccountInfo, message}, + solana_program::{account_info::AccountInfo}, PriceAccount, PriceAccountFlags, PythAccount, PythOracleSerialize, }, pythnet_sdk::{ @@ -112,7 +112,7 @@ fn activate_feature(bank: &mut Bank, feature_id: &Pubkey) { let feature = Feature { activated_at: Some(30), }; - bank.store_account(&feature_id, &feature::create_account(&feature, 42)); + bank.store_account(feature_id, &feature::create_account(&feature, 42)); bank.compute_active_feature_set(true); } @@ -708,7 +708,7 @@ fn generate_price( ); } price_account.num_ = publishers.len() as u32; - for (i, publisher) in publishers.into_iter().enumerate() { + for (i, publisher) in publishers.iter().enumerate() { price_account.comp_[i].pub_ = publisher.to_bytes().into(); } @@ -753,7 +753,7 @@ fn check_accumulator_state_matches_messages( ) { // Get the wormhole message generated by freezed. We don't need // to offset the ring index as our test is always below 10K slots. - let wormhole_message_account = get_wormhole_message_account(&bank, bank.slot() as u32); + let wormhole_message_account = get_wormhole_message_account(bank, bank.slot() as u32); assert_ne!(wormhole_message_account.data().len(), 0); let deserialized_wormhole_message = PostedMessageUnreliableData::deserialize(&mut wormhole_message_account.data()).unwrap(); @@ -797,7 +797,7 @@ fn check_accumulator_state_matches_messages( } // Verify accumulator state account. - let accumulator_state = get_accumulator_state(&bank, bank.slot() as u32); + let accumulator_state = get_accumulator_state(bank, bank.slot() as u32); let acc_state_magic = &accumulator_state[..4]; let acc_state_slot = LittleEndian::read_u64(&accumulator_state[4..12]); let acc_state_ring_size = LittleEndian::read_u32(&accumulator_state[12..16]); @@ -824,7 +824,7 @@ fn check_accumulator_state_matches_messages( // Verify sequence_tracker increments for wormhole to accept it. assert_eq!( - get_acc_sequence_tracker(&bank).sequence, + get_acc_sequence_tracker(bank).sequence, sequence_tracker_before_bank_freeze.sequence + 1 ); } From 9e3a8035cb4e2eb3927d9c5f151c2cc2811a6216 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 17:23:21 +0100 Subject: [PATCH 42/46] add measure --- runtime/src/bank/pyth_accumulator.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index f5f937e743..db18505d90 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -419,7 +419,6 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV measure.as_us() ); - let mut measure = Measure::start("update_v2_aggregate_price"); let mut any_v1_aggregations = false; let mut v2_messages = Vec::new(); @@ -429,6 +428,8 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV v2_messages.push(publisher_stake_caps_message); } + let mut measure = Measure::start("update_v2_aggregate_price"); + for (pubkey, mut account) in accounts { let mut price_account_data = account.data().to_owned(); @@ -467,6 +468,8 @@ pub fn compute_publisher_stake_caps( bank: &Bank, accounts: &[(Pubkey, AccountSharedData)], ) -> Option> { + let mut measure = Measure::start("compute_publisher_stake_caps"); + let parameters: StakeCapParameters = { let data = bank .get_account_with_fixed_root(&STAKE_CAPS_PARAMETERS_ADDR) @@ -475,10 +478,6 @@ pub fn compute_publisher_stake_caps( solana_sdk::borsh::try_from_slice_unchecked(data).unwrap_or_default() }; - info!( - "PublisherStakeCaps: Computing publisher stake caps with m : {} and z : {}", - parameters.m, parameters.z - ); let message = pyth_oracle::validator::compute_publisher_stake_caps( accounts.iter().map(|(_, account)| account.data()), bank.clock().unix_timestamp, @@ -486,6 +485,12 @@ pub fn compute_publisher_stake_caps( parameters.z, ); + measure.stop(); + info!( + "PublisherStakeCaps: Computed publisher stake caps with m : {} and z : {} in {} us", + parameters.m, parameters.z, measure.as_us() + ); + if bank .feature_set .is_active(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) From 49777525cf04ea80a8fe7415db51a0840821369a Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 17:32:28 +0100 Subject: [PATCH 43/46] fix: other tests --- runtime/src/bank/pyth_accumulator.rs | 5 +++-- runtime/src/bank/pyth_accumulator_tests.rs | 13 +++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index db18505d90..8944a82aee 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -419,7 +419,6 @@ pub fn update_v2(bank: &Bank) -> std::result::Result<(), AccumulatorUpdateErrorV measure.as_us() ); - let mut any_v1_aggregations = false; let mut v2_messages = Vec::new(); @@ -488,7 +487,9 @@ pub fn compute_publisher_stake_caps( measure.stop(); info!( "PublisherStakeCaps: Computed publisher stake caps with m : {} and z : {} in {} us", - parameters.m, parameters.z, measure.as_us() + parameters.m, + parameters.z, + measure.as_us() ); if bank diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index dbf4ffd282..1e4c943ce6 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -16,8 +16,8 @@ use { byteorder::{ByteOrder, LittleEndian, ReadBytesExt}, itertools::Itertools, pyth_oracle::{ - solana_program::{account_info::AccountInfo}, - PriceAccount, PriceAccountFlags, PythAccount, PythOracleSerialize, + solana_program::account_info::AccountInfo, PriceAccount, PriceAccountFlags, PythAccount, + PythOracleSerialize, }, pythnet_sdk::{ accumulators::{merkle::MerkleAccumulator, Accumulator}, @@ -134,6 +134,10 @@ fn test_update_accumulator_sysvar() { .accounts .remove(&feature_set::move_accumulator_to_end_of_block::id()) .unwrap(); + genesis_config + .accounts + .remove(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) + .unwrap(); // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here // due to slot 0 having special handling. @@ -413,6 +417,10 @@ fn test_update_accumulator_end_of_block() { .accounts .remove(&feature_set::move_accumulator_to_end_of_block::id()) .unwrap(); + genesis_config + .accounts + .remove(&feature_set::add_publisher_stake_caps_to_the_accumulator::id()) + .unwrap(); // Set epoch length to 32 so we can advance epochs quickly. We also skip past slot 0 here // due to slot 0 having special handling. @@ -1111,6 +1119,7 @@ fn test_get_accumulator_keys() { Pubkey::new_from_array(pythnet::ACCUMULATOR_SEQUENCE_ADDR), Pubkey::new_from_array(pythnet::WORMHOLE_PID), *ORACLE_PID, + *STAKE_CAPS_PARAMETERS_ADDR, ]; assert_eq!(accumulator_keys, expected_pyth_keys); } From 804302057ffab60e5c61dccec98325ed78afa3a3 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 17:42:29 +0100 Subject: [PATCH 44/46] fix: add bad stake parameters --- runtime/src/bank/pyth_accumulator_tests.rs | 27 +++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/runtime/src/bank/pyth_accumulator_tests.rs b/runtime/src/bank/pyth_accumulator_tests.rs index 1e4c943ce6..68a1bfe0b9 100644 --- a/runtime/src/bank/pyth_accumulator_tests.rs +++ b/runtime/src/bank/pyth_accumulator_tests.rs @@ -1004,6 +1004,9 @@ fn test_publisher_stake_caps() { publishers_with_expected_caps[2].0, ], ), + generate_price(&bank, b"seeds_5", false, &[]), + + ]; // Publishers are sorted in the publisher stake caps message so we sort them here too @@ -1019,7 +1022,7 @@ fn test_publisher_stake_caps() { .sorted_unstable() .dedup() .collect::>(); - assert_eq!(messages.len(), 8); + assert_eq!(messages.len(), 10); // We turn on the feature to add publisher stake caps to the accumulator but it won't be active until next epoch activate_feature( @@ -1054,7 +1057,7 @@ fn test_publisher_stake_caps() { // Now the messages contain the publisher caps message messages.push(&publisher_caps_message); - assert_eq!(messages.len(), 9); + assert_eq!(messages.len(), 11); let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); @@ -1066,6 +1069,24 @@ fn test_publisher_stake_caps() { bank = new_from_parent(&Arc::new(bank)); + // We add some badly formatted stake cap parameters + let mut stake_cap_parameters_account = + AccountSharedData::new(42, size_of::(), &ORACLE_PID); + stake_cap_parameters_account.set_data( + vec![1,2,3,4], + ); + bank.store_account(&STAKE_CAPS_PARAMETERS_ADDR, &stake_cap_parameters_account); + + // Nothing should change as the stake cap parameters are invalid + let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); + bank.freeze(); + check_accumulator_state_matches_messages( + &bank, + &sequence_tracker_before_bank_freeze, + &messages, + ); + bank = new_from_parent(&Arc::new(bank)); + // Now we update the stake cap parameters let mut stake_cap_parameters_account = AccountSharedData::new(42, size_of::(), &ORACLE_PID); @@ -1095,7 +1116,7 @@ fn test_publisher_stake_caps() { // Update the publisher caps message in the message arrays to match the new parameters let last_element_index = messages.len() - 1; messages[last_element_index] = &publisher_caps_message_with_new_parameters; - assert_eq!(messages.len(), 9); + assert_eq!(messages.len(), 11); let sequence_tracker_before_bank_freeze = get_acc_sequence_tracker(&bank); bank.freeze(); From 9c9c8fe251b3c5daac5ce795c5b3408b5d9a1c73 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 30 Jul 2024 17:43:02 +0100 Subject: [PATCH 45/46] fix: downgrade to debug --- runtime/src/bank/pyth_accumulator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/bank/pyth_accumulator.rs b/runtime/src/bank/pyth_accumulator.rs index 8944a82aee..f915cc3f98 100644 --- a/runtime/src/bank/pyth_accumulator.rs +++ b/runtime/src/bank/pyth_accumulator.rs @@ -485,7 +485,7 @@ pub fn compute_publisher_stake_caps( ); measure.stop(); - info!( + debug!( "PublisherStakeCaps: Computed publisher stake caps with m : {} and z : {} in {} us", parameters.m, parameters.z, From 2f5eeb2ad4ea2409b07b91fffb6dad06e9c95858 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 31 Jul 2024 20:55:02 +0100 Subject: [PATCH 46/46] fix: use released version --- Cargo.lock | 11 ++++++----- runtime/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b6099de04..34a08fbd3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3600,8 +3600,8 @@ dependencies = [ [[package]] name = "pyth-oracle" -version = "2.32.0" -source = "git+https://github.com/pyth-network/pyth-client.git?branch=feat/publisher-caps#f6de9ed368a7aa0740298740c8585cf82273c9fc" +version = "2.33.0" +source = "git+https://github.com/pyth-network/pyth-client?tag=oracle-v2.33.0#5ee8b5f63f8b1d72f84c1177adaf564d7d61d092" dependencies = [ "bindgen 0.60.1", "bitflags 2.6.0", @@ -3609,7 +3609,7 @@ dependencies = [ "byteorder", "num-derive", "num-traits", - "pythnet-sdk 2.1.0", + "pythnet-sdk 2.2.0", "solana-program 1.14.17", "solana-sdk 1.14.17", "thiserror", @@ -3635,8 +3635,9 @@ dependencies = [ [[package]] name = "pythnet-sdk" -version = "2.1.0" -source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#a9e93d96418101b3caa8e7ffaaef21d344184eb9" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f81d23a693463366640d1ba14c612841b5b07adc56ee20bbf9acaca2ad86b5" dependencies = [ "bincode", "borsh 0.10.3", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 86c48d784e..72c9fd2090 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -35,7 +35,7 @@ num-traits = { version = "0.2" } num_cpus = "1.13.1" once_cell = "1.12.0" ouroboros = "0.15.0" -pyth-oracle = { git = "https://github.com/pyth-network/pyth-client.git" ,branch="feat/publisher-caps" , features = ["library"] } +pyth-oracle = { git = "https://github.com/pyth-network/pyth-client", tag = "oracle-v2.33.0", features = ["library"] } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", version = "1.13.6", rev = "33f901aa45f4f0005aa5a84a1479b78ca9033074" } rand = "0.7.0" rayon = "1.5.3"