From 138c9c53d806e4d231b6bbcd3fa662cc4130d237 Mon Sep 17 00:00:00 2001 From: Karim Date: Mon, 15 Apr 2024 19:19:16 +0100 Subject: [PATCH] Improve migration logic (#21) * Remove proofs and storage amount migrations * Add `combine` command and `total_stuck_supply` * Fix migration data type --- Cargo.lock | 373 ++++++----------------------------------------- README.md | 8 +- src/lib.rs | 5 +- src/main.rs | 14 ++ src/migration.rs | 169 +++++++++------------ src/parser.rs | 37 ++--- 6 files changed, 144 insertions(+), 462 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index abbfacf..1eaa4be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,24 +174,6 @@ dependencies = [ "derive_arbitrary", ] -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - -[[package]] -name = "arrayvec" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" - [[package]] name = "async-stream" version = "0.3.5" @@ -245,10 +227,10 @@ dependencies = [ "base64 0.20.0", "clap", "libc", - "near-crypto 0.17.0", + "near-crypto", "near-jsonrpc-client", "near-jsonrpc-primitives", - "near-primitives 0.17.0", + "near-primitives", "near-sdk", "serde", "serde_derive", @@ -263,7 +245,6 @@ source = "git+https://github.com/aurora-is-near/aurora-engine.git?tag=3.5.0#dc3f dependencies = [ "base64 0.21.5", "borsh 0.10.3", - "borsh 0.9.3", "bs58 0.5.0", "hex", "primitive-types 0.12.2", @@ -278,12 +259,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base64" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" - [[package]] name = "base64" version = "0.13.1" @@ -314,18 +289,6 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" -[[package]] -name = "bitvec" -version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774144344a4faa177370406a7ff5f1da24303817368584c6206c8303eb07848" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "blake2" version = "0.9.2" @@ -383,7 +346,7 @@ checksum = "6441c552f230375d18e3cc377677914d2ca2b0d36e52129fe15450a2dce46775" dependencies = [ "borsh-derive-internal 0.9.3", "borsh-schema-derive-internal 0.9.3", - "proc-macro-crate 0.1.5", + "proc-macro-crate", "proc-macro2", "syn 1.0.109", ] @@ -396,7 +359,7 @@ checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" dependencies = [ "borsh-derive-internal 0.10.3", "borsh-schema-derive-internal 0.10.3", - "proc-macro-crate 0.1.5", + "proc-macro-crate", "proc-macro2", "syn 1.0.109", ] @@ -467,12 +430,6 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" -[[package]] -name = "byte-slice-cast" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" - [[package]] name = "byteorder" version = "1.5.0" @@ -870,9 +827,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" dependencies = [ - "byteorder", - "rand 0.8.5", - "rustc-hex", "static_assertions", ] @@ -931,12 +885,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "funty" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" - [[package]] name = "futures" version = "0.3.29" @@ -1264,15 +1212,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "impl-codec" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161ebdfec3c8e3b52bf61c4f3550a1eea4f9579d10dc1b936f3171ebdcd6c443" -dependencies = [ - "parity-scale-codec", -] - [[package]] name = "impl-rlp" version = "0.3.0" @@ -1291,17 +1230,6 @@ dependencies = [ "serde", ] -[[package]] -name = "impl-trait-for-tuples" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "indexmap" version = "1.9.3" @@ -1480,16 +1408,6 @@ dependencies = [ "serde", ] -[[package]] -name = "near-account-id" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d258582a1878e6db67400b0504a5099db85718d22c2e07f747fe1706ae7150" -dependencies = [ - "borsh 0.9.3", - "serde", -] - [[package]] name = "near-account-id" version = "0.17.0" @@ -1510,9 +1428,9 @@ dependencies = [ "chrono", "derive_more", "near-config-utils", - "near-crypto 0.17.0", + "near-crypto", "near-o11y", - "near-primitives 0.17.0", + "near-primitives", "num-rational", "once_cell", "serde", @@ -1534,32 +1452,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "near-crypto" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e75673d69fd7365508f3d32483669fe45b03bfb34e4d9363e90adae9dfb416c" -dependencies = [ - "arrayref", - "blake2", - "borsh 0.9.3", - "bs58 0.4.0", - "c2-chacha", - "curve25519-dalek", - "derive_more", - "ed25519-dalek", - "near-account-id 0.14.0", - "once_cell", - "parity-secp256k1", - "primitive-types 0.10.1", - "rand 0.7.3", - "rand_core 0.5.1", - "serde", - "serde_json", - "subtle", - "thiserror", -] - [[package]] name = "near-crypto" version = "0.17.0" @@ -1574,7 +1466,7 @@ dependencies = [ "derive_more", "ed25519-dalek", "hex", - "near-account-id 0.17.0", + "near-account-id", "near-config-utils", "near-stdx", "once_cell", @@ -1593,7 +1485,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c44c842c6cfcd9b8c387cccd4cd0619a5f21920cde5d5c292af3cc5d40510672" dependencies = [ - "near-primitives-core 0.17.0", + "near-primitives-core", ] [[package]] @@ -1606,9 +1498,9 @@ dependencies = [ "lazy_static", "log", "near-chain-configs", - "near-crypto 0.17.0", + "near-crypto", "near-jsonrpc-primitives", - "near-primitives 0.17.0", + "near-primitives", "reqwest", "serde", "serde_json", @@ -1623,9 +1515,9 @@ checksum = "97b2934b5ab243e25e951c984525ba0aff0e719ed915c988c5195405aa0f6987" dependencies = [ "arbitrary", "near-chain-configs", - "near-crypto 0.17.0", - "near-primitives 0.17.0", - "near-rpc-error-macro 0.17.0", + "near-crypto", + "near-primitives", + "near-rpc-error-macro", "serde", "serde_json", "thiserror", @@ -1640,8 +1532,8 @@ dependencies = [ "actix", "atty", "clap", - "near-crypto 0.17.0", - "near-primitives-core 0.17.0", + "near-crypto", + "near-primitives-core", "once_cell", "opentelemetry", "opentelemetry-otlp", @@ -1657,35 +1549,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "near-primitives" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad1a9a1640539c81f065425c31bffcfbf6b31ef1aeaade59ce905f5df6ac860" -dependencies = [ - "borsh 0.9.3", - "byteorder", - "bytesize", - "chrono", - "derive_more", - "easy-ext", - "hex", - "near-crypto 0.14.0", - "near-primitives-core 0.14.0", - "near-rpc-error-macro 0.14.0", - "near-vm-errors 0.14.0", - "num-rational", - "once_cell", - "primitive-types 0.10.1", - "rand 0.7.3", - "reed-solomon-erasure", - "serde", - "serde_json", - "smart-default", - "strum", - "thiserror", -] - [[package]] name = "near-primitives" version = "0.17.0" @@ -1701,12 +1564,12 @@ dependencies = [ "easy-ext", "enum-map", "hex", - "near-crypto 0.17.0", + "near-crypto", "near-fmt", - "near-primitives-core 0.17.0", - "near-rpc-error-macro 0.17.0", + "near-primitives-core", + "near-rpc-error-macro", "near-stdx", - "near-vm-errors 0.17.0", + "near-vm-errors", "num-rational", "once_cell", "primitive-types 0.10.1", @@ -1723,23 +1586,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "near-primitives-core" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d508f0fc340f6461e4e256417685720d3c4c00bb5a939b105160e49137caba" -dependencies = [ - "base64 0.11.0", - "borsh 0.9.3", - "bs58 0.4.0", - "derive_more", - "near-account-id 0.14.0", - "num-rational", - "serde", - "sha2 0.10.8", - "strum", -] - [[package]] name = "near-primitives-core" version = "0.17.0" @@ -1752,7 +1598,7 @@ dependencies = [ "bs58 0.4.0", "derive_more", "enum-map", - "near-account-id 0.17.0", + "near-account-id", "num-rational", "serde", "serde_repr", @@ -1762,17 +1608,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "near-rpc-error-core" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ee0b41c75ef859c193a8ff1dadfa0c8207bc0ac447cc22259721ad769a1408" -dependencies = [ - "quote", - "serde", - "syn 1.0.109", -] - [[package]] name = "near-rpc-error-core" version = "0.17.0" @@ -1784,17 +1619,6 @@ dependencies = [ "syn 2.0.42", ] -[[package]] -name = "near-rpc-error-macro" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e837bd4bacd807073ec5ceb85708da7f721b46a4c2a978de86027fb0034ce31" -dependencies = [ - "near-rpc-error-core 0.14.0", - "serde", - "syn 1.0.109", -] - [[package]] name = "near-rpc-error-macro" version = "0.17.0" @@ -1802,7 +1626,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31d2dadd765101c77e664029dd6fbec090e696877d4ae903c620d02ceda4969a" dependencies = [ "fs2", - "near-rpc-error-core 0.17.0", + "near-rpc-error-core", "serde", "syn 2.0.42", ] @@ -1810,16 +1634,15 @@ dependencies = [ [[package]] name = "near-sdk" version = "4.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15eb3de2defe3626260cc209a6cdb985c6b27b0bd4619fad97dcfae002c3c5bd" +source = "git+https://github.com/aurora-is-near/near-sdk-rs.git?tag=v4.1.1-fix-deps#96afbfa37d1e8b1ecca69dde1d5c715ceeeb3872" dependencies = [ "base64 0.13.1", - "borsh 0.9.3", + "borsh 0.10.3", "bs58 0.4.0", "near-abi", - "near-crypto 0.14.0", - "near-primitives 0.14.0", - "near-primitives-core 0.14.0", + "near-crypto", + "near-primitives", + "near-primitives-core", "near-sdk-macros", "near-sys", "near-vm-logic", @@ -1833,8 +1656,7 @@ dependencies = [ [[package]] name = "near-sdk-macros" version = "4.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4907affc9f5ed559456509188ff0024f1f2099c0830e6bdb66eb61d5b75912c0" +source = "git+https://github.com/aurora-is-near/near-sdk-rs.git?tag=v4.1.1-fix-deps#96afbfa37d1e8b1ecca69dde1d5c715ceeeb3872" dependencies = [ "Inflector", "proc-macro2", @@ -1850,21 +1672,8 @@ checksum = "6540152fba5e96fe5d575b79e8cd244cf2add747bb01362426bdc069bc3a23bc" [[package]] name = "near-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397688591acf8d3ebf2c2485ba32d4b24fc10aad5334e3ad8ec0b7179bfdf06b" - -[[package]] -name = "near-vm-errors" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0da466a30f0446639cbd788c30865086fac3e8dcb07a79e51d2b0775ed4261e" -dependencies = [ - "borsh 0.9.3", - "near-account-id 0.14.0", - "near-rpc-error-macro 0.14.0", - "serde", -] +version = "0.2.0" +source = "git+https://github.com/aurora-is-near/near-sdk-rs.git?tag=v4.1.1-fix-deps#96afbfa37d1e8b1ecca69dde1d5c715ceeeb3872" [[package]] name = "near-vm-errors" @@ -1873,8 +1682,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec545d1bede0579e7c15dd2dce9b998dc975c52f2165702ff40bec7ff69728bb" dependencies = [ "borsh 0.10.3", - "near-account-id 0.17.0", - "near-rpc-error-macro 0.17.0", + "near-account-id", + "near-rpc-error-macro", "serde", "strum", "thiserror", @@ -1882,19 +1691,20 @@ dependencies = [ [[package]] name = "near-vm-logic" -version = "0.14.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b534828419bacbf1f7b11ef7b00420f248c548c485d3f0cfda8bb6931152f2" +checksum = "30d7487c678ed1963a0ecd5033f72bb41caa58debd6fe8025a9bef6e1a6a519a" dependencies = [ - "base64 0.13.1", - "borsh 0.9.3", - "bs58 0.4.0", - "byteorder", - "near-account-id 0.14.0", - "near-crypto 0.14.0", - "near-primitives 0.14.0", - "near-primitives-core 0.14.0", - "near-vm-errors 0.14.0", + "borsh 0.10.3", + "ed25519-dalek", + "near-account-id", + "near-crypto", + "near-fmt", + "near-o11y", + "near-primitives", + "near-primitives-core", + "near-stdx", + "near-vm-errors", "ripemd", "serde", "sha2 0.10.8", @@ -2075,43 +1885,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parity-scale-codec" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373b1a4c1338d9cd3d1fa53b3a11bdab5ab6bd80a20f7f7becd76953ae2be909" -dependencies = [ - "arrayvec 0.7.4", - "bitvec", - "byte-slice-cast", - "impl-trait-for-tuples", - "parity-scale-codec-derive", - "serde", -] - -[[package]] -name = "parity-scale-codec-derive" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1557010476e0595c9b568d16dcfb81b93cdeb157612726f5170d31aa707bed27" -dependencies = [ - "proc-macro-crate 1.3.1", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "parity-secp256k1" -version = "0.7.0" -source = "git+https://github.com/paritytech/rust-secp256k1?rev=d05fd8e#d05fd8e152f8d110b587906e3d854196b086e42a" -dependencies = [ - "arrayvec 0.5.2", - "cc", - "cfg-if 0.1.10", - "rand 0.7.3", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -2208,7 +1981,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" dependencies = [ "fixed-hash 0.7.0", - "impl-codec", "uint", ] @@ -2233,16 +2005,6 @@ dependencies = [ "toml", ] -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit", -] - [[package]] name = "proc-macro2" version = "1.0.70" @@ -2335,12 +2097,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "radium" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" - [[package]] name = "rand" version = "0.7.3" @@ -2958,12 +2714,6 @@ dependencies = [ "libc", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "tempfile" version = "3.8.1" @@ -3149,23 +2899,6 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_datetime" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" - -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.1.0", - "toml_datetime", - "winnow", -] - [[package]] name = "tonic" version = "0.6.2" @@ -3726,15 +3459,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" -[[package]] -name = "winnow" -version = "0.5.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.50.0" @@ -3745,12 +3469,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "wyz" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" - [[package]] name = "zerocopy" version = "0.7.31" @@ -3804,3 +3522,8 @@ dependencies = [ "rand 0.8.5", "rustc-hex", ] + +[[patch.unused]] +name = "parity-secp256k1" +version = "0.7.0" +source = "git+https://github.com/paritytech/rust-secp256k1?rev=d05fd8e#d05fd8e152f8d110b587906e3d854196b086e42a" diff --git a/README.md b/README.md index 73209d2..28bee4c 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,9 @@ to the new [aurora-eth-connector](https://github.com/aurora-is-near/aurora-eth-c 7. Run migration-tool `prepare-for-migration` for parsed Aurora state result data (for ex: `migration_state.borsh`). 8. Stop migration-tool `indexer` 9. Run migration-tool `prepare-for-migration` for indexed result data (for ex: `migration_indexed.borsh`). -10. Run migration-tool `migrate` for previously generated `migration_state.borsh`. -11. Run migration-tool `migrate` for previously generated `migration_indexed.borsh`. -12. Run migration-tool `migrate` with check correctness command for `migration_state.borsh`. -13. Run migration-tool `migrate` with check correctness command for `migration_indexed.borsh`. -14. Unpause Aurora contract and Bridge. +10. Run migration-tool `combine-indexed-and-state-data` for indexed and state data (for ex: `migration_indexed.borsh` and `migration_state.borsh`). +11. Run migration-tool `migrate` for previously generated `migration_full.borsh`. +12. Unpause Aurora contract and Bridge. # How it works diff --git a/src/lib.rs b/src/lib.rs index 9e977e5..1b9ad59 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,12 +25,11 @@ pub struct BlockData { pub struct FungibleToken { pub total_eth_supply_on_near: NEP141Wei, pub total_eth_supply_on_aurora: NEP141Wei, - pub account_storage_usage: StorageUsage, } #[derive(Debug, Default, BorshSerialize, BorshDeserialize)] pub struct StateData { - pub contract_data: FungibleToken, + pub total_supply: NEP141Wei, + pub total_stuck_supply: NEP141Wei, pub accounts: HashMap, - pub proofs: Vec, } diff --git a/src/main.rs b/src/main.rs index c143ab9..556ba67 100644 --- a/src/main.rs +++ b/src/main.rs @@ -131,6 +131,20 @@ async fn main() -> anyhow::Result<()> { .expect("Expected output file"); Migration::prepare_indexed(input_data_file, output_file).await?; } + Some(("combine-indexed-and-state-data", cmd)) => { + let state_data_file = cmd.get_one::("state").expect("Expected data file"); + let indexed_data_file = cmd + .get_one::("indexed") + .expect("Expected data file"); + let output_file = cmd + .get_one::("output") + .expect("Expected output file"); + Migration::combine_indexed_and_state_data( + state_data_file, + indexed_data_file, + output_file, + )?; + } _ => (), } diff --git a/src/migration.rs b/src/migration.rs index f66c3f6..20f2f23 100644 --- a/src/migration.rs +++ b/src/migration.rs @@ -1,9 +1,9 @@ use crate::rpc::{Client, REQUEST_TIMEOUT}; -use aurora_engine_migration_tool::{FungibleToken, StateData}; +use aurora_engine_migration_tool::StateData; use aurora_engine_types::types::NEP141Wei; use near_sdk::borsh::{self, BorshDeserialize, BorshSerialize}; use near_sdk::json_types::U128; -use near_sdk::{AccountId, Balance, StorageUsage}; +use near_sdk::{AccountId, Balance}; use serde_json::json; use std::collections::HashMap; use std::io::Write; @@ -29,8 +29,6 @@ pub struct Migration { pub struct MigrationInputData { pub accounts: HashMap, pub total_supply: Option, - pub account_storage_usage: Option, - pub used_proofs: Vec, } #[derive(Debug, BorshSerialize, BorshDeserialize, Eq, PartialEq)] @@ -39,8 +37,6 @@ pub enum MigrationCheckResult { AccountNotExist(Vec), AccountAmount(HashMap), TotalSupply(Balance), - StorageUsage(StorageUsage), - Proof(Vec), } impl Migration { @@ -101,9 +97,6 @@ impl Migration { .await?; let correctness = MigrationCheckResult::try_from_slice(&res).unwrap(); match correctness { - MigrationCheckResult::Proof(missed) => { - println!("{msg}: {counter} [Missed: {:?}]", missed.len()); - } MigrationCheckResult::AccountNotExist(missed) => { println!("{msg}: {counter} [Missed: {:?}]", missed.len()); } @@ -114,13 +107,8 @@ impl Migration { print!("\r{msg}: {counter} [{correctness:?}]"); std::io::stdout().flush()?; } - _ => { - if let MigrationCheckResult::TotalSupply(_) = correctness { - println!("{msg} [Missed field: {correctness:?}]"); - } - if let MigrationCheckResult::StorageUsage(_) = correctness { - println!("{msg} [Missed field: {correctness:?}]"); - } + MigrationCheckResult::TotalSupply(_) => { + println!("{msg} [Missed field: {correctness:?}]"); } } Ok(()) @@ -131,44 +119,10 @@ impl Migration { // Data limit per transaction let limit = RECORDS_COUNT_PER_TX; - // Proofs migration - let mut i = 0; - let mut proofs_count = 0; - let mut reproducible_data_for_proofs: Vec<(Vec, usize)> = vec![]; - loop { - let proofs = if i + limit >= self.data.proofs.len() { - &self.data.proofs[i..] - } else { - &self.data.proofs[i..i + limit] - } - .to_vec(); - - proofs_count += proofs.len(); - let migration_data = MigrationInputData { - accounts: HashMap::new(), - total_supply: None, - account_storage_usage: None, - used_proofs: proofs, - } - .try_to_vec()?; - reproducible_data_for_proofs.push((migration_data.clone(), proofs_count)); - - self.commit_migration(migration_data, "Proofs", proofs_count) - .await?; - - if i + limit >= self.data.proofs.len() { - break; - } - - i += limit; - } - assert_eq!(proofs_count, self.data.proofs.len()); - println!(); - // Accounts migration let mut accounts: HashMap = HashMap::new(); let mut accounts_count = 0; - let mut reproducible_data_for_accounts: Vec<(Vec, usize)> = vec![]; + let mut reproducible_data_for_accounts: Vec<(HashMap, usize)> = vec![]; for (i, (account, amount)) in self.data.accounts.iter().enumerate() { accounts.insert(account.clone(), amount.as_u128()); @@ -178,55 +132,47 @@ impl Migration { } accounts_count += &accounts.len(); - let migration_data = MigrationInputData { - accounts: accounts.clone(), - total_supply: None, - account_storage_usage: None, - used_proofs: vec![], - } - .try_to_vec() - .expect("Failed serialize"); - reproducible_data_for_accounts.push((migration_data.clone(), accounts_count)); + reproducible_data_for_accounts.push((accounts.clone(), accounts_count)); - self.commit_migration(migration_data, "Accounts", accounts_count) - .await?; + let migration_data: Vec = accounts.keys().cloned().collect(); + self.commit_migration( + migration_data.try_to_vec().expect("Failed serialize"), + "Accounts", + accounts_count, + ) + .await?; // Clear accounts.clear(); } assert_eq!(self.data.accounts.len(), accounts_count); - // Migrate Contract data - println!(); - let contract_migration_data = MigrationInputData { - accounts: HashMap::new(), - total_supply: Some(self.data.contract_data.total_eth_supply_on_near.as_u128()), - account_storage_usage: Some(self.data.contract_data.account_storage_usage), - used_proofs: vec![], - } - .try_to_vec() - .expect("Failed serialize"); - - self.commit_migration(contract_migration_data.clone(), "Contract data", 1) - .await?; - //===================================== // Checking the correctness and integrity of data, regardless of // the migration process - println!("\n\n[Check correctness]"); - for (migration_data, counter) in reproducible_data_for_proofs { - self.check_migration("Proofs", migration_data, counter) - .await?; - } - println!(); - for (migration_data, counter) in reproducible_data_for_accounts { + for (accounts, counter) in reproducible_data_for_accounts { + let migration_data = MigrationInputData { + accounts: accounts.clone(), + total_supply: None, + } + .try_to_vec() + .expect("Failed serialize"); + self.check_migration("Accounts:", migration_data, counter) .await?; } println!(); + let contract_migration_data = MigrationInputData { + accounts: HashMap::new(), + total_supply: Some( + self.data.total_supply.as_u128() - self.data.total_stuck_supply.as_u128(), + ), + } + .try_to_vec() + .expect("Failed serialize"); self.check_migration("Contract data:", contract_migration_data, 1) .await?; @@ -247,21 +193,16 @@ impl Migration { let rpc = Client::new(); let mut migration_data = StateData { - contract_data: FungibleToken { - total_eth_supply_on_near: NEP141Wei::new(0), - total_eth_supply_on_aurora: NEP141Wei::new(0), - // This value impossible to request from the contract - account_storage_usage: 0, - }, + total_supply: NEP141Wei::new(0), + total_stuck_supply: NEP141Wei::new(0), accounts: HashMap::new(), - proofs: vec![], }; let data = rpc .request_view(AURORA_CONTRACT, "ft_total_supply".to_string(), vec![]) .await?; let total_supply: U128 = serde_json::from_slice(&data).unwrap(); - migration_data.contract_data.total_eth_supply_on_near = NEP141Wei::new(total_supply.0); + migration_data.total_supply = NEP141Wei::new(total_supply.0); for account in indexer_data.data.accounts { let args = json!({ "account_id": account }) @@ -280,21 +221,47 @@ impl Migration { tokio::time::sleep(REQUEST_TIMEOUT).await; } - for proof in indexer_data.data.proofs { - migration_data.proofs.push(proof); + println!("Accounts: {:?}", migration_data.accounts.len()); + println!("Total supply: {:?}", migration_data.total_supply.as_u128()); + + migration_data + .try_to_vec() + .and_then(|data| std::fs::write(output, data)) + .map_err(|e| anyhow::anyhow!("Failed save migration data, {e}")) + } + + pub fn combine_indexed_and_state_data>( + state: P, + indexed: P, + output: P, + ) -> anyhow::Result<()> { + let mut state_data = { + let data = std::fs::read(state) + .map_err(|e| anyhow::anyhow!("Failed read state data file, {e}"))?; + StateData::try_from_slice(&data) + .map_err(|e| anyhow::anyhow!("Failed deserialize state data, {e}"))? + }; + + let indexed_data = { + let data = std::fs::read(indexed) + .map_err(|e| anyhow::anyhow!("Failed read indexed data file, {e}"))?; + StateData::try_from_slice(&data) + .map_err(|e| anyhow::anyhow!("Failed deserialize indexed data, {e}"))? + }; + + for (account, balance) in indexed_data.accounts { + state_data.accounts.insert(account, balance); } + state_data.total_supply = indexed_data.total_supply; - println!("Proofs: {:?}", migration_data.proofs.len()); - println!("Accounts: {:?}", migration_data.accounts.len()); + println!("Accounts: {:?}", state_data.accounts.len()); + println!("Total supply: {:?}", state_data.total_supply.as_u128()); println!( - "Total supply: {:?}", - migration_data - .contract_data - .total_eth_supply_on_near - .as_u128() + "Total stuck supply: {:?}", + state_data.total_stuck_supply.as_u128() ); - migration_data + state_data .try_to_vec() .and_then(|data| std::fs::write(output, data)) .map_err(|e| anyhow::anyhow!("Failed save migration data, {e}")) diff --git a/src/parser.rs b/src/parser.rs index 46c0a5b..168cf8a 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -10,7 +10,6 @@ use std::str::FromStr; enum KeyType { Accounts(Vec), Contract, - Proof(Vec), Unknown, } @@ -18,10 +17,6 @@ pub fn construct_contract_key(suffix: EthConnectorStorageId) -> Vec { bytes_to_key(KeyPrefix::EthConnector, &[u8::from(suffix)]) } -pub fn prefix_proof_key() -> Vec { - construct_contract_key(EthConnectorStorageId::UsedEvent) -} - pub fn prefix_account_key() -> Vec { bytes_to_key( KeyPrefix::EthConnector, @@ -52,32 +47,28 @@ pub fn parse>(json_file: P, output: Option

) -> anyhow::Result< println!("Data size: {:.3} Gb", data.len() as f64 / 1_000_000_000.); println!("Data values: {:#?}", json_data.result.values.len()); - let mut proofs: Vec = vec![]; let mut accounts: HashMap = HashMap::new(); let mut contract_data: FungibleToken = FungibleToken::default(); + let mut total_stuck_supply = NEP141Wei::new(0); for result_value in &json_data.result.values { let key = base64::decode(&result_value.key) .map_err(|e| anyhow::anyhow!("Failed deserialize key, {e}"))?; // Get proofs match key_type(&key) { - KeyType::Proof(value) => { - let proof = String::from_utf8(value) - .map_err(|e| anyhow::anyhow!("Failed parse proof, {e}"))?; - proofs.push(proof); - } KeyType::Accounts(value) => { let account_str = std::str::from_utf8(&value) .map_err(|e| anyhow::anyhow!("Failed parse account to str, {e}"))?; - let Ok(account) = AccountId::from_str(account_str) else { - println!("\tNot fetched account: {account_str}"); - continue; - }; let account_balance = NEP141Wei::try_from_slice( &base64::decode(&result_value.value) .map_err(|e| anyhow::anyhow!("Failed get account balance, {e}"))?, ) .map_err(|e| anyhow::anyhow!("Failed parse account balance, {e}"))?; + let Ok(account) = AccountId::from_str(account_str) else { + total_stuck_supply = total_stuck_supply + account_balance; + println!("\tNot fetched account: {account_str} with balance {account_balance}"); + continue; + }; accounts.insert(account, account_balance); } KeyType::Contract => { @@ -89,14 +80,13 @@ pub fn parse>(json_file: P, output: Option

) -> anyhow::Result< KeyType::Unknown => (), //anyhow::bail!("Unknown key type"), } } - println!("Proofs: {}", proofs.len()); println!("Accounts: {}", accounts.len()); // Store result data StateData { - contract_data, + total_supply: contract_data.total_eth_supply_on_near, + total_stuck_supply, accounts, - proofs, } .try_to_vec() .and_then(|data| std::fs::write(result_file_name, data)) @@ -104,11 +94,7 @@ pub fn parse>(json_file: P, output: Option

) -> anyhow::Result< } fn key_type(key: &[u8]) -> KeyType { - if is_prefix_proof_key(key) { - let proof_prefix_len = prefix_proof_key().len(); - let value = key[proof_prefix_len..].to_vec(); - KeyType::Proof(value) - } else if is_account_prefix_key(key) { + if is_account_prefix_key(key) { let account_prefix_len = prefix_account_key().len(); let value = key[account_prefix_len..].to_vec(); KeyType::Accounts(value) @@ -119,11 +105,6 @@ fn key_type(key: &[u8]) -> KeyType { } } -fn is_prefix_proof_key(key: &[u8]) -> bool { - let proof_prefix = &prefix_proof_key(); - key.len() > proof_prefix.len() && &key[..proof_prefix.len()] == proof_prefix -} - fn is_account_prefix_key(key: &[u8]) -> bool { let account_prefix = &prefix_account_key(); key.len() > account_prefix.len() && &key[..account_prefix.len()] == account_prefix