From 358fab62bdf443a02358c67b322f0685ec072303 Mon Sep 17 00:00:00 2001 From: Captain Date: Thu, 27 Oct 2022 10:33:19 +0000 Subject: [PATCH] Reproducible builds implemented. Pruning & fastsync fixes. --- astrobwt/suffixarray.go | 4 - block/block.go | 2 +- block/miniblock.go | 2 +- blockchain/miner_block.go | 9 +- blockchain/prune_history.go | 52 ++++++++++-- blockchain/storefs.go | 4 +- blockchain/storetopo.go | 19 ++++- cmd/derod/rpc/websocket_getwork_server.go | 2 +- cmd/derod/rpc/websocket_server.go | 3 +- config/seed_nodes.go | 4 +- config/version.go | 2 +- cryptography/crypto/algebra_fieldvector.go | 2 +- cryptography/crypto/proof_verify.go | 84 +++++++++---------- cryptography/crypto/userdata.go | 1 - dvm/dvm.go | 2 +- dvm/dvm_functions.go | 12 +-- globals/globals.go | 5 +- p2p/bans.go | 6 +- p2p/chain_bootstrap.go | 52 ++++++++++-- p2p/controller.go | 8 +- p2p/peer.go | 2 +- p2p/peer_pool.go | 2 +- p2p/rpc_cbor_codec.go | 2 +- p2p/rpc_treesection.go | 1 + p2p/time.go | 9 +- p2p/wire_structs.go | 3 +- rpc/rpc.go | 2 +- rpc/wallet_rpc.go | 5 +- .../github.com/deroproject/graviton/extra.go | 4 +- walletapi/mnemonics/mnemonics.go | 2 +- 30 files changed, 195 insertions(+), 112 deletions(-) diff --git a/astrobwt/suffixarray.go b/astrobwt/suffixarray.go index 6dbdb67b..31950b87 100644 --- a/astrobwt/suffixarray.go +++ b/astrobwt/suffixarray.go @@ -13,7 +13,6 @@ // // lookup byte slice s // offsets1 := index.Lookup(s, -1) // the list of all indices where s occurs in data // offsets2 := index.Lookup(s, 3) // the list of at most 3 indices where s occurs in data -// package astrobwt import ( @@ -230,7 +229,6 @@ func (x *Index) Write(w io.Writer) error { // Bytes returns the data over which the index was created. // It must not be modified. -// func (x *Index) Bytes() []byte { return x.data } @@ -255,7 +253,6 @@ func (x *Index) lookupAll(s []byte) ints { // The result is nil if s is empty, s is not found, or n == 0. // Lookup time is O(log(N)*len(s) + len(result)) where N is the // size of the indexed data. -// func (x *Index) Lookup(s []byte, n int) (result []int) { if len(s) > 0 && n != 0 { matches := x.lookupAll(s) @@ -286,7 +283,6 @@ func (x *Index) Lookup(s []byte, n int) (result []int) { // in successive order. Otherwise, at most n matches are returned and // they may not be successive. The result is nil if there are no matches, // or if n == 0. -// func (x *Index) FindAllIndex(r *regexp.Regexp, n int) (result [][]int) { // a non-empty literal prefix is used to determine possible // match start indices with Lookup diff --git a/block/block.go b/block/block.go index 867b9110..7fd32ae5 100644 --- a/block/block.go +++ b/block/block.go @@ -185,7 +185,7 @@ func (bl *Block) GetTXSHash() (result crypto.Hash) { return } -//parse entire block completely +// parse entire block completely func (bl *Block) Deserialize(buf []byte) (err error) { done := 0 diff --git a/block/miniblock.go b/block/miniblock.go index 0087197e..f89f0162 100644 --- a/block/miniblock.go +++ b/block/miniblock.go @@ -172,7 +172,7 @@ func (mbl *MiniBlock) Serialize() (result []byte) { return b.Bytes() } -//parse entire block completely +// parse entire block completely func (mbl *MiniBlock) Deserialize(buf []byte) (err error) { if len(buf) < MINIBLOCK_SIZE { return fmt.Errorf("Expected %d bytes. Actual %d", MINIBLOCK_SIZE, len(buf)) diff --git a/blockchain/miner_block.go b/blockchain/miner_block.go index 20e03ca4..fce2bdc0 100644 --- a/blockchain/miner_block.go +++ b/blockchain/miner_block.go @@ -50,7 +50,9 @@ type BlockScore struct { } // Heighest height is ordered first, the condition is reverted see eg. at https://golang.org/pkg/sort/#Slice -// if heights are equal, nodes are sorted by their block ids which will never collide , hopefullly +// +// if heights are equal, nodes are sorted by their block ids which will never collide , hopefullly +// // block ids are sorted by lowest byte first diff func sort_descending_by_height_blid(tips_scores []BlockScore) { sort.Slice(tips_scores, func(i, j int) bool { @@ -101,7 +103,7 @@ func convert_uint32_to_crypto_hash(i uint32) crypto.Hash { return h } -//NOTE: this function is quite big since we do a lot of things in preparation of next blocks +// NOTE: this function is quite big since we do a lot of things in preparation of next blocks func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address) (cbl *block.Complete_Block, bl block.Block, err error) { //chain.Lock() //defer chain.Unlock() @@ -343,7 +345,6 @@ func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address) (cbl return } -// func ConvertBlockToMiniblock(bl block.Block, miniblock_miner_address rpc.Address) (mbl block.MiniBlock) { mbl.Version = 1 @@ -480,7 +481,7 @@ func (chain *Blockchain) Accept_new_block(tstamp uint64, miniblock_blob []byte) // lets try to check pow to detect whether the miner is cheating if !chain.simulator && !chain.VerifyMiniblockPoW(&bl, mbl) { - logger.V(1).Error(err, "Error ErrInvalidPoW") + //logger.V(1).Error(err, "Error ErrInvalidPoW") err = errormsg.ErrInvalidPoW return } diff --git a/blockchain/prune_history.go b/blockchain/prune_history.go index c7abe1cb..60d025a2 100644 --- a/blockchain/prune_history.go +++ b/blockchain/prune_history.go @@ -117,9 +117,14 @@ func Prune_Blockchain(prune_topo int64) (err error) { // any error while deleting should be considered non fatal func discard_blocks_and_transactions(store *storage, topoheight int64) { - globals.Logger.Info("Block store before pruning", "size", ByteCountIEC(DirSize(filepath.Join(store.Block_tx_store.basedir, "bltx_store")))) + size_before := ByteCountIEC(DirSize(filepath.Join(store.Block_tx_store.basedir, "bltx_store"))) for i := int64(0); i < topoheight-20; i++ { // donot some more blocks for sanity currently + + if i%1000 == 0 { + globals.Logger.Info("Deleting old block/txs", "done", float64(i*100)/float64(topoheight-20)) + } + if toporecord, err := store.Topo_store.Read(i); err == nil { blid := toporecord.BLOCK_ID @@ -138,6 +143,7 @@ func discard_blocks_and_transactions(store *storage, topoheight int64) { } } } + globals.Logger.Info("Block store before pruning", "size", size_before) globals.Logger.Info("Block store after pruning ", "size", ByteCountIEC(DirSize(filepath.Join(store.Block_tx_store.basedir, "bltx_store")))) } @@ -180,6 +186,12 @@ func clone_snapshot(rsource, wsource *graviton.Store, r_ssversion uint64) (lates } } + if latest_commit_version, err = graviton.Commit(write_balance_tree); err != nil { + return 0, err + } + + globals.Logger.Info("Main balance tree cloned") + /* h,_ := old_balance_tree.Hash() fmt.Printf("old balance hash %+v\n",h ) h,_ = write_balance_tree.Hash() @@ -214,6 +226,10 @@ func clone_snapshot(rsource, wsource *graviton.Store, r_ssversion uint64) (lates } } + if latest_commit_version, err = graviton.Commit(write_meta_tree); err != nil { + return 0, err + } + /* h,_ = old_meta_tree.Hash() fmt.Printf("old meta hash %+v\n",h ) h,_ = write_meta_tree.Hash() @@ -221,7 +237,7 @@ func clone_snapshot(rsource, wsource *graviton.Store, r_ssversion uint64) (lates os.Exit(0) */ - var sc_trees []*graviton.Tree + sc_names := map[string]bool{} // now we have to copy all scs data one by one for _, scid := range sc_list { var old_sc_tree, write_sc_tree *graviton.Tree @@ -235,12 +251,13 @@ func clone_snapshot(rsource, wsource *graviton.Store, r_ssversion uint64) (lates for k, v, err := c.First(); err == nil; k, v, err = c.Next() { write_sc_tree.Put(k, v) } - sc_trees = append(sc_trees, write_sc_tree) + sc_names[string(scid)] = true + if latest_commit_version, err = graviton.Commit(write_sc_tree); err != nil { + return + } } + globals.Logger.Info("SCs cloned") - sc_trees = append(sc_trees, write_balance_tree) - sc_trees = append(sc_trees, write_meta_tree) - latest_commit_version, err = graviton.Commit(sc_trees...) return } @@ -309,7 +326,6 @@ func diff_snapshot(rsource, wsource *graviton.Store, old_version uint64, new_ver // now we have to copy new scs data one by one for _, scid := range sc_list_new { - if old_tree, err = old_ss.GetTree(string(scid)); err != nil { return } @@ -319,7 +335,7 @@ func diff_snapshot(rsource, wsource *graviton.Store, old_version uint64, new_ver if write_tree, err = write_ss.GetTree(string(scid)); err != nil { return } - c := old_tree.Cursor() + c := new_tree.Cursor() for k, v, err := c.First(); err == nil; k, v, err = c.Next() { write_tree.Put(k, v) } @@ -399,6 +415,13 @@ func rewrite_graviton_store(store *storage, prune_topoheight int64, max_topoheig } } + if err != nil { + return + } + + if i%1000 == 0 { + globals.Logger.Info("Commiting block to block changes", "done", float64(i*100)/float64(max_topoheight)) + } } // now lets store all the commit versions in 1 go @@ -444,6 +467,10 @@ func rewrite_graviton_store(store *storage, prune_topoheight int64, max_topoheig return err } + if i%1000 == 0 { + globals.Logger.Info("Rewriting entries", "done", float64(i*100)/float64(len(new_entries))) + } + } } @@ -456,6 +483,9 @@ func rewrite_graviton_store(store *storage, prune_topoheight int64, max_topoheig globals.Logger.Error(err, "err reading toporecord", "topo", i) return err // this is irrepairable damage } + if i%1000 == 0 { + globals.Logger.Info("Filling gaps", "done", float64(i*100)/float64(prune_topoheight)) + } } // now lets remove the old graviton db @@ -484,5 +514,9 @@ func clone_tree_changes(old_tree, new_tree, write_tree *graviton.Tree) { write_tree.Put(k, new_value) } - graviton.Diff(old_tree, new_tree, nil, modify_handler, insert_handler) + delete_handler := func(k, v []byte) { // modification receives old value + write_tree.Delete(k) + } + + graviton.Diff(old_tree, new_tree, delete_handler, modify_handler, insert_handler) } diff --git a/blockchain/storefs.go b/blockchain/storefs.go index 701372df..f2a07ef7 100755 --- a/blockchain/storefs.go +++ b/blockchain/storefs.go @@ -341,6 +341,8 @@ func (s *storefs) migrate_old_tx() { defer func() { fmt.Printf("Migrated %d txs to new structure\n", migrated) }() + } else { + return } if err != nil { @@ -365,8 +367,8 @@ func (s *storefs) migrate_old_tx() { } copy(h[:], txhash[:]) - s.WriteTX(h, data) s.DeleteTX(h) // this will delete legacy version + s.WriteTX(h, data) migrated++ } diff --git a/blockchain/storetopo.go b/blockchain/storetopo.go index 0ec2e285..f4ee3685 100644 --- a/blockchain/storetopo.go +++ b/blockchain/storetopo.go @@ -35,7 +35,8 @@ const TOPORECORD_SIZE int64 = 48 // this file implements a filesystem store which is used to store topo to block mapping directly in the file system and the state version directly tied type storetopofs struct { - topomapping *os.File + topomapping *os.File + last_state_version uint64 } func (s TopoRecord) String() string { @@ -82,6 +83,7 @@ func (s *storetopofs) Read(index int64) (TopoRecord, error) { func (s *storetopofs) Write(index int64, blid [32]byte, state_version uint64, height int64) (err error) { var buf [TOPORECORD_SIZE]byte var record TopoRecord + var zero_hash [32]byte copy(buf[:], blid[:]) binary.LittleEndian.PutUint64(buf[len(record.BLOCK_ID):], state_version) @@ -90,11 +92,18 @@ func (s *storetopofs) Write(index int64, blid [32]byte, state_version uint64, he binary.LittleEndian.PutUint64(buf[len(record.BLOCK_ID)+8:], uint64(height)) _, err = s.topomapping.WriteAt(buf[:], index*TOPORECORD_SIZE) - - s.topomapping.Sync() // looks like this is the cause of corruption + if s.last_state_version != state_version || state_version == 0 { + if blid != zero_hash { // during fast sync avoid syncing overhead + s.topomapping.Sync() // looks like this is the cause of corruption + } + } + s.last_state_version = state_version return err } +func (s *storetopofs) Sync() { + s.topomapping.Sync() +} func (s *storetopofs) Clean(index int64) (err error) { var state_version uint64 @@ -163,6 +172,10 @@ func (s *storetopofs) LocatePruneTopo() int64 { prune_topo-- + if prune_topo > count { + panic("invalid prune detected") + } + pruned_till = prune_topo return prune_topo } diff --git a/cmd/derod/rpc/websocket_getwork_server.go b/cmd/derod/rpc/websocket_getwork_server.go index d786b984..56c86f88 100644 --- a/cmd/derod/rpc/websocket_getwork_server.go +++ b/cmd/derod/rpc/websocket_getwork_server.go @@ -73,7 +73,7 @@ var miners_count int var mini_found_time []int64 // this array contains a epoch timestamp in int64 var rate_lock sync.Mutex -//this function will return wrong result if too wide time glitches happen to system clock +// this function will return wrong result if too wide time glitches happen to system clock func Counter(seconds int64) (r int) { // we need atleast 1 mini to find a rate rate_lock.Lock() defer rate_lock.Unlock() diff --git a/cmd/derod/rpc/websocket_server.go b/cmd/derod/rpc/websocket_server.go index 63549572..9b35dd86 100644 --- a/cmd/derod/rpc/websocket_server.go +++ b/cmd/derod/rpc/websocket_server.go @@ -58,7 +58,7 @@ type RPCServer struct { sync.RWMutex } -//var Exit_In_Progress bool +// var Exit_In_Progress bool var chain *blockchain.Blockchain var logger logr.Logger @@ -296,7 +296,6 @@ func Echo(ctx context.Context, args []string) string { var internal_server = server.NewLocal(historical_apis, nil) // uses traditional "getinfo" for compatibility reasons // Bridge HTTP to the JSON-RPC server. var bridge = jhttp.NewBridge(internal_server.Client) - */ var historical_apis = handler.Map{"getinfo": handler.New(GetInfo), "get_info": handler.New(GetInfo), // this is just an alias to above diff --git a/config/seed_nodes.go b/config/seed_nodes.go index 716a59c7..4b8c7c52 100644 --- a/config/seed_nodes.go +++ b/config/seed_nodes.go @@ -22,9 +22,9 @@ package config // only version 2 var Mainnet_seed_nodes = []string{ "89.38.99.117:8443", - "45.82.66.54:8080", + "109.236.81.137:8080", "89.38.97.110:11011", - "45.82.66.55:11011", + "190.2.136.120:11011", } // some seed node for testnet diff --git a/config/version.go b/config/version.go index 1939a24d..c068770b 100644 --- a/config/version.go +++ b/config/version.go @@ -20,4 +20,4 @@ import "github.com/blang/semver/v4" // right now it has to be manually changed // do we need to include git commitsha?? -var Version = semver.MustParse("3.5.2-113.DEROHE.STARGATE+01102022") +var Version = semver.MustParse("3.5.2-114.DEROHE.STARGATE+01102022") diff --git a/cryptography/crypto/algebra_fieldvector.go b/cryptography/crypto/algebra_fieldvector.go index a069b423..5b98c5dc 100644 --- a/cryptography/crypto/algebra_fieldvector.go +++ b/cryptography/crypto/algebra_fieldvector.go @@ -55,7 +55,7 @@ func (fv *FieldVector) Slice(start, end int) *FieldVector { return &result } -//copy and return +// copy and return func (fv *FieldVector) Clone() *FieldVector { return fv.Slice(0, len(fv.vector)) } diff --git a/cryptography/crypto/proof_verify.go b/cryptography/crypto/proof_verify.go index 1ef466d0..1de18549 100644 --- a/cryptography/crypto/proof_verify.go +++ b/cryptography/crypto/proof_verify.go @@ -469,53 +469,53 @@ func (proof *Proof) Verify(scid Hash, scid_index int, s *Statement, txid Hash, e } /* -func (proof *Proof) String() string { - klog.V(1).Infof("proof BA %s\n", proof.BA.String()) - klog.V(1).Infof("proof BS %s\n", proof.BS.String()) - klog.V(1).Infof("proof A %s\n", proof.A.String()) - klog.V(1).Infof("proof B %s\n", proof.B.String()) - - for i := range proof.CLnG { - klog.V(1).Infof("CLnG %d %s \n", i, proof.CLnG[i].String()) - } - for i := range proof.CRnG { - klog.V(1).Infof("CRnG %d %s \n", i, proof.CRnG[i].String()) - } + func (proof *Proof) String() string { + klog.V(1).Infof("proof BA %s\n", proof.BA.String()) + klog.V(1).Infof("proof BS %s\n", proof.BS.String()) + klog.V(1).Infof("proof A %s\n", proof.A.String()) + klog.V(1).Infof("proof B %s\n", proof.B.String()) + + for i := range proof.CLnG { + klog.V(1).Infof("CLnG %d %s \n", i, proof.CLnG[i].String()) + } + for i := range proof.CRnG { + klog.V(1).Infof("CRnG %d %s \n", i, proof.CRnG[i].String()) + } - for i := range proof.C_0G { - klog.V(1).Infof("C_0G %d %s \n", i, proof.C_0G[i].String()) - } - for i := range proof.DG { - klog.V(1).Infof("DG %d %s \n", i, proof.DG[i].String()) - } - for i := range proof.y_0G { - klog.V(1).Infof("y_0G %d %s \n", i, proof.y_0G[i].String()) - } - for i := range proof.gG { - klog.V(1).Infof("gG %d %s \n", i, proof.gG[i].String()) - } + for i := range proof.C_0G { + klog.V(1).Infof("C_0G %d %s \n", i, proof.C_0G[i].String()) + } + for i := range proof.DG { + klog.V(1).Infof("DG %d %s \n", i, proof.DG[i].String()) + } + for i := range proof.y_0G { + klog.V(1).Infof("y_0G %d %s \n", i, proof.y_0G[i].String()) + } + for i := range proof.gG { + klog.V(1).Infof("gG %d %s \n", i, proof.gG[i].String()) + } - for i := range proof.C_XG { - klog.V(1).Infof("C_XG %d %s \n", i, proof.C_XG[i].String()) - } - for i := range proof.y_XG { - klog.V(1).Infof("y_XG %d %s \n", i, proof.y_XG[i].String()) - } + for i := range proof.C_XG { + klog.V(1).Infof("C_XG %d %s \n", i, proof.C_XG[i].String()) + } + for i := range proof.y_XG { + klog.V(1).Infof("y_XG %d %s \n", i, proof.y_XG[i].String()) + } - //for i := range proof.tCommits.vector { - // klog.V(1).Infof("tCommits %d %s \n", i, proof.tCommits.vector[i].String()) - //} + //for i := range proof.tCommits.vector { + // klog.V(1).Infof("tCommits %d %s \n", i, proof.tCommits.vector[i].String()) + //} - klog.V(1).Infof("proof z_A %s\n", proof.z_A.Text(16)) - klog.V(1).Infof("proof that %s\n", proof.that.Text(16)) - klog.V(1).Infof("proof mu %s\n", proof.mu.Text(16)) - klog.V(1).Infof("proof C %s\n", proof.c.Text(16)) - klog.V(1).Infof("proof s_sk %s\n", proof.s_sk.Text(16)) - klog.V(1).Infof("proof s_r %s\n", proof.s_r.Text(16)) - klog.V(1).Infof("proof s_b %s\n", proof.s_b.Text(16)) - klog.V(1).Infof("proof s_tau %s\n", proof.s_tau.Text(16)) + klog.V(1).Infof("proof z_A %s\n", proof.z_A.Text(16)) + klog.V(1).Infof("proof that %s\n", proof.that.Text(16)) + klog.V(1).Infof("proof mu %s\n", proof.mu.Text(16)) + klog.V(1).Infof("proof C %s\n", proof.c.Text(16)) + klog.V(1).Infof("proof s_sk %s\n", proof.s_sk.Text(16)) + klog.V(1).Infof("proof s_r %s\n", proof.s_r.Text(16)) + klog.V(1).Infof("proof s_b %s\n", proof.s_b.Text(16)) + klog.V(1).Infof("proof s_tau %s\n", proof.s_tau.Text(16)) - return "" + return "" } */ diff --git a/cryptography/crypto/userdata.go b/cryptography/crypto/userdata.go index dd59d163..daa77448 100644 --- a/cryptography/crypto/userdata.go +++ b/cryptography/crypto/userdata.go @@ -26,7 +26,6 @@ var Logger logr.Logger = logr.Discard() // default discard all logs, someone nee // this function is used to encrypt/decrypt payment id,srcid and other userdata // as the operation is symmetric XOR, is the same in both direction -// func EncryptDecryptUserData(key [32]byte, inputs ...[]byte) { var nonce [24]byte // nonce is 24 bytes, we will use xchacha20 diff --git a/dvm/dvm.go b/dvm/dvm.go index 22d92ed5..5f66f3c3 100644 --- a/dvm/dvm.go +++ b/dvm/dvm.go @@ -435,7 +435,7 @@ type Shared_State struct { } -//consumr and check compute gas +// consumr and check compute gas func (state *Shared_State) ConsumeGas(c int64) { if state != nil { state.GasComputeUsed += c diff --git a/dvm/dvm_functions.go b/dvm/dvm_functions.go index 010e1368..a7613830 100644 --- a/dvm/dvm_functions.go +++ b/dvm/dvm_functions.go @@ -292,12 +292,12 @@ func dvm_block_height(dvm *DVM_Interpreter, expr *ast.CallExpr) (handled bool, r } /* -func dvm_block_topoheight(dvm *DVM_Interpreter, expr *ast.CallExpr)(handled bool, result interface{}){ - if len(expr.Args) != 0 { - panic("BLOCK_HEIGHT function expects 0 parameters") - } - return true, dvm.State.Chain_inputs.BL_TOPOHEIGHT -} + func dvm_block_topoheight(dvm *DVM_Interpreter, expr *ast.CallExpr)(handled bool, result interface{}){ + if len(expr.Args) != 0 { + panic("BLOCK_HEIGHT function expects 0 parameters") + } + return true, dvm.State.Chain_inputs.BL_TOPOHEIGHT + } */ func dvm_block_timestamp(dvm *DVM_Interpreter, expr *ast.CallExpr) (handled bool, result uint64) { checkargscount(0, len(expr.Args)) // check number of arguments diff --git a/globals/globals.go b/globals/globals.go index 3d30743d..be27158c 100644 --- a/globals/globals.go +++ b/globals/globals.go @@ -266,7 +266,6 @@ func GetDataDirectory() string { // never do any division operation on money due to floating point issues // newbies, see type the next in python interpretor "3.33-3.13" -// func FormatMoney(amount uint64) string { return FormatMoneyPrecision(amount, 5) // default is 5 precision after floating point } @@ -276,12 +275,12 @@ func FormatMoney0(amount uint64) string { return FormatMoneyPrecision(amount, 0) } -//5 precision +// 5 precision func FormatMoney5(amount uint64) string { return FormatMoneyPrecision(amount, 5) } -//8 precision +// 8 precision func FormatMoney8(amount uint64) string { return FormatMoneyPrecision(amount, 8) } diff --git a/p2p/bans.go b/p2p/bans.go index 0065592e..8727614e 100644 --- a/p2p/bans.go +++ b/p2p/bans.go @@ -80,7 +80,7 @@ func load_ban_list() { } } -//save ban list to disk +// save ban list to disk func save_ban_list() { ban_clean_up() // cleanup before saving @@ -163,7 +163,9 @@ func ParseAddress(address string) (ipnet *net.IPNet, result string, err error) { } // check whether an IP is in the map already -// we should loop and search in subnets also +// +// we should loop and search in subnets also +// // TODO make it fast, however we are not expecting millions of bans, so this may be okay for time being func IsAddressInBanList(address string) bool { ban_mutex.Lock() diff --git a/p2p/chain_bootstrap.go b/p2p/chain_bootstrap.go index 38090284..cc023958 100644 --- a/p2p/chain_bootstrap.go +++ b/p2p/chain_bootstrap.go @@ -104,7 +104,7 @@ func (connection *Connection) bootstrap_chain() { binary.BigEndian.PutUint64(section[:], bits.Reverse64(uint64(i))) // place reverse path ts_request := Request_Tree_Section_Struct{Topo: request.TopoHeights[0], TreeName: []byte(config.BALANCE_TREE), Section: section[:], SectionLength: uint64(path_length)} var ts_response Response_Tree_Section_Struct - fill_common(&ts_response.Common) + fill_common(&ts_request.Common) if err := connection.Client.Call("Peer.TreeSection", ts_request, &ts_response); err != nil { connection.logger.V(1).Error(err, "Call failed TreeSection") return @@ -168,7 +168,7 @@ func (connection *Connection) bootstrap_chain() { binary.BigEndian.PutUint64(section[:], bits.Reverse64(uint64(i))) // place reverse path ts_request := Request_Tree_Section_Struct{Topo: request.TopoHeights[0], TreeName: []byte(config.SC_META), Section: section[:], SectionLength: uint64(path_length)} var ts_response Response_Tree_Section_Struct - fill_common(&ts_response.Common) + fill_common(&ts_request.Common) if err := connection.Client.Call("Peer.TreeSection", ts_request, &ts_response); err != nil { connection.logger.V(1).Error(err, "Call failed TreeSection") return @@ -198,21 +198,58 @@ func (connection *Connection) bootstrap_chain() { sc_request := Request_Tree_Section_Struct{Topo: request.TopoHeights[0], TreeName: ts_response.Keys[j], Section: section[:], SectionLength: uint64(0)} var sc_response Response_Tree_Section_Struct - fill_common(&sc_response.Common) + fill_common(&sc_request.Common) if err := connection.Client.Call("Peer.TreeSection", sc_request, &sc_response); err != nil { connection.logger.V(1).Error(err, "Call failed TreeSection") return } else { + var sc_data_tree *graviton.Tree if sc_data_tree, err = ss.GetTree(string(ts_response.Keys[j])); err != nil { panic(err) - } else { - for j := range sc_response.Keys { - sc_data_tree.Put(sc_response.Keys[j], sc_response.Values[j]) + } else if sc_response.KeyCount < 4096 { + for k := range sc_response.Keys { + sc_data_tree.Put(sc_response.Keys[k], sc_response.Values[k]) + } + } else { // tree is a huge tree, get it in chunks + sc_chunks_estm := sc_response.KeyCount / chunksize + sc_chunks := int64(1) // chunks need to be in power of 2 + sc_path_length := 0 + for sc_chunks < sc_chunks_estm { + sc_chunks = sc_chunks * 2 + sc_path_length++ + } + + if sc_chunks < 2 { + sc_chunks = 2 + sc_path_length = 1 } - changed_trees = append(changed_trees, sc_data_tree) + var sc_section [8]byte + for k := int64(0); k < sc_chunks; k++ { + binary.BigEndian.PutUint64(sc_section[:], bits.Reverse64(uint64(k))) // place reverse path + sc_ts_request := Request_Tree_Section_Struct{Topo: request.TopoHeights[0], TreeName: ts_response.Keys[j], Section: sc_section[:], SectionLength: uint64(sc_path_length)} + var sc_ts_response Response_Tree_Section_Struct + fill_common(&sc_ts_request.Common) + if err := connection.Client.Call("Peer.TreeSection", sc_ts_request, &sc_ts_response); err != nil { + connection.logger.V(1).Error(err, "Call failed TreeSection") + return + } else { // request was successfull + + if len(sc_ts_response.Keys) != len(sc_ts_response.Values) { + connection.logger.V(1).Error(nil, "Wrong key/values", "Keycount", len(sc_ts_response.Keys), "valuecount", len(sc_ts_response.Values)) + connection.exit() + return + } + //fmt.Printf("writing SC chunk %d/%d (%d) writing %d keys %x\n", k, sc_chunks, sc_response.KeyCount, len(sc_ts_response.Keys), ts_response.Keys[j]) + for l := range sc_ts_response.Keys { + sc_data_tree.Put(sc_ts_response.Keys[l], sc_ts_response.Values[l]) + } + } + + } } + changed_trees = append(changed_trees, sc_data_tree) } @@ -237,6 +274,7 @@ func (connection *Connection) bootstrap_chain() { for i := int64(0); i <= request.TopoHeights[0]; i++ { chain.Store.Topo_store.Write(i, zerohash, commit_version, 0) // commit everything } + chain.Store.Topo_store.Sync() for i := range response.CBlocks { // we must store the blocks diff --git a/p2p/controller.go b/p2p/controller.go index 0f4658f4..165d128b 100644 --- a/p2p/controller.go +++ b/p2p/controller.go @@ -263,11 +263,11 @@ func P2P_engine() { // trigger connection to all seed nodes hoping some will be up if globals.IsMainnet() { // initial boot strap should be quick for i := range config.Mainnet_seed_nodes { - go connect_with_endpoint(config.Mainnet_seed_nodes[i], true) + go connect_with_endpoint(config.Mainnet_seed_nodes[i], sync_node) } } else { // initial bootstrap for i := range config.Testnet_seed_nodes { - go connect_with_endpoint(config.Testnet_seed_nodes[i], true) + go connect_with_endpoint(config.Testnet_seed_nodes[i], sync_node) } } @@ -434,8 +434,8 @@ func maintain_seed_node_connection() { endpoint = config.Testnet_seed_nodes[r.Int64()%int64(len(config.Testnet_seed_nodes))] } if endpoint != "" { - //connect_with_endpoint(endpoint, sync_node) - connect_with_endpoint(endpoint, true) // seed nodes always have sync mode + connect_with_endpoint(endpoint, sync_node) + //connect_with_endpoint(endpoint, true) // seed nodes always have sync mode } } } diff --git a/p2p/peer.go b/p2p/peer.go index d889ccaa..c3272400 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -45,7 +45,7 @@ func GetPeerID() uint64 { if peerid == 0 { var buf [8]byte rand.Read(buf[:]) - peerid = binary.LittleEndian.Uint64(buf[:]) & 0x7FFFFFFFFFFFFFFF + peerid = binary.LittleEndian.Uint64(buf[:]) & 0x7FFFFFFFFFFFFFFF } return peerid } diff --git a/p2p/peer_pool.go b/p2p/peer_pool.go index c48348fe..1e2813bf 100644 --- a/p2p/peer_pool.go +++ b/p2p/peer_pool.go @@ -92,7 +92,7 @@ func load_peer_list() { } -//save peer list to disk +// save peer list to disk func save_peer_list() { clean_up() diff --git a/p2p/rpc_cbor_codec.go b/p2p/rpc_cbor_codec.go index 1956d697..7137189e 100644 --- a/p2p/rpc_cbor_codec.go +++ b/p2p/rpc_cbor_codec.go @@ -13,7 +13,7 @@ import "github.com/fxamacker/cbor/v2" import "github.com/deroproject/derohe/config" // only used get constants such as max data per frame -// it processes both +// it processes both type RequestResponse struct { Method string `cbor:"M"` // format: "Service.Method" Seq uint64 `cbor:"S"` // echoes that of the request diff --git a/p2p/rpc_treesection.go b/p2p/rpc_treesection.go index af864d70..ec2d8858 100644 --- a/p2p/rpc_treesection.go +++ b/p2p/rpc_treesection.go @@ -39,6 +39,7 @@ func (c *Connection) TreeSection(request Request_Tree_Section_Struct, response * if topo_ss, err = chain.Store.Balance_store.LoadSnapshot(topo_sr.State_Version); err == nil { if topo_balance_tree, err = topo_ss.GetTree(string(request.TreeName)); err == nil { cursor := topo_balance_tree.Cursor() + response.KeyCount = topo_balance_tree.KeyCountEstimate() for k, v, err := cursor.SpecialFirst(request.Section, uint(request.SectionLength)); err == nil; k, v, err = cursor.Next() { response.Keys = append(response.Keys, k) response.Values = append(response.Values, v) diff --git a/p2p/time.go b/p2p/time.go index e3699007..9d6e8213 100644 --- a/p2p/time.go +++ b/p2p/time.go @@ -4,11 +4,10 @@ package p2p // yes, this project contains a local averaging distributed clock syncronisation algorithm import "time" -// -// org = Origin Timestamp (client send time) -// rec = Receive Timestamp (server receive time) -// xmt = Transmit Timestamp (server reply time) -// dst = Destination Timestamp (client receive time) +// org = Origin Timestamp (client send time) +// rec = Receive Timestamp (server receive time) +// xmt = Transmit Timestamp (server reply time) +// dst = Destination Timestamp (client receive time) func rtt(org, rec, xmt, dst time.Time) time.Duration { // round trip delay time rtt = (dst-org) - (xmt-rec) a := dst.Sub(org) diff --git a/p2p/wire_structs.go b/p2p/wire_structs.go index 14ac43b2..eb46898f 100644 --- a/p2p/wire_structs.go +++ b/p2p/wire_structs.go @@ -99,7 +99,7 @@ type Objects struct { Chunks []Block_Chunk `cbor:"CHUNKS,omitempty"` // all requested chunks are here } -// used to request what all changes are done by the block to the chain +// used to request what all changes are done by the block to the chain type ChangeList struct { Common Common_Struct `cbor:"COMMON"` // add all fields of Common TopoHeights []int64 `cbor:"TOPO"` @@ -129,6 +129,7 @@ type Response_Tree_Section_Struct struct { StateHash [32]byte `cbor:"STATE"` Keys [][]byte `cbor:"KEYS,omitempty"` // changes to state tree Values [][]byte `cbor:"VALUES,omitempty"` // changes to state tree + KeyCount int64 `cbor:"KEYCOUNT"` // estimated keys in this tree } type Tree_Changes struct { diff --git a/rpc/rpc.go b/rpc/rpc.go index 461fdfa9..d00bca4e 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -78,7 +78,7 @@ func (d DataType) String() string { } -//type DataType byte +// type DataType byte type Argument struct { Name string `json:"name"` // string name must be atleast 1 byte DataType DataType `json:"datatype"` // Type must one of the known data types diff --git a/rpc/wallet_rpc.go b/rpc/wallet_rpc.go index 8a5185a3..9a38e60a 100644 --- a/rpc/wallet_rpc.go +++ b/rpc/wallet_rpc.go @@ -130,7 +130,6 @@ func (e *Entry) ProcessPayload() (args Arguments, err error) { // never do any division operation on money due to floating point issues // newbies, see type the next in python interpretor "3.33-3.13" -// func FormatMoney(amount uint64) string { return FormatMoneyPrecision(amount, 5) // default is 5 precision after floating point } @@ -140,12 +139,12 @@ func FormatMoney0(amount uint64) string { return FormatMoneyPrecision(amount, 0) } -//5 precision +// 5 precision func FormatMoney5(amount uint64) string { return FormatMoneyPrecision(amount, 5) } -//8 precision +// 8 precision func FormatMoney8(amount uint64) string { return FormatMoneyPrecision(amount, 8) } diff --git a/vendor/github.com/deroproject/graviton/extra.go b/vendor/github.com/deroproject/graviton/extra.go index 84094d93..7943502d 100644 --- a/vendor/github.com/deroproject/graviton/extra.go +++ b/vendor/github.com/deroproject/graviton/extra.go @@ -72,8 +72,8 @@ func (t *Tree) KeyCountEstimate() (count int64) { break } } - if len(depth_array) <= 4 { - return int64(count) + if len(depth_array) <= 19 { + return int64(len(depth_array)) } avg := floatsum / float64(len(depth_array)+1) return int64(math.Exp2(avg)) diff --git a/walletapi/mnemonics/mnemonics.go b/walletapi/mnemonics/mnemonics.go index b4f5440c..41017007 100644 --- a/walletapi/mnemonics/mnemonics.go +++ b/walletapi/mnemonics/mnemonics.go @@ -69,7 +69,7 @@ func Language_List() (list []string) { return } -//this function converts a list of words to a key +// this function converts a list of words to a key func Words_To_Key(words_line string) (language_name string, keybig *big.Int, err error) { var key [32]byte