From 12f0990d4f3d61f7a909714c41e88eeae0452c33 Mon Sep 17 00:00:00 2001 From: Sergi Rene Date: Wed, 4 Sep 2024 11:26:20 +0200 Subject: [PATCH 1/7] add last submitted height to state stored --- block/manager.go | 10 ++++++++++ block/state.go | 1 + block/submit.go | 5 +++++ types/state.go | 3 +++ 4 files changed, 19 insertions(+) diff --git a/block/manager.go b/block/manager.go index 1b68d7261..fa02383fa 100644 --- a/block/manager.go +++ b/block/manager.go @@ -256,6 +256,11 @@ func (m *Manager) syncFromSettlement() error { // The SL hasn't got any batches for this chain yet. m.logger.Info("No batches for chain found in SL.") m.LastSubmittedHeight.Store(uint64(m.Genesis.InitialHeight - 1)) + m.State.LastSubmittedHeight = uint64(m.Genesis.InitialHeight - 1) + _, err = m.Store.SaveState(m.State, nil) + if err != nil { + return fmt.Errorf("save state: %w", err) + } return nil } @@ -264,6 +269,11 @@ func (m *Manager) syncFromSettlement() error { return err } m.LastSubmittedHeight.Store(res.EndHeight) + m.State.LastSubmittedHeight = res.EndHeight + _, err = m.Store.SaveState(m.State, nil) + if err != nil { + return fmt.Errorf("save state: %w", err) + } err = m.syncToTargetHeight(res.EndHeight) m.UpdateTargetHeight(res.EndHeight) if err != nil { diff --git a/block/state.go b/block/state.go index 202268bee..a358c0e36 100644 --- a/block/state.go +++ b/block/state.go @@ -65,6 +65,7 @@ func NewStateFromGenesis(genDoc *tmtypes.GenesisDoc) (*types.State, error) { LastHeightConsensusParamsChanged: genDoc.InitialHeight, } s.SetHeight(0) + s.LastSubmittedHeight = 0 copy(s.AppHash[:], genDoc.AppHash) err = s.SetConsensusParamsFromGenesis(genDoc.AppState) diff --git a/block/submit.go b/block/submit.go index cf0fc5280..66fb8b2aa 100644 --- a/block/submit.go +++ b/block/submit.go @@ -228,6 +228,11 @@ func (m *Manager) SubmitBatch(batch *types.Batch) error { types.RollappHubHeightGauge.Set(float64(batch.EndHeight())) m.LastSubmittedHeight.Store(batch.EndHeight()) + m.State.LastSubmittedHeight = batch.EndHeight() + _, err = m.Store.SaveState(m.State, nil) + if err != nil { + return fmt.Errorf("save state: %w", err) + } return nil } diff --git a/types/state.go b/types/state.go index 5a32a8b9d..a4066da1e 100644 --- a/types/state.go +++ b/types/state.go @@ -38,6 +38,9 @@ type State struct { // the latest AppHash we've received from calling abci.Commit() AppHash [32]byte + + // LastSubmittedHeight to DA + LastSubmittedHeight uint64 } func (s *State) IsGenesis() bool { From b51d78eaf754ff679ed6100393f3c4bb72722987 Mon Sep 17 00:00:00 2001 From: Sergi Rene Date: Wed, 4 Sep 2024 12:01:47 +0200 Subject: [PATCH 2/7] missing proto --- proto/types/dymint/state.proto | 2 +- types/pb/dymint/state.pb.go | 129 +++++++++++++++++++++------------ types/serialization.go | 2 + 3 files changed, 87 insertions(+), 46 deletions(-) diff --git a/proto/types/dymint/state.proto b/proto/types/dymint/state.proto index a12631126..d08a07006 100755 --- a/proto/types/dymint/state.proto +++ b/proto/types/dymint/state.proto @@ -44,7 +44,7 @@ message State { SequencerSet sequencerSet = 18 [(gogoproto.nullable) = false]; RollappConsensusParams consensus_params = 19 [(gogoproto.nullable) = false]; - + uint64 last_submitted_height = 20; } //rollapp params defined in genesis and updated via gov proposal diff --git a/types/pb/dymint/state.pb.go b/types/pb/dymint/state.pb.go index 742a1c91d..42157ff2d 100644 --- a/types/pb/dymint/state.pb.go +++ b/types/pb/dymint/state.pb.go @@ -46,6 +46,7 @@ type State struct { BaseHeight uint64 `protobuf:"varint,17,opt,name=base_height,json=baseHeight,proto3" json:"base_height,omitempty"` SequencerSet SequencerSet `protobuf:"bytes,18,opt,name=sequencerSet,proto3" json:"sequencerSet"` ConsensusParams RollappConsensusParams `protobuf:"bytes,19,opt,name=consensus_params,json=consensusParams,proto3" json:"consensus_params"` + LastSubmittedHeight uint64 `protobuf:"varint,20,opt,name=last_submitted_height,json=lastSubmittedHeight,proto3" json:"last_submitted_height,omitempty"` } func (m *State) Reset() { *m = State{} } @@ -187,6 +188,13 @@ func (m *State) GetConsensusParams() RollappConsensusParams { return RollappConsensusParams{} } +func (m *State) GetLastSubmittedHeight() uint64 { + if m != nil { + return m.LastSubmittedHeight + } + return 0 +} + //rollapp params defined in genesis and updated via gov proposal type RollappConsensusParams struct { //maximum amount of gas that all transactions included in a block can use @@ -268,51 +276,53 @@ func init() { func init() { proto.RegisterFile("types/dymint/state.proto", fileDescriptor_4b679420add07272) } var fileDescriptor_4b679420add07272 = []byte{ - // 703 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x94, 0xcb, 0x6e, 0xda, 0x4a, - 0x18, 0xc7, 0x31, 0x10, 0x30, 0x03, 0x04, 0x32, 0x89, 0x22, 0x27, 0x47, 0x32, 0x1c, 0xce, 0x45, - 0xa8, 0x0b, 0x23, 0x35, 0xfb, 0x56, 0x72, 0xb2, 0x08, 0x28, 0x6a, 0xab, 0xa1, 0xca, 0xa2, 0x1b, - 0x34, 0xd8, 0x53, 0x7b, 0x54, 0xdf, 0xca, 0x0c, 0x51, 0x92, 0x07, 0xe8, 0x3a, 0x8f, 0x95, 0x65, - 0x96, 0x5d, 0xa5, 0x15, 0x79, 0x88, 0x6e, 0xab, 0xb9, 0x98, 0x98, 0xa4, 0x59, 0xc1, 0xfc, 0xbf, - 0xdf, 0xf7, 0xf7, 0xf8, 0xbb, 0x18, 0x58, 0xfc, 0x2a, 0x23, 0x6c, 0xe4, 0x5f, 0xc5, 0x34, 0xe1, - 0x23, 0xc6, 0x31, 0x27, 0x4e, 0xb6, 0x48, 0x79, 0x0a, 0x6b, 0x4a, 0x3b, 0xdc, 0x0b, 0xd2, 0x20, - 0x95, 0xd2, 0x48, 0xfc, 0x53, 0xd1, 0xc3, 0x5e, 0x90, 0xa6, 0x41, 0x44, 0x46, 0xf2, 0x34, 0x5f, - 0x7e, 0x1e, 0x71, 0x1a, 0x13, 0xc6, 0x71, 0x9c, 0x69, 0xe0, 0x6f, 0x65, 0xcc, 0x49, 0xe2, 0x93, - 0x85, 0x34, 0xc7, 0x73, 0x8f, 0x8e, 0xa4, 0xaa, 0x91, 0xc1, 0x33, 0x44, 0x0b, 0x05, 0xe6, 0xff, - 0x17, 0x98, 0x0b, 0x1c, 0x51, 0x1f, 0xf3, 0x74, 0xa1, 0xb9, 0x7f, 0x5e, 0xe0, 0x32, 0xbc, 0xc0, - 0xf1, 0xcb, 0x0f, 0x94, 0x2f, 0xbc, 0xf1, 0xc0, 0x83, 0x8d, 0x82, 0xa8, 0x1f, 0x15, 0x1a, 0xfc, - 0xaa, 0x81, 0xad, 0xa9, 0x48, 0x80, 0x47, 0xa0, 0x7e, 0x41, 0x16, 0x8c, 0xa6, 0x89, 0x65, 0xf4, - 0x8d, 0x61, 0xf3, 0xf5, 0x81, 0xf3, 0x68, 0xea, 0xa8, 0x2a, 0x9e, 0x2b, 0x00, 0xe5, 0x24, 0x3c, - 0x00, 0xa6, 0x17, 0x62, 0x9a, 0xcc, 0xa8, 0x6f, 0x95, 0xfb, 0xc6, 0xb0, 0x81, 0xea, 0xf2, 0x3c, - 0xf6, 0xe1, 0x7f, 0x60, 0x9b, 0x26, 0x94, 0x53, 0x1c, 0xcd, 0x42, 0x42, 0x83, 0x90, 0x5b, 0x95, - 0xbe, 0x31, 0xac, 0xa0, 0xb6, 0x56, 0x4f, 0xa5, 0x08, 0x5f, 0x81, 0x9d, 0x08, 0x33, 0x3e, 0x9b, - 0x47, 0xa9, 0xf7, 0x25, 0x27, 0xab, 0x92, 0xec, 0x88, 0x80, 0x2b, 0x74, 0xcd, 0x22, 0xd0, 0x2e, - 0xb0, 0xd4, 0xb7, 0xb6, 0x9e, 0x5f, 0x54, 0xbd, 0xb7, 0xcc, 0x1a, 0x9f, 0xb8, 0xbb, 0xb7, 0xf7, - 0xbd, 0xd2, 0xea, 0xbe, 0xd7, 0x3c, 0xcb, 0xad, 0xc6, 0x27, 0xa8, 0xb9, 0xf6, 0x1d, 0xfb, 0xf0, - 0x0c, 0x74, 0x0a, 0x9e, 0xa2, 0xe3, 0x56, 0x4d, 0xba, 0x1e, 0x3a, 0x6a, 0x1c, 0x9c, 0x7c, 0x1c, - 0x9c, 0x8f, 0xf9, 0x38, 0xb8, 0xa6, 0xb0, 0xbd, 0xf9, 0xd1, 0x33, 0x50, 0x7b, 0xed, 0x25, 0xa2, - 0xd0, 0x05, 0x60, 0xdd, 0x45, 0x66, 0x35, 0xa4, 0x91, 0xfd, 0xfc, 0x7a, 0xe7, 0x39, 0x33, 0x25, - 0xdc, 0x2d, 0x5b, 0x06, 0x2a, 0x64, 0xc1, 0x63, 0x60, 0xcb, 0x1b, 0xa9, 0x5a, 0xcc, 0x1e, 0x23, - 0x33, 0x2f, 0xc4, 0x49, 0x40, 0x7c, 0xab, 0x29, 0xcb, 0xf3, 0x97, 0xa0, 0x54, 0x65, 0xd6, 0x7e, - 0xec, 0x58, 0x21, 0xf0, 0x1d, 0xf8, 0xb7, 0x68, 0xe2, 0xa5, 0x09, 0x23, 0x09, 0x5b, 0xb2, 0x99, - 0x1a, 0x9e, 0xb5, 0x55, 0x5b, 0x5a, 0xf5, 0x1f, 0xad, 0x8e, 0x73, 0xf2, 0x83, 0x04, 0x73, 0xbf, - 0xbc, 0x4d, 0x0b, 0xc2, 0x96, 0x11, 0x67, 0xb3, 0x10, 0xb3, 0xd0, 0xda, 0xee, 0x1b, 0xc3, 0x96, - 0x6a, 0x13, 0x52, 0xfa, 0x29, 0x66, 0xa1, 0x18, 0x0a, 0x9c, 0x65, 0x0a, 0xe9, 0x48, 0xa4, 0x8e, - 0xb3, 0x4c, 0x86, 0xde, 0x6a, 0x1b, 0xc6, 0xd3, 0x05, 0xc9, 0xbb, 0xdd, 0xed, 0x1b, 0xc3, 0xaa, - 0xbb, 0xbb, 0xba, 0xef, 0x75, 0x44, 0x9b, 0xa6, 0x22, 0xa6, 0x2e, 0xa3, 0xbc, 0x0b, 0x02, 0xec, - 0x81, 0xe6, 0x1c, 0xb3, 0x75, 0xea, 0x8e, 0x48, 0x45, 0x40, 0x48, 0x1a, 0x78, 0x03, 0x5a, 0x8c, - 0x7c, 0x5d, 0x92, 0xc4, 0x23, 0xa2, 0xba, 0x16, 0x94, 0x3d, 0xd8, 0x73, 0xf4, 0xd4, 0x4f, 0x0b, - 0x31, 0xb7, 0x2a, 0xda, 0x88, 0x36, 0x78, 0xf8, 0x1e, 0x74, 0x9f, 0x16, 0xcb, 0xda, 0xd5, 0x7d, - 0xd4, 0x1e, 0x28, 0x8d, 0x22, 0x9c, 0x65, 0x4f, 0x2a, 0xa5, 0xdd, 0x3a, 0xde, 0xa6, 0x3c, 0xa9, - 0x9a, 0xf5, 0xae, 0x39, 0xa9, 0x9a, 0x66, 0xb7, 0x31, 0xa9, 0x9a, 0xa0, 0xdb, 0x9c, 0x54, 0xcd, - 0x56, 0xb7, 0x3d, 0xf8, 0x66, 0x80, 0xfd, 0x3f, 0x3b, 0xc1, 0x3e, 0x68, 0xca, 0x71, 0x8c, 0xf1, - 0x65, 0x80, 0x99, 0x5c, 0xc7, 0x0a, 0x2a, 0x4a, 0x70, 0x00, 0x5a, 0xf9, 0x91, 0xd1, 0x6b, 0x22, - 0x77, 0xaf, 0x82, 0x36, 0x34, 0xb8, 0x0d, 0xca, 0x3e, 0x96, 0x4b, 0xd7, 0x40, 0x65, 0x1f, 0xc3, - 0x7d, 0x50, 0xf3, 0xd2, 0x38, 0xa6, 0x6a, 0xbd, 0x1a, 0x48, 0x9f, 0xdc, 0xd3, 0xdb, 0x95, 0x6d, - 0xdc, 0xad, 0x6c, 0xe3, 0xe7, 0xca, 0x36, 0x6e, 0x1e, 0xec, 0xd2, 0xdd, 0x83, 0x5d, 0xfa, 0xfe, - 0x60, 0x97, 0x3e, 0x39, 0x01, 0xe5, 0xe1, 0x72, 0xee, 0x78, 0x69, 0x2c, 0xbe, 0x1a, 0x24, 0x11, - 0x3b, 0x7f, 0x79, 0x75, 0x9d, 0x7f, 0x49, 0xf4, 0xe7, 0x68, 0xae, 0xcf, 0xf3, 0x9a, 0x5c, 0x95, - 0xa3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb2, 0x64, 0xcb, 0x4f, 0x81, 0x05, 0x00, 0x00, + // 729 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x94, 0xcd, 0x6e, 0xda, 0x4a, + 0x14, 0xc7, 0x31, 0x10, 0x30, 0x03, 0x04, 0x32, 0xe4, 0x46, 0x4e, 0xae, 0x64, 0xb8, 0xdc, 0x7b, + 0x2b, 0xd4, 0x85, 0x91, 0x92, 0x7d, 0x2b, 0x39, 0x59, 0x04, 0x14, 0xb5, 0x95, 0xa9, 0xb2, 0xe8, + 0x06, 0x8d, 0xed, 0xa9, 0x3d, 0xaa, 0xbf, 0xca, 0x0c, 0x51, 0x92, 0x07, 0xe8, 0x3a, 0xef, 0xd1, + 0x17, 0xc9, 0x32, 0xcb, 0xae, 0xd2, 0x8a, 0xbc, 0x48, 0x35, 0x1f, 0x26, 0x26, 0x69, 0x56, 0x30, + 0xff, 0xf3, 0x9b, 0xbf, 0xcf, 0x9c, 0x73, 0x66, 0x80, 0xc1, 0xae, 0x32, 0x4c, 0xc7, 0xfe, 0x55, + 0x4c, 0x12, 0x36, 0xa6, 0x0c, 0x31, 0x6c, 0x65, 0x8b, 0x94, 0xa5, 0xb0, 0x26, 0xb5, 0x83, 0xdd, + 0x20, 0x0d, 0x52, 0x21, 0x8d, 0xf9, 0x3f, 0x19, 0x3d, 0xe8, 0x07, 0x69, 0x1a, 0x44, 0x78, 0x2c, + 0x56, 0xee, 0xf2, 0xf3, 0x98, 0x91, 0x18, 0x53, 0x86, 0xe2, 0x4c, 0x01, 0xff, 0x48, 0x63, 0x86, + 0x13, 0x1f, 0x2f, 0x84, 0x39, 0x72, 0x3d, 0x32, 0x16, 0xaa, 0x42, 0x86, 0xcf, 0x10, 0x25, 0x14, + 0x98, 0x57, 0x2f, 0x30, 0x17, 0x28, 0x22, 0x3e, 0x62, 0xe9, 0x42, 0x71, 0xff, 0xbe, 0xc0, 0x65, + 0x68, 0x81, 0xe2, 0x97, 0x3f, 0x28, 0x0e, 0xbc, 0xf1, 0xc1, 0xfd, 0x8d, 0x82, 0xc8, 0x1f, 0x19, + 0x1a, 0x7e, 0xaf, 0x83, 0xad, 0x19, 0xdf, 0x00, 0x8f, 0x40, 0xfd, 0x02, 0x2f, 0x28, 0x49, 0x13, + 0x43, 0x1b, 0x68, 0xa3, 0xe6, 0xe1, 0xbe, 0xf5, 0x68, 0x6a, 0xc9, 0x2a, 0x9e, 0x4b, 0xc0, 0xc9, + 0x49, 0xb8, 0x0f, 0x74, 0x2f, 0x44, 0x24, 0x99, 0x13, 0xdf, 0x28, 0x0f, 0xb4, 0x51, 0xc3, 0xa9, + 0x8b, 0xf5, 0xc4, 0x87, 0xff, 0x83, 0x6d, 0x92, 0x10, 0x46, 0x50, 0x34, 0x0f, 0x31, 0x09, 0x42, + 0x66, 0x54, 0x06, 0xda, 0xa8, 0xe2, 0xb4, 0x95, 0x7a, 0x2a, 0x44, 0xf8, 0x1a, 0xec, 0x44, 0x88, + 0xb2, 0xb9, 0x1b, 0xa5, 0xde, 0x97, 0x9c, 0xac, 0x0a, 0xb2, 0xc3, 0x03, 0x36, 0xd7, 0x15, 0xeb, + 0x80, 0x76, 0x81, 0x25, 0xbe, 0xb1, 0xf5, 0x3c, 0x51, 0x79, 0x6e, 0xb1, 0x6b, 0x72, 0x62, 0xf7, + 0x6e, 0xef, 0xfb, 0xa5, 0xd5, 0x7d, 0xbf, 0x79, 0x96, 0x5b, 0x4d, 0x4e, 0x9c, 0xe6, 0xda, 0x77, + 0xe2, 0xc3, 0x33, 0xd0, 0x29, 0x78, 0xf2, 0x8e, 0x1b, 0x35, 0xe1, 0x7a, 0x60, 0xc9, 0x71, 0xb0, + 0xf2, 0x71, 0xb0, 0x3e, 0xe6, 0xe3, 0x60, 0xeb, 0xdc, 0xf6, 0xe6, 0x67, 0x5f, 0x73, 0xda, 0x6b, + 0x2f, 0x1e, 0x85, 0x36, 0x00, 0xeb, 0x2e, 0x52, 0xa3, 0x21, 0x8c, 0xcc, 0xe7, 0xe9, 0x9d, 0xe7, + 0xcc, 0x0c, 0x33, 0xbb, 0x6c, 0x68, 0x4e, 0x61, 0x17, 0x3c, 0x06, 0xa6, 0xc8, 0x48, 0xd6, 0x62, + 0xfe, 0x18, 0x99, 0x7b, 0x21, 0x4a, 0x02, 0xec, 0x1b, 0x4d, 0x51, 0x9e, 0xbf, 0x39, 0x25, 0x2b, + 0xb3, 0xf6, 0xa3, 0xc7, 0x12, 0x81, 0xef, 0xc0, 0x7f, 0x45, 0x13, 0x2f, 0x4d, 0x28, 0x4e, 0xe8, + 0x92, 0xce, 0xe5, 0xf0, 0xac, 0xad, 0xda, 0xc2, 0x6a, 0xf0, 0x68, 0x75, 0x9c, 0x93, 0x1f, 0x04, + 0x98, 0xfb, 0xe5, 0x6d, 0x5a, 0x60, 0xba, 0x8c, 0x18, 0x9d, 0x87, 0x88, 0x86, 0xc6, 0xf6, 0x40, + 0x1b, 0xb5, 0x64, 0x9b, 0x1c, 0xa9, 0x9f, 0x22, 0x1a, 0xf2, 0xa1, 0x40, 0x59, 0x26, 0x91, 0x8e, + 0x40, 0xea, 0x28, 0xcb, 0x44, 0xe8, 0xad, 0xb2, 0xa1, 0x2c, 0x5d, 0xe0, 0xbc, 0xdb, 0xdd, 0x81, + 0x36, 0xaa, 0xda, 0xbd, 0xd5, 0x7d, 0xbf, 0xc3, 0xdb, 0x34, 0xe3, 0x31, 0x99, 0x8c, 0xf4, 0x2e, + 0x08, 0xb0, 0x0f, 0x9a, 0x2e, 0xa2, 0xeb, 0xad, 0x3b, 0x7c, 0xab, 0x03, 0xb8, 0xa4, 0x80, 0x37, + 0xa0, 0x45, 0xf1, 0xd7, 0x25, 0x4e, 0x3c, 0xcc, 0xab, 0x6b, 0x40, 0xd1, 0x83, 0x5d, 0x4b, 0x4d, + 0xfd, 0xac, 0x10, 0xb3, 0xab, 0xbc, 0x8d, 0xce, 0x06, 0x0f, 0xdf, 0x83, 0xee, 0xd3, 0x62, 0x19, + 0x3d, 0xd5, 0x47, 0xe5, 0xe1, 0xa4, 0x51, 0x84, 0xb2, 0xec, 0x49, 0xa5, 0x94, 0x5b, 0xc7, 0xdb, + 0x94, 0xe1, 0x21, 0xf8, 0x4b, 0x1e, 0x79, 0xe9, 0xc6, 0x84, 0x31, 0xec, 0xe7, 0xb9, 0xef, 0x8a, + 0xdc, 0x7b, 0xe2, 0x84, 0x79, 0x4c, 0x1e, 0x62, 0x5a, 0xd5, 0xeb, 0x5d, 0x7d, 0x5a, 0xd5, 0xf5, + 0x6e, 0x63, 0x5a, 0xd5, 0x41, 0xb7, 0x39, 0xad, 0xea, 0xad, 0x6e, 0x7b, 0xf8, 0x4d, 0x03, 0x7b, + 0x7f, 0xfe, 0x3a, 0x1c, 0x80, 0xa6, 0x18, 0xe1, 0x18, 0x5d, 0x06, 0x88, 0x8a, 0x2b, 0x5c, 0x71, + 0x8a, 0x12, 0x1c, 0x82, 0x56, 0xbe, 0xa4, 0xe4, 0x1a, 0x8b, 0xfb, 0x5a, 0x71, 0x36, 0x34, 0xb8, + 0x0d, 0xca, 0x3e, 0x12, 0x17, 0xb5, 0xe1, 0x94, 0x7d, 0x04, 0xf7, 0x40, 0xcd, 0x4b, 0xe3, 0x98, + 0xc8, 0x2b, 0xd9, 0x70, 0xd4, 0xca, 0x3e, 0xbd, 0x5d, 0x99, 0xda, 0xdd, 0xca, 0xd4, 0x7e, 0xad, + 0x4c, 0xed, 0xe6, 0xc1, 0x2c, 0xdd, 0x3d, 0x98, 0xa5, 0x1f, 0x0f, 0x66, 0xe9, 0x93, 0x15, 0x10, + 0x16, 0x2e, 0x5d, 0xcb, 0x4b, 0x63, 0xfe, 0xd2, 0xe0, 0x84, 0xbf, 0x13, 0x97, 0x57, 0xd7, 0xf9, + 0xeb, 0xa3, 0x9e, 0x30, 0x57, 0xad, 0xdd, 0x9a, 0xb8, 0x5e, 0x47, 0xbf, 0x03, 0x00, 0x00, 0xff, + 0xff, 0x89, 0x09, 0x4d, 0x10, 0xb5, 0x05, 0x00, 0x00, } func (m *State) Marshal() (dAtA []byte, err error) { @@ -335,6 +345,13 @@ func (m *State) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.LastSubmittedHeight != 0 { + i = encodeVarintState(dAtA, i, uint64(m.LastSubmittedHeight)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa0 + } { size, err := m.ConsensusParams.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -569,6 +586,9 @@ func (m *State) Size() (n int) { n += 2 + l + sovState(uint64(l)) l = m.ConsensusParams.Size() n += 2 + l + sovState(uint64(l)) + if m.LastSubmittedHeight != 0 { + n += 2 + sovState(uint64(m.LastSubmittedHeight)) + } return n } @@ -1048,6 +1068,25 @@ func (m *State) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 20: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastSubmittedHeight", wireType) + } + m.LastSubmittedHeight = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowState + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastSubmittedHeight |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipState(dAtA[iNdEx:]) diff --git a/types/serialization.go b/types/serialization.go index 99b81a8cd..0b920f0b0 100644 --- a/types/serialization.go +++ b/types/serialization.go @@ -266,6 +266,7 @@ func (s *State) ToProto() (*pb.State, error) { LastHeightConsensusParamsChanged: s.LastHeightConsensusParamsChanged, LastResultsHash: s.LastResultsHash[:], AppHash: s.AppHash[:], + LastSubmittedHeight: s.LastSubmittedHeight, }, nil } @@ -285,6 +286,7 @@ func (s *State) FromProto(other *pb.State) error { s.ConsensusParams = other.ConsensusParams s.LastHeightConsensusParamsChanged = other.LastHeightConsensusParamsChanged + s.LastSubmittedHeight = other.LastSubmittedHeight copy(s.LastResultsHash[:], other.LastResultsHash) copy(s.AppHash[:], other.AppHash) return nil From 9412b0f7008d61dfef1b594b6808081818153742 Mon Sep 17 00:00:00 2001 From: Sergi Rene Date: Wed, 4 Sep 2024 12:14:52 +0200 Subject: [PATCH 3/7] indexer pruning command --- indexers/blockindexer/block.go | 3 +++ indexers/blockindexer/kv/kv.go | 21 +++++++++++++++++++++ indexers/blockindexer/null/null.go | 4 ++++ 3 files changed, 28 insertions(+) diff --git a/indexers/blockindexer/block.go b/indexers/blockindexer/block.go index a3a2abc5b..13fa06290 100644 --- a/indexers/blockindexer/block.go +++ b/indexers/blockindexer/block.go @@ -19,4 +19,7 @@ type BlockIndexer interface { // Search performs a query for block heights that match a given BeginBlock // and Endblock event search criteria. Search(ctx context.Context, q *query.Query) ([]int64, error) + + // Prune function used by indexer pruning command + Prune(to int64) (uint64, error) } diff --git a/indexers/blockindexer/kv/kv.go b/indexers/blockindexer/kv/kv.go index cc10fbe2d..71264bb21 100644 --- a/indexers/blockindexer/kv/kv.go +++ b/indexers/blockindexer/kv/kv.go @@ -516,3 +516,24 @@ func (idx *BlockerIndexer) indexEvents(batch store.KVBatch, events []abci.Event, return nil } + +func (idx *BlockerIndexer) Prune(to int64) (uint64, error) { + + blocksPruned := uint64(0) + for h := int64(1); h < to; h++ { + + key, err := heightKey(h) + if err != nil { + return blocksPruned, fmt.Errorf("create block height index key: %w", err) + } + _, err = idx.store.Get(key) + if err != nil { + continue + } + if err := idx.store.Delete(key); err != nil { + continue + } + } + + return blocksPruned, nil +} diff --git a/indexers/blockindexer/null/null.go b/indexers/blockindexer/null/null.go index 62658e00e..fa5aec197 100644 --- a/indexers/blockindexer/null/null.go +++ b/indexers/blockindexer/null/null.go @@ -25,3 +25,7 @@ func (idx *BlockerIndexer) Index(types.EventDataNewBlockHeader) error { func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, error) { return []int64{}, nil } + +func (idx *BlockerIndexer) Prune(to int64) (uint64, error) { + return 0, nil +} From 9840d4308c279b19dec6b390c0ef9b9525f4d4e0 Mon Sep 17 00:00:00 2001 From: Sergi Rene Date: Wed, 4 Sep 2024 17:53:48 +0200 Subject: [PATCH 4/7] txs prune added --- indexers/blockindexer/kv/kv.go | 29 +++++++++++++++---- indexers/blockindexer/kv/util.go | 18 ++++++++++++ indexers/txindex/indexer.go | 3 ++ indexers/txindex/kv/kv.go | 49 ++++++++++++++++++++++++++++++++ indexers/txindex/null/null.go | 4 +++ 5 files changed, 97 insertions(+), 6 deletions(-) diff --git a/indexers/blockindexer/kv/kv.go b/indexers/blockindexer/kv/kv.go index 71264bb21..1d627bb44 100644 --- a/indexers/blockindexer/kv/kv.go +++ b/indexers/blockindexer/kv/kv.go @@ -105,7 +105,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, // If there is an exact height query, return the result immediately // (if it exists). - height, ok := lookForHeight(conditions) + /*height, ok := lookForHeight(conditions) if ok { ok, err := idx.Has(height) if err != nil { @@ -117,7 +117,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, } return results, nil - } + }*/ var heightsInitialized bool filteredHeights := make(map[string][]byte) @@ -519,12 +519,11 @@ func (idx *BlockerIndexer) indexEvents(batch store.KVBatch, events []abci.Event, func (idx *BlockerIndexer) Prune(to int64) (uint64, error) { - blocksPruned := uint64(0) + eventsPruned := uint64(0) for h := int64(1); h < to; h++ { - key, err := heightKey(h) if err != nil { - return blocksPruned, fmt.Errorf("create block height index key: %w", err) + return eventsPruned, fmt.Errorf("create block height index key: %w", err) } _, err = idx.store.Get(key) if err != nil { @@ -533,7 +532,25 @@ func (idx *BlockerIndexer) Prune(to int64) (uint64, error) { if err := idx.store.Delete(key); err != nil { continue } + + } + + it := idx.store.PrefixIterator([]byte{}) + defer it.Discard() + + for ; it.Valid(); it.Next() { + key := it.Key()[13:] + height, err := parseHeightFromEventKey(key) + if err != nil || height >= to { + continue + } + err = idx.store.Delete(key) + if err != nil { + continue + } + + eventsPruned++ } - return blocksPruned, nil + return eventsPruned, nil } diff --git a/indexers/blockindexer/kv/util.go b/indexers/blockindexer/kv/util.go index bde07a541..291d34736 100644 --- a/indexers/blockindexer/kv/util.go +++ b/indexers/blockindexer/kv/util.go @@ -85,6 +85,24 @@ func parseValueFromEventKey(key []byte) (string, error) { return eventValue, nil } +func parseHeightFromEventKey(key []byte) (int64, error) { + var ( + compositeKey, typ, eventValue string + height int64 + ) + + remaining, err := orderedcode.Parse(string(key), &compositeKey, &eventValue, &height, &typ) + if err != nil { + return -1, fmt.Errorf("parse event key: %w", err) + } + + if len(remaining) != 0 { + return -1, fmt.Errorf("unexpected remainder in key: %s", remaining) + } + + return height, nil +} + func lookForHeight(conditions []query.Condition) (int64, bool) { for _, c := range conditions { if c.CompositeKey == types.BlockHeightKey && c.Op == query.OpEqual { diff --git a/indexers/txindex/indexer.go b/indexers/txindex/indexer.go index 6d9f6807d..b545d530e 100644 --- a/indexers/txindex/indexer.go +++ b/indexers/txindex/indexer.go @@ -22,6 +22,9 @@ type TxIndexer interface { // Search allows you to query for transactions. Search(ctx context.Context, q *query.Query) ([]*abci.TxResult, error) + + // Prune function used by indexer pruning command + Prune(to int64) (uint64, error) } // Batch groups together multiple Index operations to be performed at the same time. diff --git a/indexers/txindex/kv/kv.go b/indexers/txindex/kv/kv.go index 6c3435e4e..cb6edaab9 100644 --- a/indexers/txindex/kv/kv.go +++ b/indexers/txindex/kv/kv.go @@ -553,6 +553,46 @@ LOOP: return filteredHashes } +func (txi *TxIndex) Prune(to int64) (uint64, error) { + + eventsPruned := uint64(0) + + for h := int64(1); h < to; h++ { + prefix := []byte(fmt.Sprintf("%s/%d/", tmtypes.TxHeightKey, h)) + + it := txi.store.PrefixIterator(prefix) + defer it.Discard() + for ; it.Valid(); it.Next() { + err := txi.store.Delete(it.Key()) + if err != nil { + continue + } + err = txi.store.Delete(it.Value()) + if err != nil { + continue + } + } + } + it := txi.store.PrefixIterator([]byte{}) + defer it.Discard() + + for ; it.Valid(); it.Next() { + key := it.Key()[13:] + height, err := extractHeightFromKey(key) + if err != nil || height >= to { + continue + } + err = txi.store.Delete(key) + if err != nil { + continue + } + + eventsPruned++ + } + + return eventsPruned, nil +} + // Keys func isTagKey(key []byte) bool { @@ -564,6 +604,15 @@ func extractValueFromKey(key []byte) string { return parts[1] } +func extractHeightFromKey(key []byte) (int64, error) { + parts := strings.SplitN(string(key), tagKeySeparator, 3) + height, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + return 0, err + } + return height, nil +} + func keyForEvent(key string, value []byte, result *abci.TxResult) []byte { return []byte(fmt.Sprintf("%s/%s/%d/%d", key, diff --git a/indexers/txindex/null/null.go b/indexers/txindex/null/null.go index 4cf66cbbf..d625bbd11 100644 --- a/indexers/txindex/null/null.go +++ b/indexers/txindex/null/null.go @@ -32,3 +32,7 @@ func (txi *TxIndex) Index(result *abci.TxResult) error { func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*abci.TxResult, error) { return []*abci.TxResult{}, nil } + +func (txi *TxIndex) Prune(to int64) (uint64, error) { + return 0, nil +} From f3fc6891e03d9115927e3c7b53d24cfccc60d923 Mon Sep 17 00:00:00 2001 From: Sergi Rene Date: Wed, 4 Sep 2024 18:00:51 +0200 Subject: [PATCH 5/7] uncomment code --- indexers/blockindexer/kv/kv.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexers/blockindexer/kv/kv.go b/indexers/blockindexer/kv/kv.go index 1d627bb44..7117f7afb 100644 --- a/indexers/blockindexer/kv/kv.go +++ b/indexers/blockindexer/kv/kv.go @@ -105,7 +105,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, // If there is an exact height query, return the result immediately // (if it exists). - /*height, ok := lookForHeight(conditions) + height, ok := lookForHeight(conditions) if ok { ok, err := idx.Has(height) if err != nil { @@ -117,7 +117,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, } return results, nil - }*/ + } var heightsInitialized bool filteredHeights := make(map[string][]byte) From 663f8d742bafca79d462cf2241caae4cea4b1d23 Mon Sep 17 00:00:00 2001 From: Sergi Rene Date: Wed, 4 Sep 2024 18:03:38 +0200 Subject: [PATCH 6/7] lint fix --- indexers/blockindexer/kv/kv.go | 1 - indexers/txindex/kv/kv.go | 1 - 2 files changed, 2 deletions(-) diff --git a/indexers/blockindexer/kv/kv.go b/indexers/blockindexer/kv/kv.go index 7117f7afb..1bb0f5e2d 100644 --- a/indexers/blockindexer/kv/kv.go +++ b/indexers/blockindexer/kv/kv.go @@ -518,7 +518,6 @@ func (idx *BlockerIndexer) indexEvents(batch store.KVBatch, events []abci.Event, } func (idx *BlockerIndexer) Prune(to int64) (uint64, error) { - eventsPruned := uint64(0) for h := int64(1); h < to; h++ { key, err := heightKey(h) diff --git a/indexers/txindex/kv/kv.go b/indexers/txindex/kv/kv.go index cb6edaab9..ccbeeba69 100644 --- a/indexers/txindex/kv/kv.go +++ b/indexers/txindex/kv/kv.go @@ -554,7 +554,6 @@ LOOP: } func (txi *TxIndex) Prune(to int64) (uint64, error) { - eventsPruned := uint64(0) for h := int64(1); h < to; h++ { From 719b98edfb8e1ff6bb0af32224d2eec04d703300 Mon Sep 17 00:00:00 2001 From: Sergi Rene Date: Thu, 5 Sep 2024 10:12:05 +0200 Subject: [PATCH 7/7] improve block indexer pruning --- indexers/blockindexer/kv/kv.go | 35 ++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/indexers/blockindexer/kv/kv.go b/indexers/blockindexer/kv/kv.go index 1bb0f5e2d..7328ec956 100644 --- a/indexers/blockindexer/kv/kv.go +++ b/indexers/blockindexer/kv/kv.go @@ -519,6 +519,17 @@ func (idx *BlockerIndexer) indexEvents(batch store.KVBatch, events []abci.Event, func (idx *BlockerIndexer) Prune(to int64) (uint64, error) { eventsPruned := uint64(0) + + batch := idx.store.NewBatch() + + flush := func(batch store.KVBatch, height int64) error { + err := batch.Commit() + if err != nil { + return fmt.Errorf("flush batch to disk: height %d: %w", height, err) + } + return nil + } + for h := int64(1); h < to; h++ { key, err := heightKey(h) if err != nil { @@ -531,7 +542,15 @@ func (idx *BlockerIndexer) Prune(to int64) (uint64, error) { if err := idx.store.Delete(key); err != nil { continue } - + eventsPruned++ + if eventsPruned%1000 == 0 && eventsPruned > 0 { + err := flush(batch, h) + if err != nil { + return 0, err + } + batch.Discard() + batch = idx.store.NewBatch() + } } it := idx.store.PrefixIterator([]byte{}) @@ -543,12 +562,24 @@ func (idx *BlockerIndexer) Prune(to int64) (uint64, error) { if err != nil || height >= to { continue } - err = idx.store.Delete(key) + err = batch.Delete(key) if err != nil { continue } eventsPruned++ + if eventsPruned%1000 == 0 && eventsPruned > 0 { + err := flush(batch, to) + if err != nil { + return 0, err + } + batch.Discard() + batch = idx.store.NewBatch() + } + } + err := flush(batch, to) + if err != nil { + return 0, err } return eventsPruned, nil