From ce9725c20e3eabb3ae0fd59a6794f78f2661135e Mon Sep 17 00:00:00 2001 From: "iteyelmp@gmail.com" Date: Fri, 15 Dec 2023 14:51:59 +0800 Subject: [PATCH 1/4] copy code --- ethstorage/storage_manager_test.go | 205 +++++++++++++++++++++++++---- 1 file changed, 182 insertions(+), 23 deletions(-) diff --git a/ethstorage/storage_manager_test.go b/ethstorage/storage_manager_test.go index 372228ff..0dee0dfc 100644 --- a/ethstorage/storage_manager_test.go +++ b/ethstorage/storage_manager_test.go @@ -1,58 +1,217 @@ // Copyright 2022-2023, EthStorage. // For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE -//go:build !ci - package ethstorage import ( + "encoding/binary" + "errors" + "fmt" + "os" "testing" + "github.com/detailyang/go-fallocate" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + prv "github.com/ethstorage/go-ethstorage/ethstorage/prover" +) - "github.com/ethstorage/go-ethstorage/ethstorage/eth" +const ( + metafileName = "metafile.dat.meta" + defaultEncodeType = ENCODE_BLOB_POSEIDON + blobEmptyFillingMask = byte(0b10000000) + kvEntries = uint64(16) + lastKvIndex = uint64(16) ) var ( - contractAddress = common.HexToAddress("0x31c2078945359086152687E43F30d94A52141bEc") - rawURL = "http://65.108.236.27:8545" - filename = "../cmd/es-node/storage.dat" - pc *eth.PollingClient - sm *ShardManager + contractAddress = common.HexToAddress("0x0000000000000000000000000000000003330001") + testLog = log.New("TestStorageManager") + prover = prv.NewKZGProver(testLog) storageManager *StorageManager ) -func setup(t *testing.T) { - var err error - pc, err = eth.Dial(rawURL, contractAddress, log.New()) +type mockL1Source struct { + lastBlobIndex uint64 + metaFile *os.File +} + +func (l1 *mockL1Source) getMetadata(idx uint64) ([32]byte, error) { + bs := make([]byte, 32) + l, err := l1.metaFile.ReadAt(bs, int64(idx*32)) if err != nil { - t.Fatal("create polling client failed") - return + return common.Hash{}, fmt.Errorf("get metadata fail, err %s", err.Error()) } + if l != 32 { + return common.Hash{}, errors.New("get metadata fail, err read less than 32 bytes") + } + return common.BytesToHash(bs), nil +} + +func (l1 *mockL1Source) GetKvMetas(kvIndices []uint64, blockNumber int64) ([][32]byte, error) { + metas := make([][32]byte, 0) + for _, idx := range kvIndices { + meta, err := l1.getMetadata(idx) + if err != nil { + log.Debug("read meta fail", "err", err.Error()) + continue + } + metas = append(metas, meta) + } + return metas, nil +} - sm = NewShardManager(contractAddress, 131072, 16, 131072) +func (l1 *mockL1Source) GetStorageLastBlobIdx(blockNumber int64) (uint64, error) { + return l1.lastBlobIndex, nil +} - var df *DataFile - df, err = OpenDataFile(filename) +func createMetaFile(filename string, len int64) (*os.File, error) { + file, err := os.Create(filename) if err != nil { - t.Fatal("open data file failed") - return + return nil, err + } + err = fallocate.Fallocate(file, int64((32)*len), int64(32)) + if err != nil { + return nil, err + } + return file, nil +} + +func newMockL1Source(lastBlobIndex uint64, metafile string) Il1Source { + if len(metafile) == 0 { + panic("metafile param is needed when using mock l1") } - err = sm.AddDataFileAndShard(df) + file, err := os.OpenFile(metafile, os.O_RDONLY, 0600) if err != nil { - t.Fatal("add data file failed") - return + panic(fmt.Sprintf("open metafile faiil with err %s", err.Error())) + } + return &mockL1Source{lastBlobIndex: lastBlobIndex, metaFile: file} +} + +func createEthStorage(contract common.Address, shardIdxList []uint64, chunkSize, kvSize, kvEntries uint64, + miner common.Address, encodeType uint64) (*ShardManager, []string) { + sm := NewShardManager(contract, kvSize, kvEntries, chunkSize) + ContractToShardManager[contract] = sm + chunkPerKv := kvSize / chunkSize + + files := make([]string, 0) + for _, shardIdx := range shardIdxList { + sm.AddDataShard(shardIdx) + fileName := fmt.Sprintf(".\\ss%d.dat", shardIdx) + files = append(files, fileName) + startChunkId := shardIdx * chunkPerKv * kvEntries + _, err := Create(fileName, startChunkId, kvEntries*chunkPerKv, 0, kvSize, encodeType, miner, sm.ChunkSize()) + if err != nil { + log.Crit("open failed", "error", err) + } + + var df *DataFile + df, err = OpenDataFile(fileName) + if err != nil { + log.Crit("open failed", "error", err) + } + sm.AddDataFile(df) + } + + return sm, files +} + +type BlobPayloadWithRowData struct { + MinerAddress common.Address `json:"minerAddress"` + BlobIndex uint64 `json:"blobIndex"` + BlobCommit common.Hash `json:"blobCommit"` + EncodeType uint64 `json:"encodeType"` + EncodedBlob []byte `json:"blob"` + RowData []byte +} + +// makeKVStorage generate a range of storage Data and its metadata +func makeKVStorage(contract common.Address, shards []uint64, chunkSize, kvSize, kvCount, lastKvIndex uint64, + miner common.Address, encodeType uint64, metafile *os.File) map[common.Address]map[uint64]*BlobPayloadWithRowData { + + shardData := make(map[common.Address]map[uint64]*BlobPayloadWithRowData) + smData := make(map[uint64]*BlobPayloadWithRowData) + shardData[contract] = smData + sm := ContractToShardManager[contract] + + for _, sidx := range shards { + for i := sidx * kvCount; i < (sidx+1)*kvCount; i++ { + val := make([]byte, kvSize) + root := common.Hash{} + if i < lastKvIndex { + copy(val[:20], contract.Bytes()) + binary.BigEndian.PutUint64(val[20:28], i) + root, _ = prover.GetRoot(val, kvSize/chunkSize, chunkSize) + } + + commit := generateMetadata(root) + encodeData, _, _ := sm.EncodeKV(i, val, commit, miner, encodeType) + smData[i] = &BlobPayloadWithRowData{ + MinerAddress: miner, + BlobIndex: i, + BlobCommit: commit, + EncodeType: encodeType, + EncodedBlob: encodeData, + RowData: val, + } + meta := GenerateMetadata(i, kvSize, root[:]) + metafile.WriteAt(meta.Bytes(), int64(i*32)) + } } - storageManager = NewStorageManager(sm, pc) + return shardData +} + +func generateMetadata(hash common.Hash) common.Hash { + meta := make([]byte, 32) + copy(meta[0:HashSizeInContract], hash[0:HashSizeInContract]) + meta[HashSizeInContract] = meta[HashSizeInContract] | blobEmptyFillingMask + return common.BytesToHash(meta) +} + +func GenerateMetadata(idx, size uint64, hash []byte) common.Hash { + meta := make([]byte, 0) + idx_bs := make([]byte, 8) + binary.BigEndian.PutUint64(idx_bs, idx) + meta = append(meta, idx_bs[3:]...) + size_bs := make([]byte, 8) + binary.BigEndian.PutUint64(size_bs, size) + meta = append(meta, size_bs[5:]...) + meta = append(meta, hash[:24]...) + return common.BytesToHash(meta) +} + +func setup(t *testing.T) { + // create l1 + metafile, err := createMetaFile(metafileName, int64(kvEntries)) + if err != nil { + t.Error("Create metafileName fail", err.Error()) + } + defer metafile.Close() + l1 := newMockL1Source(lastKvIndex, metafileName) + + // create shard manage + sm, files := createEthStorage(contractAddress, []uint64{0}, + 131072, 131072, kvEntries, common.Address{}, defaultEncodeType) + if sm == nil { + t.Fatalf("createEthStorage failed") + } + defer func(files []string) { + for _, file := range files { + os.Remove(file) + } + }(files) + //data := makeKVStorage(contractAddress, []uint64{0}, 131072, 131072, + // kvEntries, lastKvIndex, common.Address{}, defaultEncodeType, metafile) + //t.Log("sm files", data) + + storageManager = NewStorageManager(sm, l1) err = storageManager.DownloadFinished(97528, []uint64{}, [][]byte{}, []common.Hash{}) if err != nil { t.Fatal("set local L1 failed", err) return } - } func TestStorageManager_LastKvIndex(t *testing.T) { From 7662eae443d32c18dc0e63d74efb636aef6f8e32 Mon Sep 17 00:00:00 2001 From: "iteyelmp@gmail.com" Date: Fri, 15 Dec 2023 18:32:36 +0800 Subject: [PATCH 2/4] fix commit error --- ethstorage/storage_manager_test.go | 112 +++++++---------------------- 1 file changed, 27 insertions(+), 85 deletions(-) diff --git a/ethstorage/storage_manager_test.go b/ethstorage/storage_manager_test.go index 0dee0dfc..b41f6849 100644 --- a/ethstorage/storage_manager_test.go +++ b/ethstorage/storage_manager_test.go @@ -17,11 +17,10 @@ import ( ) const ( - metafileName = "metafile.dat.meta" - defaultEncodeType = ENCODE_BLOB_POSEIDON - blobEmptyFillingMask = byte(0b10000000) - kvEntries = uint64(16) - lastKvIndex = uint64(16) + metafileName = "metafile.dat.meta" + defaultEncodeType = ENCODE_BLOB_POSEIDON + kvEntries = uint64(16) + lastKvIndex = uint64(16) ) var ( @@ -117,69 +116,13 @@ func createEthStorage(contract common.Address, shardIdxList []uint64, chunkSize, return sm, files } -type BlobPayloadWithRowData struct { - MinerAddress common.Address `json:"minerAddress"` - BlobIndex uint64 `json:"blobIndex"` - BlobCommit common.Hash `json:"blobCommit"` - EncodeType uint64 `json:"encodeType"` - EncodedBlob []byte `json:"blob"` - RowData []byte -} - -// makeKVStorage generate a range of storage Data and its metadata -func makeKVStorage(contract common.Address, shards []uint64, chunkSize, kvSize, kvCount, lastKvIndex uint64, - miner common.Address, encodeType uint64, metafile *os.File) map[common.Address]map[uint64]*BlobPayloadWithRowData { - - shardData := make(map[common.Address]map[uint64]*BlobPayloadWithRowData) - smData := make(map[uint64]*BlobPayloadWithRowData) - shardData[contract] = smData - sm := ContractToShardManager[contract] - - for _, sidx := range shards { - for i := sidx * kvCount; i < (sidx+1)*kvCount; i++ { - val := make([]byte, kvSize) - root := common.Hash{} - if i < lastKvIndex { - copy(val[:20], contract.Bytes()) - binary.BigEndian.PutUint64(val[20:28], i) - root, _ = prover.GetRoot(val, kvSize/chunkSize, chunkSize) - } - - commit := generateMetadata(root) - encodeData, _, _ := sm.EncodeKV(i, val, commit, miner, encodeType) - smData[i] = &BlobPayloadWithRowData{ - MinerAddress: miner, - BlobIndex: i, - BlobCommit: commit, - EncodeType: encodeType, - EncodedBlob: encodeData, - RowData: val, - } - meta := GenerateMetadata(i, kvSize, root[:]) - metafile.WriteAt(meta.Bytes(), int64(i*32)) - } - } - - return shardData -} - -func generateMetadata(hash common.Hash) common.Hash { - meta := make([]byte, 32) - copy(meta[0:HashSizeInContract], hash[0:HashSizeInContract]) - meta[HashSizeInContract] = meta[HashSizeInContract] | blobEmptyFillingMask - return common.BytesToHash(meta) -} - -func GenerateMetadata(idx, size uint64, hash []byte) common.Hash { - meta := make([]byte, 0) - idx_bs := make([]byte, 8) - binary.BigEndian.PutUint64(idx_bs, idx) - meta = append(meta, idx_bs[3:]...) - size_bs := make([]byte, 8) - binary.BigEndian.PutUint64(size_bs, size) - meta = append(meta, size_bs[5:]...) - meta = append(meta, hash[:24]...) - return common.BytesToHash(meta) +func createBlob(kvIndex uint64) (blob []byte, hash common.Hash) { + val := make([]byte, 131072) + copy(val[:20], contractAddress.Bytes()) + binary.BigEndian.PutUint64(val[20:28], kvIndex) + root := common.Hash{} + root, _ = prover.GetRoot(val, 1, 131072) + return val, root } func setup(t *testing.T) { @@ -188,7 +131,10 @@ func setup(t *testing.T) { if err != nil { t.Error("Create metafileName fail", err.Error()) } - defer metafile.Close() + defer func(file *os.File) { + file.Close() + os.Remove(file.Name()) + }(metafile) l1 := newMockL1Source(lastKvIndex, metafileName) // create shard manage @@ -202,16 +148,9 @@ func setup(t *testing.T) { os.Remove(file) } }(files) - //data := makeKVStorage(contractAddress, []uint64{0}, 131072, 131072, - // kvEntries, lastKvIndex, common.Address{}, defaultEncodeType, metafile) - //t.Log("sm files", data) storageManager = NewStorageManager(sm, l1) - err = storageManager.DownloadFinished(97528, []uint64{}, [][]byte{}, []common.Hash{}) - if err != nil { - t.Fatal("set local L1 failed", err) - return - } + storageManager.DownloadThreadNum = 1 } func TestStorageManager_LastKvIndex(t *testing.T) { @@ -222,7 +161,7 @@ func TestStorageManager_LastKvIndex(t *testing.T) { func TestStorageManager_DownloadFinished(t *testing.T) { setup(t) - h := common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + h := common.Hash{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} err := storageManager.DownloadFinished(97529, []uint64{2}, [][]byte{{10}}, []common.Hash{h}) if err != nil { @@ -236,31 +175,34 @@ func TestStorageManager_DownloadFinished(t *testing.T) { meta := common.Hash{} copy(meta[:], bs) - if meta != h { + if meta != prepareCommit(h) { t.Fatal("failed to write meta", err) } } func TestStorageManager_CommitBlobs(t *testing.T) { setup(t) - h := common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2} - failedCommited, err := storageManager.CommitBlobs([]uint64{2}, [][]byte{{10}}, []common.Hash{h}) + + kvIndex := uint64(2) + blob, h := createBlob(kvIndex) + storageManager.updateLocalMetas([]uint64{kvIndex}, []common.Hash{h}) + successCommitted, err := storageManager.CommitBlobs([]uint64{kvIndex}, [][]byte{blob}, []common.Hash{h}) if err != nil { t.Fatal("failed to commit blob", err) } - if len(failedCommited) != 0 { + if len(successCommitted) < 1 { t.Fatal("should commit all the blobs") } - bs, success, err := storageManager.TryReadMeta(2) + bs, success, err := storageManager.TryReadMeta(kvIndex) if err != nil || !success { t.Fatal("failed to read meta", err) } meta := common.Hash{} copy(meta[:], bs) - if meta != h { - t.Fatal("failed to write meta", err) + if meta != prepareCommit(h) { + t.Fatal("failed to write meta", meta, h) } } From 65f50924162fe8300327106a2286c71ae5b7a4db Mon Sep 17 00:00:00 2001 From: "iteyelmp@gmail.com" Date: Fri, 15 Dec 2023 18:59:03 +0800 Subject: [PATCH 3/4] init data --- ethstorage/storage_manager_test.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ethstorage/storage_manager_test.go b/ethstorage/storage_manager_test.go index b41f6849..bc5ece9b 100644 --- a/ethstorage/storage_manager_test.go +++ b/ethstorage/storage_manager_test.go @@ -151,6 +151,19 @@ func setup(t *testing.T) { storageManager = NewStorageManager(sm, l1) storageManager.DownloadThreadNum = 1 + + kvIndexes := []uint64{1, 2, 3} + blobs := make([][]byte, len(kvIndexes)) + hashes := make([]common.Hash, len(kvIndexes)) + for i, idx := range kvIndexes { + blob, hash := createBlob(idx) + blobs[i] = blob + hashes[i] = hash + } + err = storageManager.DownloadFinished(97528, kvIndexes, blobs, hashes) + if err != nil { + t.Fatal("init error") + } } func TestStorageManager_LastKvIndex(t *testing.T) { @@ -184,9 +197,8 @@ func TestStorageManager_CommitBlobs(t *testing.T) { setup(t) kvIndex := uint64(2) - blob, h := createBlob(kvIndex) - storageManager.updateLocalMetas([]uint64{kvIndex}, []common.Hash{h}) - successCommitted, err := storageManager.CommitBlobs([]uint64{kvIndex}, [][]byte{blob}, []common.Hash{h}) + b, h := createBlob(kvIndex) + successCommitted, err := storageManager.CommitBlobs([]uint64{kvIndex}, [][]byte{b}, []common.Hash{h}) if err != nil { t.Fatal("failed to commit blob", err) } @@ -203,6 +215,6 @@ func TestStorageManager_CommitBlobs(t *testing.T) { meta := common.Hash{} copy(meta[:], bs) if meta != prepareCommit(h) { - t.Fatal("failed to write meta", meta, h) + t.Fatal("failed to write meta", err) } } From b75fc3e451ccf2d0843d97eb212827d9de2ac328 Mon Sep 17 00:00:00 2001 From: "iteyelmp@gmail.com" Date: Thu, 21 Dec 2023 17:02:41 +0800 Subject: [PATCH 4/4] add DownloadAllMeta --- ethstorage/storage_manager_test.go | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ethstorage/storage_manager_test.go b/ethstorage/storage_manager_test.go index bc5ece9b..b1e4d293 100644 --- a/ethstorage/storage_manager_test.go +++ b/ethstorage/storage_manager_test.go @@ -125,6 +125,18 @@ func createBlob(kvIndex uint64) (blob []byte, hash common.Hash) { return val, root } +func generateMetadata(idx, size uint64, hash []byte) common.Hash { + meta := make([]byte, 0) + idx_bs := make([]byte, 8) + binary.BigEndian.PutUint64(idx_bs, idx) + meta = append(meta, idx_bs[3:]...) + size_bs := make([]byte, 8) + binary.BigEndian.PutUint64(size_bs, size) + meta = append(meta, size_bs[5:]...) + meta = append(meta, hash[:24]...) + return common.BytesToHash(meta) +} + func setup(t *testing.T) { // create l1 metafile, err := createMetaFile(metafileName, int64(kvEntries)) @@ -159,6 +171,8 @@ func setup(t *testing.T) { blob, hash := createBlob(idx) blobs[i] = blob hashes[i] = hash + meta := generateMetadata(uint64(i), kvIndexes[i], hash[:]) + metafile.WriteAt(meta.Bytes(), int64(i*32)) } err = storageManager.DownloadFinished(97528, kvIndexes, blobs, hashes) if err != nil { @@ -218,3 +232,24 @@ func TestStorageManager_CommitBlobs(t *testing.T) { t.Fatal("failed to write meta", err) } } + +func TestStorageManager_DownloadAllMeta(t *testing.T) { + setup(t) + err := storageManager.DownloadAllMetas(4) + if err != nil { + t.Fatal("failed to Downloand Finished", err) + } + + kvIndex := uint64(3) + _, h := createBlob(kvIndex) + bs, success, err := storageManager.TryReadMeta(kvIndex) + if err != nil || !success { + t.Fatal("failed to read meta", err) + } + + meta := common.Hash{} + copy(meta[:], bs) + if meta != prepareCommit(h) { + t.Fatal("failed to compare meta", err) + } +}