diff --git a/be/src/script/script.cpp b/be/src/script/script.cpp index 51011b3c3e5388..3a9f7baf9f10fe 100644 --- a/be/src/script/script.cpp +++ b/be/src/script/script.cpp @@ -31,6 +31,9 @@ #include "runtime/exec_env.h" #include "runtime/mem_tracker.h" #include "storage/del_vector.h" +#include "storage/lake/tablet.h" +#include "storage/lake/tablet_manager.h" +#include "storage/lake/tablet_metadata.h" #include "storage/primary_key_dump.h" #include "storage/storage_engine.h" #include "storage/tablet.h" @@ -38,6 +41,7 @@ #include "storage/tablet_meta_manager.h" #include "storage/tablet_updates.h" #include "util/stack_util.h" +#include "util/url_coding.h" #include "wrenbind17/wrenbind17.hpp" using namespace wrenbind17; @@ -289,6 +293,25 @@ class StorageEngineRef { return ptr; } + static std::string get_lake_tablet_metadata_json(int64_t tablet_id, int64_t version) { + auto tablet_manager = ExecEnv::GetInstance()->lake_tablet_manager(); + RETURN_IF(nullptr == tablet_manager, ""); + auto meta_st = tablet_manager->get_tablet_metadata(tablet_id, version, false); + RETURN_IF(!meta_st.ok(), meta_st.status().to_string()); + std::string json; + return proto_to_json(*meta_st.value()); + } + + static std::string decode_encryption_meta(const std::string& meta_base64) { + EncryptionMetaPB pb; + std::string meta_bytes; + RETURN_IF(!base64_decode(meta_base64, &meta_bytes), "bad base64 string"); + if (!pb.ParseFromString(meta_bytes)) { + return "parse encryption meta failed"; + } + return proto_to_json(pb); + } + static std::shared_ptr get_tablet_info(int64_t tablet_id) { std::vector tablet_infos; auto manager = StorageEngine::instance()->tablet_manager(); @@ -563,6 +586,8 @@ class StorageEngineRef { REG_STATIC_METHOD(StorageEngineRef, get_tablet_info); REG_STATIC_METHOD(StorageEngineRef, get_tablet_infos); REG_STATIC_METHOD(StorageEngineRef, get_tablet_meta_json); + REG_STATIC_METHOD(StorageEngineRef, get_lake_tablet_metadata_json); + REG_STATIC_METHOD(StorageEngineRef, decode_encryption_meta); REG_STATIC_METHOD(StorageEngineRef, reset_delvec); REG_STATIC_METHOD(StorageEngineRef, get_tablet); REG_STATIC_METHOD(StorageEngineRef, drop_tablet); diff --git a/be/src/storage/lake/rowset.cpp b/be/src/storage/lake/rowset.cpp index c451d0037b0d8a..32d66010290ab7 100644 --- a/be/src/storage/lake/rowset.cpp +++ b/be/src/storage/lake/rowset.cpp @@ -420,13 +420,14 @@ Status Rowset::load_segments(std::vector* segments, SegmentReadOptio int index = 0; std::vector, std::string>>> segment_futures; - auto check_status = [&](StatusOr& segment_or, const std::string& seg_name) -> Status { + auto check_status = [&](StatusOr& segment_or, const std::string& seg_name, int seg_id) -> Status { if (segment_or.ok()) { segments->emplace_back(std::move(segment_or.value())); } else if (segment_or.status().is_not_found() && ignore_lost_segment) { LOG(WARNING) << "Ignored lost segment " << seg_name; } else { - return segment_or.status(); + return segment_or.status().clone_and_prepend(fmt::format( + "load_segments failed tablet:{} rowset:{} segid:{}", _tablet_id, metadata().id(), seg_id)); } return Status::OK(); }; @@ -470,25 +471,26 @@ Status Rowset::load_segments(std::vector* segments, SegmentReadOptio << ", try to load segment serially, seg_id: " << seg_id; auto segment_or = _tablet_mgr->load_segment(segment_info, seg_id, &footer_size_hint, lake_io_opts, lake_io_opts.fill_metadata_cache, _tablet_schema); - if (auto status = check_status(segment_or, seg_name); !status.ok()) { + if (auto status = check_status(segment_or, seg_name, seg_id); !status.ok()) { return status; } } seg_id++; segment_futures.push_back(task->get_future()); } else { - auto segment_or = _tablet_mgr->load_segment(segment_info, seg_id++, &footer_size_hint, lake_io_opts, + auto segment_or = _tablet_mgr->load_segment(segment_info, seg_id, &footer_size_hint, lake_io_opts, lake_io_opts.fill_metadata_cache, _tablet_schema); - if (auto status = check_status(segment_or, seg_name); !status.ok()) { + if (auto status = check_status(segment_or, seg_name, seg_id); !status.ok()) { return status; } + seg_id++; } } - for (auto& fut : segment_futures) { - auto result_pair = fut.get(); + for (int i = 0; i < segment_futures.size(); i++) { + auto result_pair = segment_futures[i].get(); auto segment_or = result_pair.first; - if (auto status = check_status(segment_or, result_pair.second); !status.ok()) { + if (auto status = check_status(segment_or, result_pair.second, i); !status.ok()) { return status; } } diff --git a/be/src/storage/rowset/rowset.cpp b/be/src/storage/rowset/rowset.cpp index 2dd8371d688822..23a9f37f0b03b3 100644 --- a/be/src/storage/rowset/rowset.cpp +++ b/be/src/storage/rowset/rowset.cpp @@ -179,9 +179,12 @@ Status Rowset::do_load() { auto res = Segment::open(fs, seg_info, seg_id, _schema, &footer_size_hint, rowset_meta()->partial_rowset_footer(seg_id)); if (!res.ok()) { - LOG(WARNING) << "Fail to open " << seg_path << ": " << res.status(); + auto st = res.status().clone_and_prepend(fmt::format( + "Load rowset failed tablet:{} rowset:{} rssid:{} seg:{} path:{}", _rowset_meta->tablet_id(), + rowset_id().to_string(), _rowset_meta->get_rowset_seg_id(), seg_id, seg_path)); + LOG(WARNING) << st.message(); _segments.clear(); - return res.status(); + return st; } _segments.push_back(std::move(res).value()); } @@ -216,9 +219,12 @@ Status Rowset::reload() { FileInfo seg_info{.path = seg_path}; auto res = Segment::open(fs, seg_info, seg_id, _schema, &footer_size_hint); if (!res.ok()) { - LOG(WARNING) << "Fail to open " << seg_path << ": " << res.status(); + auto st = res.status().clone_and_prepend(fmt::format( + "reload_segment failed tablet:{} rowset:{} rssid:{} seg:{} path:{}", _rowset_meta->tablet_id(), + rowset_id().to_string(), _rowset_meta->get_rowset_seg_id(), seg_id, seg_path)); + LOG(WARNING) << st.message(); _segments.clear(); - return res.status(); + return st; } _segments.push_back(std::move(res).value()); } @@ -237,8 +243,11 @@ Status Rowset::reload_segment(int32_t segment_id) { FileInfo seg_info{.path = seg_path}; auto res = Segment::open(fs, seg_info, segment_id, _schema, &footer_size_hint); if (!res.ok()) { - LOG(WARNING) << "Fail to open " << seg_path << ": " << res.status(); - return res.status(); + auto st = res.status().clone_and_prepend(fmt::format( + "reload_segment failed tablet:{} rowset:{} rssid:{} seg:{} path:{}", _rowset_meta->tablet_id(), + rowset_id().to_string(), _rowset_meta->get_rowset_seg_id(), segment_id, seg_path)); + LOG(WARNING) << st.message(); + return st; } _segments[segment_id] = std::move(res).value(); return Status::OK(); @@ -256,8 +265,11 @@ Status Rowset::reload_segment_with_schema(int32_t segment_id, TabletSchemaCSPtr& FileInfo seg_info{.path = seg_path}; auto res = Segment::open(fs, seg_info, segment_id, schema, &footer_size_hint); if (!res.ok()) { - LOG(WARNING) << "Fail to open " << seg_path << ": " << res.status(); - return res.status(); + auto st = res.status().clone_and_prepend(fmt::format( + "reload_segment failed tablet:{} rowset:{} rssid:{} seg:{} path:{}", _rowset_meta->tablet_id(), + rowset_id().to_string(), _rowset_meta->get_rowset_seg_id(), segment_id, seg_path)); + LOG(WARNING) << st.message(); + return st; } _segments[segment_id] = std::move(res).value(); return Status::OK(); diff --git a/be/test/fs/key_cache_test.cpp b/be/test/fs/key_cache_test.cpp index 016c31f05f5e30..721fcae0d9f802 100644 --- a/be/test/fs/key_cache_test.cpp +++ b/be/test/fs/key_cache_test.cpp @@ -91,6 +91,14 @@ TEST_F(KeyCacheTest, AddKey) { ASSERT_EQ(2, cache.size()); std::string result; ASSERT_TRUE(execute_script("System.print(ExecEnv.key_cache_info())", result).ok()); + ASSERT_TRUE(execute_script("System.print(StorageEngine.decode_encryption_meta(" + "\"Ch4IARiztZ64BiAAKAE6EHp8EnQlAIgiy8dgbPRP53kKPAgCEAEYs7WeuAYgACgBMiyZegO5j9P16bHelpUAU" + "xEj1c5P4xWQsJSy6sc2yIKC0g/rRPqsGNumdy6WQgo0EAIgACgBMixDCSo3rP5l8oiZLcgtts8x7xJ+M4+/" + "INZvGPhCOA1m9zf2vpCRbjbVoOl2EQ==\"))", + result) + .ok()); + ASSERT_TRUE(result.find("keyHierarchy") != result.npos); + ASSERT_TRUE(execute_script("System.print(\"StorageEngine.get_lake_tablet_metadata_json(1,1)\")", result).ok()); } static void wrap_unwrap_test(int num_level) {