Move compute_new_fragment_name into StorageFormat (#4520)

Move `compute_new_fragment_name` from `tiledb/sm` into the `StorageFormat`. A new file is added to maintain storage format of `fragment_name`, and contains `compute_new_fragment_name` and `generate_fragment_name`. Tests are added for these APIs, as well as `get_timestamp_range`. --- TYPE: FORMAT DESC: Migrate fragment_name parsing into StorageFormat
TileDB-Inc · Nov 17, 2023 · 2c04a10 · 2c04a10
1 parent f774124
commit 2c04a10
Show file tree

Hide file tree

Showing 17 changed files with 340 additions and 122 deletions.
diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc
@@ -41,14 +41,13 @@
 #include "tiledb/sm/storage_manager/context_resources.h"
 #include "tiledb/sm/tile/generic_tile_io.h"
 #include "tiledb/sm/tile/tile.h"
-#include "tiledb/storage_format/uri/parse_uri.h"
+#include "tiledb/storage_format/uri/generate_uri.h"
 
 #include <numeric>
 
 using namespace tiledb::common;
 
-namespace tiledb {
-namespace sm {
+namespace tiledb::sm {
 
 /** Class for ArrayDirectory status exceptions. */
 class ArrayDirectoryException : public StatusException {
@@ -266,7 +265,7 @@ const uint64_t& ArrayDirectory::timestamp_end() const {
 
 void ArrayDirectory::write_commit_ignore_file(
     const std::vector<URI>& commit_uris_to_ignore) {
-  auto name = compute_new_fragment_name(
+  auto name = storage_format::generate_consolidated_fragment_name(
       commit_uris_to_ignore.front(),
       commit_uris_to_ignore.back(),
       constants::format_version);
@@ -563,27 +562,6 @@ URI ArrayDirectory::get_vacuum_uri(const URI& fragment_uri) const {
   return URI(temp_uri.to_string() + constants::vacuum_file_suffix);
 }
 
-std::string ArrayDirectory::compute_new_fragment_name(
-    const URI& first, const URI& last, format_version_t format_version) const {
-  // Get uuid
-  std::string uuid;
-  throw_if_not_ok(uuid::generate_uuid(&uuid, false));
-
-  // For creating the new fragment URI
-
-  // Get timestamp ranges
-  std::pair<uint64_t, uint64_t> t_first, t_last;
-  throw_if_not_ok(utils::parse::get_timestamp_range(first, &t_first));
-  throw_if_not_ok(utils::parse::get_timestamp_range(last, &t_last));
-
-  // Create new URI
-  std::stringstream ss;
-  ss << "/__" << t_first.first << "_" << t_last.second << "_" << uuid << "_"
-     << format_version;
-
-  return ss.str();
-}
-
 bool ArrayDirectory::loaded() const {
   return loaded_;
 }
@@ -1334,5 +1312,5 @@ shared_ptr<const Enumeration> ArrayDirectory::load_enumeration(
   Deserializer deserializer(tile.data(), tile.size());
   return Enumeration::deserialize(deserializer);
 }
-}  // namespace sm
-}  // namespace tiledb
+
+}  // namespace tiledb::sm
diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h
@@ -48,8 +48,7 @@
 
 using namespace tiledb::common;
 
-namespace tiledb {
-namespace sm {
+namespace tiledb::sm {
 
 /** Mode for the ArrayDirectory class. */
 enum class ArrayDirectoryMode {
@@ -456,13 +455,6 @@ class ArrayDirectory {
   /** Returns the URI for a vacuum file. */
   URI get_vacuum_uri(const URI& fragment_uri) const;
 
-  /**
-   * The new fragment name is computed
-   * as `__<first_URI_timestamp>_<last_URI_timestamp>_<uuid>`.
-   */
-  std::string compute_new_fragment_name(
-      const URI& first, const URI& last, format_version_t format_version) const;
-
   /** Returns `true` if `load` has been run. */
   bool loaded() const;
 
@@ -832,7 +824,6 @@ class ArrayDirectory {
       MemoryTracker& memory_tracker) const;
 };
 
-}  // namespace sm
-}  // namespace tiledb
+}  // namespace tiledb::sm
 
 #endif  // TILEDB_ARRAY_DIRECTORY_H
diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc
@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2022 TileDB, Inc.
+ * @copyright Copyright (c) 2022-2023 TileDB, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -40,6 +40,7 @@
 #include "tiledb/sm/query/query.h"
 #include "tiledb/sm/stats/global_stats.h"
 #include "tiledb/sm/storage_manager/storage_manager.h"
+#include "tiledb/storage_format/uri/generate_uri.h"
 #include "tiledb/storage_format/uri/parse_uri.h"
 
 #include <iostream>
@@ -649,9 +650,8 @@ Status FragmentConsolidator::create_queries(
   auto last = (*query_r)->last_fragment_uri();
 
   auto write_version = array_for_reads->array_schema_latest().write_version();
-  auto fragment_name =
-      array_for_reads->array_directory().compute_new_fragment_name(
-          first, last, write_version);
+  auto fragment_name = storage_format::generate_consolidated_fragment_name(
+      first, last, write_version);
 
   // Create write query
   *query_w = tdb_new(Query, storage_manager_, array_for_writes, fragment_name);

diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc
@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2022 TileDB, Inc.
+ * @copyright Copyright (c) 2022-2023 TileDB, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -40,6 +40,7 @@
 #include "tiledb/sm/storage_manager/storage_manager.h"
 #include "tiledb/sm/tile/generic_tile_io.h"
 #include "tiledb/sm/tile/tile.h"
+#include "tiledb/storage_format/uri/generate_uri.h"
 #include "tiledb/storage_format/uri/parse_uri.h"
 
 using namespace tiledb::common;
@@ -92,7 +93,8 @@ Status FragmentMetaConsolidator::consolidate(
   auto first = meta.front()->fragment_uri();
   auto last = meta.back()->fragment_uri();
   auto write_version = array.array_schema_latest().write_version();
-  auto name = array_dir.compute_new_fragment_name(first, last, write_version);
+  auto name = storage_format::generate_consolidated_fragment_name(
+      first, last, write_version);
 
   auto frag_md_uri = array_dir.get_fragment_metadata_dir(write_version);
   RETURN_NOT_OK(storage_manager_->vfs()->create_dir(frag_md_uri));

diff --git a/tiledb/sm/group/group_directory.cc b/tiledb/sm/group/group_directory.cc
@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2017-2022 TileDB, Inc.
+ * @copyright Copyright (c) 2017-2023 TileDB, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -42,8 +42,7 @@
 
 using namespace tiledb::common;
 
-namespace tiledb {
-namespace sm {
+namespace tiledb::sm {
 
 /* ********************************* */
 /*     CONSTRUCTORS & DESTRUCTORS    */
@@ -168,29 +167,6 @@ Status GroupDirectory::load() {
   return Status::Ok();
 }
 
-tuple<Status, optional<std::string>> GroupDirectory::compute_new_fragment_name(
-    const URI& first, const URI& last, format_version_t format_version) const {
-  // Get uuid
-  std::string uuid;
-  RETURN_NOT_OK_TUPLE(uuid::generate_uuid(&uuid, false), nullopt);
-
-  // For creating the new fragment URI
-
-  // Get timestamp ranges
-  std::pair<uint64_t, uint64_t> t_first, t_last;
-  RETURN_NOT_OK_TUPLE(
-      utils::parse::get_timestamp_range(first, &t_first), nullopt);
-  RETURN_NOT_OK_TUPLE(
-      utils::parse::get_timestamp_range(last, &t_last), nullopt);
-
-  // Create new URI
-  std::stringstream ss;
-  ss << "/__" << t_first.first << "_" << t_last.second << "_" << uuid << "_"
-     << format_version;
-
-  return {Status::Ok(), ss.str()};
-}
-
 bool GroupDirectory::loaded() const {
   return loaded_;
 }
@@ -384,5 +360,4 @@ bool GroupDirectory::is_vacuum_file(const URI& uri) const {
   return false;
 }
 
-}  // namespace sm
-}  // namespace tiledb
+}  // namespace tiledb::sm
diff --git a/tiledb/sm/group/group_directory.h b/tiledb/sm/group/group_directory.h
@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2017-2022 TileDB, Inc.
+ * @copyright Copyright (c) 2017-2023 TileDB, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -42,8 +42,7 @@
 
 using namespace tiledb::common;
 
-namespace tiledb {
-namespace sm {
+namespace tiledb::sm {
 
 /** Mode for the GroupDirectory class. */
 enum class GroupDirectoryMode {
@@ -123,13 +122,6 @@ class GroupDirectory {
   /** Return the latest group details URI. */
   const URI& latest_group_details_uri() const;
 
-  /**
-   * The new fragment name is computed
-   * as `__<first_URI_timestamp>_<last_URI_timestamp>_<uuid>`.
-   */
-  tuple<Status, optional<std::string>> compute_new_fragment_name(
-      const URI& first, const URI& last, format_version_t format_version) const;
-
   /** Returns `true` if `load` has been run. */
   bool loaded() const;
 
@@ -243,7 +235,6 @@ class GroupDirectory {
   bool is_vacuum_file(const URI& uri) const;
 };
 
-}  // namespace sm
-}  // namespace tiledb
+}  // namespace tiledb::sm
 
 #endif  // TILEDB_GROUP_DIRECTORY_H
diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc
@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2022 TileDB, Inc.
+ * @copyright Copyright (c) 2022-2023 TileDB, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -41,8 +41,7 @@ using namespace tiledb;
 using namespace tiledb::common;
 using namespace tiledb::sm::stats;
 
-namespace tiledb {
-namespace sm {
+namespace tiledb::sm {
 
 class DeleteAndUpdateStatusException : public StatusException {
  public:
@@ -139,7 +138,7 @@ Status DeletesAndUpdates::dowork() {
   uint64_t timestamp = array_->timestamp_end_opened_at();
   auto write_version = array_->array_schema_latest().write_version();
   auto new_fragment_str =
-      storage_format::generate_fragment_name(timestamp, write_version);
+      storage_format::generate_timestamped_name(timestamp, write_version);
 
   // Check that the delete or update isn't in the middle of a fragment
   // consolidated without timestamps.
@@ -191,5 +190,4 @@ std::string DeletesAndUpdates::name() {
   return "DeletesAndUpdates";
 }
 
-}  // namespace sm
-}  // namespace tiledb
+}  // namespace tiledb::sm
diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc
@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2022 TileDB, Inc.
+ * @copyright Copyright (c) 2022-2023 TileDB, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -90,7 +90,7 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries(
         // or to get the timestamp_end from the parent array. This fix is
         // blocked by current discussion on a timestamp refactor design.
         if (!fragment_name_.has_value()) {
-          fragment_name_ = storage_format::generate_fragment_name(
+          fragment_name_ = storage_format::generate_timestamped_name(
               array->timestamp_end_opened_at(),
               array->array_schema_latest().write_version());
         }

diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc
@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2017-2022 TileDB, Inc.
+ * @copyright Copyright (c) 2017-2023 TileDB, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -58,8 +58,7 @@ using namespace tiledb;
 using namespace tiledb::common;
 using namespace tiledb::sm::stats;
 
-namespace tiledb {
-namespace sm {
+namespace tiledb::sm {
 
 class GlobalOrderWriterStatusException : public StatusException {
  public:
@@ -1438,7 +1437,7 @@ Status GlobalOrderWriter::start_new_fragment() {
   const auto write_version = array_->array_schema_latest().write_version();
   auto frag_dir_uri =
       array_->array_directory().get_fragments_dir(write_version);
-  auto new_fragment_str = storage_format::generate_uri(
+  auto new_fragment_str = storage_format::generate_timestamped_name(
       fragment_timestamp_range_.first,
       fragment_timestamp_range_.second,
       write_version);
@@ -1452,5 +1451,4 @@ Status GlobalOrderWriter::start_new_fragment() {
   return Status::Ok();
 }
 
-}  // namespace sm
-}  // namespace tiledb
+}  // namespace tiledb::sm
diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc
@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2017-2022 TileDB, Inc.
+ * @copyright Copyright (c) 2017-2023 TileDB, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -60,8 +60,7 @@ using namespace tiledb;
 using namespace tiledb::common;
 using namespace tiledb::sm::stats;
 
-namespace tiledb {
-namespace sm {
+namespace tiledb::sm {
 
 class WriterBaseStatusException : public StatusException {
  public:
@@ -225,7 +224,7 @@ WriterBase::WriterBase(
   auto new_fragment_str =
       fragment_name.has_value() ?
           fragment_name.value() :
-          storage_format::generate_fragment_name(timestamp, write_version);
+          storage_format::generate_timestamped_name(timestamp, write_version);
   auto frag_dir_uri =
       array_->array_directory().get_fragments_dir(write_version);
   fragment_uri_ = frag_dir_uri.join_path(new_fragment_str);
@@ -1205,5 +1204,4 @@ bool WriterBase::remote_query() const {
   return remote_query_;
 }
 
-}  // namespace sm
-}  // namespace tiledb
+}  // namespace tiledb::sm