Merge branch 'main' into benchmark

.github/workflows/build.yml

@@ -34,7 +34,7 @@ jobs:
     - name: Upload to codecov
       uses: codecov/codecov-action@v3
       with:
-        token: be8874e2-10d6-434f-9d52-db6094de31d6
+        token: b74fae07-452c-41fb-8eb2-b164ed90340d
         files: lcov.info
         name: codecov-umbrella # optional
         fail_ci_if_error: true

.gitignore

@@ -20,4 +20,4 @@ Cargo.lock
 
 database
 
-test/
+test/

Cargo.toml

@@ -14,6 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
 [package]
 name = "catalog2"
 version = "0.1.0"

doc/design_doc.md

@@ -10,24 +10,20 @@ The goal of this project is to design and implement a **Catalog Service** for an
 ## Architectural Design
 We follow the logic model described below. The input of our service comes from execution engine and I/O service. And we will provide metadata to planner and scheduler. We will use [pickleDB](https://docs.rs/pickledb/latest/pickledb/) as the key-value store to store (namespace, tables) and (table_name, metadata) as two (key, value) pairs as local db files.
 We will use [Rocket](https://rocket.rs) as the web framework handling incoming API traffic.
-We follow the logic model described below. The input of our service comes from execution engine and I/O service. And we will provide metadata to planner and scheduler. We will use [pickleDB](https://docs.rs/pickledb/latest/pickledb/) as the key-value store to store (namespace, tables) and (table_name, metadata) as two (key, value) pairs as local db files.
-We will use [Rocket](https://rocket.rs) as the web framework handling incoming API traffic.
+
 ![system architecture](./assets/system-architecture.png)
 ### Data Model
 We adhere to the Iceberg data model, arranging tables based on namespaces, with each table uniquely identified by its name.
 For every namespace in the database, there are associated list of tables.
 For every table in the catalog, there are associated metadata, including statistics, version, table-uuid, location, last-column-id, schema, and partition-spec.
 The parameters for request and response can be referenced from [REST API](https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml). We directly import Iceberg-Rust as a starting point.
-We adhere to the Iceberg data model, arranging tables based on namespaces, with each table uniquely identified by its name.
-For every namespace in the database, there are associated list of tables.
-For every table in the catalog, there are associated metadata, including statistics, version, table-uuid, location, last-column-id, schema, and partition-spec.
-The parameters for request and response can be referenced from [REST API](https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml). We directly import Iceberg-Rust as a starting point.
 
 ### Use Cases
 #### Namespace
 create/delete/rename/list namespace
 #### Table
 create/delete/rename/list table 
+
 #### Query Table’s Metadata (including statistics, version, table-uuid, location, last-column-id, schema, and partition-spec)
 get metadeta by {namespace}/{table}
 
@@ -41,7 +37,7 @@ get metadeta by {namespace}/{table}
   * Centralized metadata management achieved by separating data and metadata, reducing complexity and facilitating consistent metadata handling.
   * Code modularity and clear interfaces facilitate easier updates and improvements.
   * We adopt the existing kvstore ([pickleDB](https://docs.rs/pickledb/latest/pickledb/)) and server ([Rocket](https://github.com/rwf2/Rocket)) to mitigate the engineering complexity.
-  * We adopt the existing kvstore ([pickleDB](https://docs.rs/pickledb/latest/pickledb/)) and server ([Rocket](https://github.com/rwf2/Rocket)) to mitigate the engineering complexity.
+
 * Testing:
   * Comprehensive testing plans cover correctness through unit tests and performance through long-running regression tests. Unit tests focus on individual components of the catalog service, while regression tests evaluate system-wide performance and stability.
 * Other Implementations:

libs/iceberg/Cargo.toml

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "iceberg"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+
+categories = ["database"]
+description = "Apache Iceberg Rust implementation"
+repository = { workspace = true }
+license = { workspace = true }
+keywords = ["iceberg"]
+
+[dependencies]
+anyhow = { workspace = true }
+apache-avro = { workspace = true }
+arrow-arith = { workspace = true }
+arrow-array = { workspace = true }
+arrow-schema = { workspace = true }
+async-trait = { workspace = true }
+bimap = { workspace = true }
+bitvec = { workspace = true }
+chrono = { workspace = true }
+derive_builder = { workspace = true }
+either = { workspace = true }
+futures = { workspace = true }
+itertools = { workspace = true }
+lazy_static = { workspace = true }
+log = { workspace = true }
+murmur3 = { workspace = true }
+once_cell = { workspace = true }
+opendal = { workspace = true }
+ordered-float = { workspace = true }
+reqwest = { workspace = true }
+rust_decimal = { workspace = true }
+serde = { workspace = true }
+serde_bytes = { workspace = true }
+serde_derive = { workspace = true }
+serde_json = { workspace = true }
+serde_repr = { workspace = true }
+serde_with = { workspace = true }
+typed-builder = { workspace = true }
+url = { workspace = true }
+urlencoding = { workspace = true }
+uuid = { workspace = true }
+
+[dev-dependencies]
+pretty_assertions = { workspace = true }
+tempfile = { workspace = true }
+tera = { workspace = true }
+tokio = { workspace = true }