feat: support pg_namespace, pg_class and related psql command (#4428)

* feat: add function 'pg_catalog.pg_table_is_visible'q

* feat: add 'pg_class' and 'pg_namespace', now we can run '\d' and '\dt'!

* refactor: move memory_table::tables to utils::tables

* refactor: move out predicate to system_schema to reuse it

* feat: predicates pushdown

* test: add pg_namespace, pg_class related sqlness test

* fix: typos and license header

* fix: sqlness test

* refactor: use `expect` instead of `unwrap` here

* refactor: remove the `information_schema::utils` mod

* doc: make the comment in pg_get_userbyid more precise

* doc: add TODO and comment in pg_catalog

* fix: typo

* fix: sqlness

* doc: change to comment on PGClassBuilder to TODO

src/catalog/src/system_schema.rs

@@ -15,6 +15,8 @@
 pub mod information_schema;
 mod memory_table;
 pub mod pg_catalog;
+mod predicate;
+mod utils;
 
 use std::collections::HashMap;
 use std::sync::Arc;

src/catalog/src/system_schema/information_schema.rs

@@ -18,14 +18,12 @@ pub mod flows;
 mod information_memory_table;
 pub mod key_column_usage;
 mod partitions;
-mod predicate;
 mod region_peers;
 mod runtime_metrics;
 pub mod schemata;
 mod table_constraints;
 mod table_names;
 pub mod tables;
-pub(crate) mod utils;
 mod views;
 
 use std::collections::HashMap;
@@ -37,7 +35,6 @@ use common_recordbatch::SendableRecordBatchStream;
 use datatypes::schema::SchemaRef;
 use lazy_static::lazy_static;
 use paste::paste;
-pub(crate) use predicate::Predicates;
 use store_api::storage::{ScanRequest, TableId};
 use table::metadata::TableType;
 use table::TableRef;
@@ -58,6 +55,7 @@ use crate::system_schema::information_schema::schemata::InformationSchemaSchemat
 use crate::system_schema::information_schema::table_constraints::InformationSchemaTableConstraints;
 use crate::system_schema::information_schema::tables::InformationSchemaTables;
 use crate::system_schema::memory_table::MemoryTable;
+pub(crate) use crate::system_schema::predicate::Predicates;
 use crate::system_schema::SystemSchemaProvider;
 use crate::CatalogManager;
 

src/catalog/src/system_schema/information_schema/cluster_info.rs

@@ -41,7 +41,8 @@ use store_api::storage::{ScanRequest, TableId};
 
 use super::CLUSTER_INFO;
 use crate::error::{CreateRecordBatchSnafu, InternalSnafu, ListNodesSnafu, Result};
-use crate::system_schema::information_schema::{utils, InformationTable, Predicates};
+use crate::system_schema::information_schema::{InformationTable, Predicates};
+use crate::system_schema::utils;
 use crate::CatalogManager;
 
 const PEER_ID: &str = "peer_id";

src/catalog/src/system_schema/information_schema/information_memory_table.rs

@@ -19,7 +19,7 @@ use datatypes::schema::{Schema, SchemaRef};
 use datatypes::vectors::{Int64Vector, StringVector, VectorRef};
 
 use super::table_names::*;
-use crate::system_schema::memory_table::tables::{
+use crate::system_schema::utils::tables::{
  bigint_column, datetime_column, string_column, string_columns,
 };
 

src/catalog/src/system_schema/information_schema/schemata.rs

@@ -36,7 +36,8 @@ use crate::error::{
  CreateRecordBatchSnafu, InternalSnafu, Result, TableMetadataManagerSnafu,
  UpgradeWeakCatalogManagerRefSnafu,
 };
-use crate::system_schema::information_schema::{utils, InformationTable, Predicates};
+use crate::system_schema::information_schema::{InformationTable, Predicates};
+use crate::system_schema::utils;
 use crate::CatalogManager;
 
 pub const CATALOG_NAME: &str = "catalog_name";

src/catalog/src/system_schema/memory_table.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 mod table_columns;
-pub mod tables;
 
 use std::sync::Arc;
 

src/catalog/src/system_schema/pg_catalog.rs

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 mod pg_catalog_memory_table;
+mod pg_class;
+mod pg_namespace;
 mod table_names;
 
 use std::collections::HashMap;
@@ -23,11 +25,13 @@ use datatypes::schema::ColumnSchema;
 use lazy_static::lazy_static;
 use paste::paste;
 use pg_catalog_memory_table::get_schema_columns;
+use pg_class::PGClass;
+use pg_namespace::PGNamespace;
 use table::TableRef;
 pub use table_names::*;
 
-use super::memory_table::tables::u32_column;
 use super::memory_table::MemoryTable;
+use super::utils::tables::u32_column;
 use super::{SystemSchemaProvider, SystemSchemaProviderInner, SystemTableRef};
 use crate::CatalogManager;
 
@@ -46,7 +50,7 @@ fn oid_column() -> ColumnSchema {
 /// [`PGCatalogProvider`] is the provider for a schema named `pg_catalog`, it is not a catalog.
 pub struct PGCatalogProvider {
  catalog_name: String,
- _catalog_manager: Weak<dyn CatalogManager>,
+ catalog_manager: Weak<dyn CatalogManager>,
  tables: HashMap<String, TableRef>,
 }
 
@@ -79,7 +83,7 @@ impl PGCatalogProvider {
  pub fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
  let mut provider = Self {
  catalog_name,
- _catalog_manager: catalog_manager,
+ catalog_manager,
  tables: HashMap::new(),
  };
  provider.build_tables();
@@ -90,9 +94,19 @@ impl PGCatalogProvider {
  // SECURITY NOTE:
  // Must follow the same security rules as [`InformationSchemaProvider::build_tables`].
  let mut tables = HashMap::new();
+ // TODO(J0HN50N133): modeling the table_name as a enum type to get rid of expect/unwrap here
+ // It's safe to unwrap here because we are sure that the constants have been handle correctly inside system_table.
  for name in MEMORY_TABLES.iter() {
  tables.insert(name.to_string(), self.build_table(name).expect(name));
  }
+ tables.insert(
+ PG_NAMESPACE.to_string(),
+ self.build_table(PG_NAMESPACE).expect(PG_NAMESPACE),
+ );
+ tables.insert(
+ PG_CLASS.to_string(),
+ self.build_table(PG_CLASS).expect(PG_NAMESPACE),
+ );
  self.tables = tables;
  }
 }
@@ -105,6 +119,14 @@ impl SystemSchemaProviderInner for PGCatalogProvider {
  fn system_table(&self, name: &str) -> Option<SystemTableRef> {
  match name {
  table_names::PG_TYPE => setup_memory_table!(PG_TYPE),
+ table_names::PG_NAMESPACE => Some(Arc::new(PGNamespace::new(
+ self.catalog_name.clone(),
+ self.catalog_manager.clone(),
+ ))),
+ table_names::PG_CLASS => Some(Arc::new(PGClass::new(
+ self.catalog_name.clone(),
+ self.catalog_manager.clone(),
+ ))),
  _ => None,
  }
  }

src/catalog/src/system_schema/pg_catalog/pg_catalog_memory_table.rs

@@ -20,7 +20,7 @@ use datatypes::vectors::{Int16Vector, StringVector, UInt32Vector, VectorRef};
 use super::oid_column;
 use super::table_names::PG_TYPE;
 use crate::memory_table_cols;
-use crate::system_schema::memory_table::tables::{i16_column, string_column};
+use crate::system_schema::utils::tables::{i16_column, string_column};
 
 fn pg_type_schema_columns() -> (Vec<ColumnSchema>, Vec<VectorRef>) {
  // TODO(j0hn50n133): acquire this information from `DataType` instead of hardcoding it to avoid regression.