From d2447a28a9f99db86ec86dda3ddc123c785348f1 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 25 Sep 2024 19:33:56 +0800 Subject: [PATCH 01/89] Add a framework to support multi-storage in a pluginized manner for fileset catalog --- .gitignore | 4 ++ bundles/build.gradle.kts | 22 ++++++++++ bundles/hadoop-gcs-bundle/build.gradle.kts | 42 +++++++++++++++++++ .../fileset/gcs/GCSConfigurationProvider.java | 31 ++++++++++++++ bundles/hadoop-s3-bundle/build.gradle.kts | 42 +++++++++++++++++++ .../fileset/s3/S3ConfigurationProvider.java | 31 ++++++++++++++ .../catalog/hadoop/ConfigurationProvider.java | 37 ++++++++++++++++ .../hadoop/DefaultConfigurationProvider.java | 31 ++++++++++++++ .../hadoop/HadoopCatalogOperations.java | 14 +++++++ .../HadoopCatalogPropertiesMetadata.java | 13 ++++++ gradle/libs.versions.toml | 3 ++ settings.gradle.kts | 3 ++ 12 files changed, 273 insertions(+) create mode 100644 bundles/build.gradle.kts create mode 100644 bundles/hadoop-gcs-bundle/build.gradle.kts create mode 100644 bundles/hadoop-gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java create mode 100644 bundles/hadoop-s3-bundle/build.gradle.kts create mode 100644 bundles/hadoop-s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java create mode 100644 catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/ConfigurationProvider.java create mode 100644 catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java diff --git a/.gitignore b/.gitignore index 7889cf7a923..eae3d3c952c 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,7 @@ include clients/client-python/.gitignore **/metastore_db **/spark-warehouse derby.log + +web/node_modules +web/dist +web/.next diff --git a/bundles/build.gradle.kts b/bundles/build.gradle.kts new file mode 100644 index 00000000000..043fbfec673 --- /dev/null +++ b/bundles/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +tasks.all { + enabled = false +} \ No newline at end of file diff --git a/bundles/hadoop-gcs-bundle/build.gradle.kts b/bundles/hadoop-gcs-bundle/build.gradle.kts new file mode 100644 index 00000000000..69ccb1c13d3 --- /dev/null +++ b/bundles/hadoop-gcs-bundle/build.gradle.kts @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(libs.hadoop3.common) + implementation(libs.hadoop3.gcs) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} diff --git a/bundles/hadoop-gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java b/bundles/hadoop-gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java new file mode 100644 index 00000000000..74475c78e9b --- /dev/null +++ b/bundles/hadoop-gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.fileset.gcs; + +import java.util.Map; +import org.apache.gravitino.catalog.hadoop.DefaultConfigurationProvider; +import org.apache.hadoop.conf.Configuration; + +public class GCSConfigurationProvider extends DefaultConfigurationProvider { + // Add your own implementation here + @Override + public Configuration getConfiguration(Map conf) { + return super.getConfiguration(conf); + } +} diff --git a/bundles/hadoop-s3-bundle/build.gradle.kts b/bundles/hadoop-s3-bundle/build.gradle.kts new file mode 100644 index 00000000000..eeb05230fbc --- /dev/null +++ b/bundles/hadoop-s3-bundle/build.gradle.kts @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(libs.hadoop3.common) + implementation(libs.hadoop3.s3) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} diff --git a/bundles/hadoop-s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java b/bundles/hadoop-s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java new file mode 100644 index 00000000000..be62169a501 --- /dev/null +++ b/bundles/hadoop-s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.fileset.s3; + +import java.util.Map; +import org.apache.gravitino.catalog.hadoop.DefaultConfigurationProvider; +import org.apache.hadoop.conf.Configuration; + +public class S3ConfigurationProvider extends DefaultConfigurationProvider { + // Add your own implementation here + @Override + public Configuration getConfiguration(Map conf) { + return super.getConfiguration(conf); + } +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/ConfigurationProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/ConfigurationProvider.java new file mode 100644 index 00000000000..dd7ed972c6d --- /dev/null +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/ConfigurationProvider.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.catalog.hadoop; + +import java.util.Map; +import org.apache.hadoop.conf.Configuration; + +public interface ConfigurationProvider { + + default void initialize(Map conf) { + // Do nothing; + } + /** + * Get the configuration from the given properties. + * + * @param conf The properties to get the configuration from. + * @return The configuration. + */ + Configuration getConfiguration(Map conf); +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java new file mode 100644 index 00000000000..43dff12148e --- /dev/null +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.hadoop; + +import java.util.Map; +import org.apache.hadoop.conf.Configuration; + +public class DefaultConfigurationProvider implements ConfigurationProvider { + @Override + public Configuration getConfiguration(Map conf) { + Configuration configuration = new Configuration(); + conf.forEach(configuration::set); + return configuration; + } +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index da4d0e1a18e..98423c047ca 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -132,6 +132,20 @@ public void initialize( Map.Entry::getValue)); bypassConfigs.forEach(hadoopConf::set); + String configProviderClass = + config.getOrDefault( + HadoopCatalogPropertiesMetadata.CONFIGURATION_PROVIDER, + DefaultConfigurationProvider.class.getCanonicalName()); + try { + Class providerClass = Class.forName(configProviderClass); + ConfigurationProvider provider = + (ConfigurationProvider) providerClass.getDeclaredConstructor().newInstance(); + provider.initialize(bypassConfigs); + this.hadoopConf = provider.getConfiguration(bypassConfigs); + } catch (Exception e) { + throw new RuntimeException("Failed to initialize Hadoop configuration", e); + } + String catalogLocation = (String) propertiesMetadata diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 9a68e2d5522..e7dac0f1184 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -34,6 +34,11 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada // If not, users have to specify the storage location in the Schema or Fileset level. public static final String LOCATION = "location"; + /** + * The configuration provider class name, default value is {@link DefaultConfigurationProvider}. + */ + public static final String CONFIGURATION_PROVIDER = "configuration.provider"; + private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() .put( @@ -44,6 +49,14 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada false /* immutable */, null, false /* hidden */)) + .put( + CONFIGURATION_PROVIDER, + PropertyEntry.stringOptionalPropertyEntry( + CONFIGURATION_PROVIDER, + "The configuration provider class name", + false /* immutable */, + DefaultConfigurationProvider.class.getCanonicalName(), + false /* hidden */)) // The following two are about authentication. .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES) .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 255306c983c..60e47fd7660 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -32,6 +32,7 @@ airlift-resolver = "1.6" hive2 = "2.3.9" hadoop2 = "2.10.2" hadoop3 = "3.1.0" +hadoop3-gcs = "3.0.0" hadoop-minikdc = "3.3.6" htrace-core4 = "4.1.0-incubating" httpclient5 = "5.2.1" @@ -151,6 +152,8 @@ hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop3"} hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"} hadoop3-minicluster = { group = "org.apache.hadoop", name = "hadoop-minicluster", version.ref = "hadoop-minikdc"} +hadoop3-s3 = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref = "hadoop3"} +hadoop3-gcs = { group = "com.google.cloud.bigdataoss", name = "gcs-connector", version.ref = "hadoop3-gcs"} htrace-core4 = { group = "org.apache.htrace", name = "htrace-core4", version.ref = "htrace-core4" } airlift-json = { group = "io.airlift", name = "json", version.ref = "airlift-json"} airlift-resolver = { group = "io.airlift.resolver", name = "resolver", version.ref = "airlift-resolver"} diff --git a/settings.gradle.kts b/settings.gradle.kts index e98f81d39c0..25dcf8461ea 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -70,3 +70,6 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") +include("bundles:hadoop-s3-bundle") +include("bundles:hadoop-gcs-bundle") +findProject(":bundles:hadoop-gcs-bundle")?.name = "hadoop-gcs-bundle" From 7e5a8b5616dd099f8b14a502cc0d25f0cb3a29a8 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 25 Sep 2024 19:51:28 +0800 Subject: [PATCH 02/89] Fix compile distribution error. --- build.gradle.kts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index c6ea7f13cda..8920e9b378e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -744,7 +744,8 @@ tasks { if (!it.name.startsWith("catalog") && !it.name.startsWith("authorization") && !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && - it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") + it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && + it.name != "hadoop-gcs-bundle" && it.name != "hadoop-s3-bundle" ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -763,7 +764,8 @@ tasks { !it.name.startsWith("integration-test") && !it.name.startsWith("flink") && !it.name.startsWith("trino-connector") && - it.name != "hive-metastore-common" + it.name != "hive-metastore-common" && + it.name != "hadoop-gcs-bundle" && it.name != "hadoop-s3-bundle" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") From f53c5efc97830569357d327516a9b923004d5284 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 25 Sep 2024 22:10:04 +0800 Subject: [PATCH 03/89] fix --- bundles/hadoop-gcs-bundle/build.gradle.kts | 4 ++++ bundles/hadoop-s3-bundle/build.gradle.kts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/bundles/hadoop-gcs-bundle/build.gradle.kts b/bundles/hadoop-gcs-bundle/build.gradle.kts index 69ccb1c13d3..d0c25b7b40b 100644 --- a/bundles/hadoop-gcs-bundle/build.gradle.kts +++ b/bundles/hadoop-gcs-bundle/build.gradle.kts @@ -40,3 +40,7 @@ tasks.jar { dependsOn(tasks.named("shadowJar")) archiveClassifier.set("empty") } + +tasks.build { + dependsOn(tasks.named(":catalogs:catalog-hadoop:runtimeJars")) +} \ No newline at end of file diff --git a/bundles/hadoop-s3-bundle/build.gradle.kts b/bundles/hadoop-s3-bundle/build.gradle.kts index eeb05230fbc..6dd4c2d5a59 100644 --- a/bundles/hadoop-s3-bundle/build.gradle.kts +++ b/bundles/hadoop-s3-bundle/build.gradle.kts @@ -40,3 +40,7 @@ tasks.jar { dependsOn(tasks.named("shadowJar")) archiveClassifier.set("empty") } + +tasks.build { + dependsOn(tasks.named(":catalogs:catalog-hadoop:runtimeJars")) +} From 36fedcd85a2045cb5356033144e9b7bb3468b58a Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 25 Sep 2024 23:18:43 +0800 Subject: [PATCH 04/89] fix --- bundles/hadoop-gcs-bundle/build.gradle.kts | 4 ++-- bundles/hadoop-s3-bundle/build.gradle.kts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bundles/hadoop-gcs-bundle/build.gradle.kts b/bundles/hadoop-gcs-bundle/build.gradle.kts index d0c25b7b40b..21dc9093c48 100644 --- a/bundles/hadoop-gcs-bundle/build.gradle.kts +++ b/bundles/hadoop-gcs-bundle/build.gradle.kts @@ -41,6 +41,6 @@ tasks.jar { archiveClassifier.set("empty") } -tasks.build { - dependsOn(tasks.named(":catalogs:catalog-hadoop:runtimeJars")) +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") } \ No newline at end of file diff --git a/bundles/hadoop-s3-bundle/build.gradle.kts b/bundles/hadoop-s3-bundle/build.gradle.kts index 6dd4c2d5a59..f01941b27b2 100644 --- a/bundles/hadoop-s3-bundle/build.gradle.kts +++ b/bundles/hadoop-s3-bundle/build.gradle.kts @@ -41,6 +41,6 @@ tasks.jar { archiveClassifier.set("empty") } -tasks.build { - dependsOn(tasks.named(":catalogs:catalog-hadoop:runtimeJars")) +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") } From e93fba54507477999d44d7f99cbfb739d1bf6950 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 25 Sep 2024 23:30:50 +0800 Subject: [PATCH 05/89] fix --- bundles/hadoop-gcs-bundle/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundles/hadoop-gcs-bundle/build.gradle.kts b/bundles/hadoop-gcs-bundle/build.gradle.kts index 21dc9093c48..9433a600429 100644 --- a/bundles/hadoop-gcs-bundle/build.gradle.kts +++ b/bundles/hadoop-gcs-bundle/build.gradle.kts @@ -43,4 +43,4 @@ tasks.jar { tasks.compileJava { dependsOn(":catalogs:catalog-hadoop:runtimeJars") -} \ No newline at end of file +} From b1e04b644004702c8a418d6613eedb1f7c039a30 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 26 Sep 2024 12:10:24 +0800 Subject: [PATCH 06/89] fix --- build.gradle.kts | 4 ++-- .../build.gradle.kts | 0 .../fileset/gcs/GCSConfigurationProvider.java | 0 .../build.gradle.kts | 0 .../fileset/s3/S3ConfigurationProvider.java | 0 .../hadoop/DefaultConfigurationProvider.java | 15 ++++++++++++- .../hadoop/HadoopCatalogOperations.java | 22 +++++-------------- settings.gradle.kts | 5 ++--- 8 files changed, 23 insertions(+), 23 deletions(-) rename bundles/{hadoop-gcs-bundle => gcs-bundle}/build.gradle.kts (100%) rename bundles/{hadoop-gcs-bundle => gcs-bundle}/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java (100%) rename bundles/{hadoop-s3-bundle => s3-bundle}/build.gradle.kts (100%) rename bundles/{hadoop-s3-bundle => s3-bundle}/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java (100%) diff --git a/build.gradle.kts b/build.gradle.kts index 8920e9b378e..38c8c53ae90 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -745,7 +745,7 @@ tasks { !it.name.startsWith("authorization") && !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && - it.name != "hadoop-gcs-bundle" && it.name != "hadoop-s3-bundle" + it.name != "gcs-bundle" && it.name != "s3-bundle" ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -765,7 +765,7 @@ tasks { !it.name.startsWith("flink") && !it.name.startsWith("trino-connector") && it.name != "hive-metastore-common" && - it.name != "hadoop-gcs-bundle" && it.name != "hadoop-s3-bundle" + it.name != "gcs-bundle" && it.name != "s3-bundle" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/bundles/hadoop-gcs-bundle/build.gradle.kts b/bundles/gcs-bundle/build.gradle.kts similarity index 100% rename from bundles/hadoop-gcs-bundle/build.gradle.kts rename to bundles/gcs-bundle/build.gradle.kts diff --git a/bundles/hadoop-gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java similarity index 100% rename from bundles/hadoop-gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java rename to bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java diff --git a/bundles/hadoop-s3-bundle/build.gradle.kts b/bundles/s3-bundle/build.gradle.kts similarity index 100% rename from bundles/hadoop-s3-bundle/build.gradle.kts rename to bundles/s3-bundle/build.gradle.kts diff --git a/bundles/hadoop-s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java b/bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java similarity index 100% rename from bundles/hadoop-s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java rename to bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java index 43dff12148e..335c2cf0618 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java @@ -18,14 +18,27 @@ */ package org.apache.gravitino.catalog.hadoop; +import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; + import java.util.Map; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; public class DefaultConfigurationProvider implements ConfigurationProvider { @Override public Configuration getConfiguration(Map conf) { Configuration configuration = new Configuration(); - conf.forEach(configuration::set); + + // Get all configurations that start with the 'gravitino.bypass' prefix and remove the prefix + // to set them in the configuration. + Map bypassConfigs = + conf.entrySet().stream() + .filter(e -> e.getKey().startsWith(CATALOG_BYPASS_PREFIX)) + .collect( + Collectors.toMap( + e -> e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), + Map.Entry::getValue)); + bypassConfigs.forEach(configuration::set); return configuration; } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 98423c047ca..141461b0c5d 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -18,8 +18,6 @@ */ package org.apache.gravitino.catalog.hadoop; -import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; @@ -30,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; @@ -119,18 +116,10 @@ public void initialize( Map config, CatalogInfo info, HasPropertyMetadata propertiesMetadata) throws RuntimeException { this.propertiesMetadata = propertiesMetadata; + this.catalogInfo = info; + // Initialize Hadoop Configuration. this.conf = config; - this.hadoopConf = new Configuration(); - this.catalogInfo = info; - Map bypassConfigs = - config.entrySet().stream() - .filter(e -> e.getKey().startsWith(CATALOG_BYPASS_PREFIX)) - .collect( - Collectors.toMap( - e -> e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), - Map.Entry::getValue)); - bypassConfigs.forEach(hadoopConf::set); String configProviderClass = config.getOrDefault( @@ -140,19 +129,18 @@ public void initialize( Class providerClass = Class.forName(configProviderClass); ConfigurationProvider provider = (ConfigurationProvider) providerClass.getDeclaredConstructor().newInstance(); - provider.initialize(bypassConfigs); - this.hadoopConf = provider.getConfiguration(bypassConfigs); + this.hadoopConf = provider.getConfiguration(config); } catch (Exception e) { throw new RuntimeException("Failed to initialize Hadoop configuration", e); } + conf.forEach(hadoopConf::set); + String catalogLocation = (String) propertiesMetadata .catalogPropertiesMetadata() .getOrDefault(config, HadoopCatalogPropertiesMetadata.LOCATION); - conf.forEach(hadoopConf::set); - this.catalogStorageLocation = StringUtils.isNotBlank(catalogLocation) ? Optional.of(catalogLocation).map(Path::new) diff --git a/settings.gradle.kts b/settings.gradle.kts index 25dcf8461ea..de3168b278e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -70,6 +70,5 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") -include("bundles:hadoop-s3-bundle") -include("bundles:hadoop-gcs-bundle") -findProject(":bundles:hadoop-gcs-bundle")?.name = "hadoop-gcs-bundle" +include("bundles:s3-bundle") +include("bundles:gcs-bundle") From db00e650d7a7ea9e7decbd98d8f4e2591d5a7efd Mon Sep 17 00:00:00 2001 From: yuqi Date: Sun, 29 Sep 2024 13:46:23 +0800 Subject: [PATCH 07/89] Changed according to comments. --- ...ovider.java => GCSFileSystemProvider.java} | 28 ++++-- ...rovider.java => S3FileSystemProvider.java} | 27 ++++-- .../hadoop/DefaultConfigurationProvider.java | 44 --------- .../catalog/hadoop/FileSystemProvider.java | 53 +++++++++++ .../hadoop/HadoopCatalogOperations.java | 95 +++++++++++++++---- .../HadoopCatalogPropertiesMetadata.java | 14 +-- .../hadoop/fs/HDFSFileSystemProvider.java | 74 +++++++++++++++ .../LocalFileSystemProvider.java} | 35 ++++--- .../hadoop/TestHadoopCatalogOperations.java | 18 ++-- 9 files changed, 284 insertions(+), 104 deletions(-) rename bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/{GCSConfigurationProvider.java => GCSFileSystemProvider.java} (54%) rename bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/{S3ConfigurationProvider.java => S3FileSystemProvider.java} (54%) delete mode 100644 catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java create mode 100644 catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java create mode 100644 catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java rename catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/{ConfigurationProvider.java => fs/LocalFileSystemProvider.java} (51%) diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java similarity index 54% rename from bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java rename to bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java index 74475c78e9b..038bb884708 100644 --- a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSConfigurationProvider.java +++ b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java @@ -18,14 +18,30 @@ */ package org.apache.gravitino.fileset.gcs; -import java.util.Map; -import org.apache.gravitino.catalog.hadoop.DefaultConfigurationProvider; +import java.io.IOException; +import java.net.URI; +import org.apache.gravitino.catalog.hadoop.FileSystemProvider; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public class GCSFileSystemProvider implements FileSystemProvider { + + @Override + public FileSystem getFileSystem(Configuration configuration, Path path) throws IOException { + URI uri = path.toUri(); + if (uri.getScheme() == null || !uri.getScheme().equals("gs")) { + throw new IllegalArgumentException("The path should be a GCS path."); + } + + // TODO Check whether GCS related configurations are set such as filesystem.gs.impl, access key, + // secret key, etc. + + return FileSystem.get(uri, configuration); + } -public class GCSConfigurationProvider extends DefaultConfigurationProvider { - // Add your own implementation here @Override - public Configuration getConfiguration(Map conf) { - return super.getConfiguration(conf); + public String getScheme() { + return "gs"; } } diff --git a/bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java b/bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3FileSystemProvider.java similarity index 54% rename from bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java rename to bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3FileSystemProvider.java index be62169a501..d27fca9136c 100644 --- a/bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3ConfigurationProvider.java +++ b/bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3FileSystemProvider.java @@ -18,14 +18,29 @@ */ package org.apache.gravitino.fileset.s3; -import java.util.Map; -import org.apache.gravitino.catalog.hadoop.DefaultConfigurationProvider; +import java.io.IOException; +import java.net.URI; +import org.apache.gravitino.catalog.hadoop.FileSystemProvider; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public class S3FileSystemProvider implements FileSystemProvider { + + @Override + public FileSystem getFileSystem(Configuration configuration, Path path) throws IOException { + URI uri = path.toUri(); + if (uri.getScheme() == null || !uri.getScheme().equals("s3a")) { + throw new IllegalArgumentException("The path should be a S3 path."); + } + + // TODO Check whether S3 related configurations are set such as filesystem.s3a.impl, access key, + // secret key, etc. + return FileSystem.get(uri, configuration); + } -public class S3ConfigurationProvider extends DefaultConfigurationProvider { - // Add your own implementation here @Override - public Configuration getConfiguration(Map conf) { - return super.getConfiguration(conf); + public String getScheme() { + return "s3a"; } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java deleted file mode 100644 index 335c2cf0618..00000000000 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/DefaultConfigurationProvider.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.catalog.hadoop; - -import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; - -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; - -public class DefaultConfigurationProvider implements ConfigurationProvider { - @Override - public Configuration getConfiguration(Map conf) { - Configuration configuration = new Configuration(); - - // Get all configurations that start with the 'gravitino.bypass' prefix and remove the prefix - // to set them in the configuration. - Map bypassConfigs = - conf.entrySet().stream() - .filter(e -> e.getKey().startsWith(CATALOG_BYPASS_PREFIX)) - .collect( - Collectors.toMap( - e -> e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), - Map.Entry::getValue)); - bypassConfigs.forEach(configuration::set); - return configuration; - } -} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java new file mode 100644 index 00000000000..3332656b51d --- /dev/null +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.catalog.hadoop; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public interface FileSystemProvider { + + /** + * Get the FileSystem instance according to the configuration and the path. + * + *

Compared to the FileSystem.get method, this method allows the provider to create a + * FileSystem instance with a specific configuration and path and do further initialization if + * needed. + * + *

For example, we can check endpoint configurations for S3AFileSystem, or set the default one. + * + * @param configuration The configuration. + * @param path The path. + * @return The FileSystem instance. + * @throws IOException If the FileSystem instance cannot be created. + */ + FileSystem getFileSystem(Configuration configuration, Path path) throws IOException; + + /** + * Get the scheme of this FileSystem provider. + * + * @return The scheme of this FileSystem provider. + */ + default String getScheme() { + return "file"; + } +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 141461b0c5d..15485b40799 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.catalog.hadoop; +import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; @@ -28,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; @@ -41,6 +44,8 @@ import org.apache.gravitino.audit.CallerContext; import org.apache.gravitino.audit.FilesetAuditConstants; import org.apache.gravitino.audit.FilesetDataOperation; +import org.apache.gravitino.catalog.hadoop.fs.HDFSFileSystemProvider; +import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.connector.CatalogInfo; import org.apache.gravitino.connector.CatalogOperations; import org.apache.gravitino.connector.HasPropertyMetadata; @@ -74,6 +79,7 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private static final String SLASH = "/"; private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogOperations.class); + public static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); private final EntityStore store; @@ -87,6 +93,14 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private CatalogInfo catalogInfo; + static { + FileSystemProvider localFileSystemProvider = new LocalFileSystemProvider(); + FileSystemProvider hdfsFileSystemProvider = new HDFSFileSystemProvider(); + + FILE_SYSTEM_PROVIDERS.put(localFileSystemProvider.getScheme(), localFileSystemProvider); + FILE_SYSTEM_PROVIDERS.put(hdfsFileSystemProvider.getScheme(), hdfsFileSystemProvider); + } + HadoopCatalogOperations(EntityStore store) { this.store = store; } @@ -121,20 +135,17 @@ public void initialize( // Initialize Hadoop Configuration. this.conf = config; - String configProviderClass = - config.getOrDefault( - HadoopCatalogPropertiesMetadata.CONFIGURATION_PROVIDER, - DefaultConfigurationProvider.class.getCanonicalName()); - try { - Class providerClass = Class.forName(configProviderClass); - ConfigurationProvider provider = - (ConfigurationProvider) providerClass.getDeclaredConstructor().newInstance(); - this.hadoopConf = provider.getConfiguration(config); - } catch (Exception e) { - throw new RuntimeException("Failed to initialize Hadoop configuration", e); - } + hadoopConf = new Configuration(); + Map bypassConfigs = + conf.entrySet().stream() + .filter(e -> e.getKey().startsWith(CATALOG_BYPASS_PREFIX)) + .collect( + Collectors.toMap( + e -> e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), + Map.Entry::getValue)); + bypassConfigs.forEach(hadoopConf::set); - conf.forEach(hadoopConf::set); + initPluginFileSystem(conf); String catalogLocation = (String) @@ -147,6 +158,29 @@ public void initialize( : Optional.empty(); } + private void initPluginFileSystem(Map config) { + String fileSystemProviders = + (String) + propertiesMetadata + .catalogPropertiesMetadata() + .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDER); + + if (StringUtils.isNotBlank(fileSystemProviders)) { + String[] providers = fileSystemProviders.split(","); + for (String provider : providers) { + try { + FileSystemProvider fileSystemProvider = + (FileSystemProvider) + Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); + FILE_SYSTEM_PROVIDERS.put(fileSystemProvider.getScheme(), fileSystemProvider); + } catch (Exception e) { + throw new GravitinoRuntimeException( + e, "Failed to initialize file system provider: %s", provider); + } + } + } + } + @Override public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaException { try { @@ -238,7 +272,8 @@ public Fileset createFileset( try { // formalize the path to avoid path without scheme, uri, authority, etc. filesetPath = formalizePath(filesetPath, hadoopConf); - FileSystem fs = filesetPath.getFileSystem(hadoopConf); + + FileSystem fs = getFileSystem(filesetPath, hadoopConf); if (!fs.exists(filesetPath)) { if (!fs.mkdirs(filesetPath)) { throw new RuntimeException( @@ -341,7 +376,7 @@ public boolean dropFileset(NameIdentifier ident) { // For managed fileset, we should delete the related files. if (filesetEntity.filesetType() == Fileset.Type.MANAGED) { - FileSystem fs = filesetPath.getFileSystem(hadoopConf); + FileSystem fs = getFileSystem(filesetPath, hadoopConf); if (fs.exists(filesetPath)) { if (!fs.delete(filesetPath, true)) { LOG.warn("Failed to delete fileset {} location {}", ident, filesetPath); @@ -461,7 +496,7 @@ public Schema createSchema(NameIdentifier ident, String comment, Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() @@ -50,12 +52,12 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada null, false /* hidden */)) .put( - CONFIGURATION_PROVIDER, + FILESYSTEM_PROVIDER, PropertyEntry.stringOptionalPropertyEntry( - CONFIGURATION_PROVIDER, - "The configuration provider class name", + FILESYSTEM_PROVIDER, + "The file system provider class name", false /* immutable */, - DefaultConfigurationProvider.class.getCanonicalName(), + null, false /* hidden */)) // The following two are about authentication. .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java new file mode 100644 index 00000000000..aa1f88df0cd --- /dev/null +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.hadoop.fs; + +import java.io.IOException; +import java.net.URI; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.catalog.hadoop.FileSystemProvider; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public class HDFSFileSystemProvider implements FileSystemProvider { + + @Override + public FileSystem getFileSystem(Configuration configuration, Path path) throws IOException { + Path fileSystemPath = path; + if (fileSystemPath == null) { + String pathString = configuration.get("fs.defaultFS"); + if (StringUtils.isNotBlank(pathString)) { + fileSystemPath = new Path(pathString); + } + } + + if (fileSystemPath == null) { + throw new IllegalArgumentException("The path should be specified."); + } + + URI uri = path.toUri(); + if (uri.getScheme() == null || !uri.getScheme().equals("hdfs")) { + throw new IllegalArgumentException("The path should be a HDFS path."); + } + + // Should we call DistributedFileSystem to create file system instance explicitly? If we + // explicitly create a HDFS file system here, we can't reuse the file system cache in the + // FileSystem class. + if (configuration.get("fs.hdfs.impl") != null) { + configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); + } + + try { + if (HDFSFileSystemProvider.class.getClassLoader().loadClass(configuration.get("fs.hdfs.impl")) + == null) { + throw new IllegalArgumentException( + "The HDFS file system implementation class is not found."); + } + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("The HDFS file system implementation class is not found."); + } + + return FileSystem.get(uri, configuration); + } + + @Override + public String getScheme() { + return "hdfs"; + } +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/ConfigurationProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java similarity index 51% rename from catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/ConfigurationProvider.java rename to catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index dd7ed972c6d..75840be2def 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/ConfigurationProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -16,22 +16,31 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.gravitino.catalog.hadoop.fs; -package org.apache.gravitino.catalog.hadoop; - -import java.util.Map; +import java.io.IOException; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.catalog.hadoop.FileSystemProvider; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public class LocalFileSystemProvider implements FileSystemProvider { + + @Override + public FileSystem getFileSystem(Configuration configuration, Path path) throws IOException { + Path fileSystemPath = path; + if (fileSystemPath == null) { + String pathString = configuration.get("fs.defaultFS"); + if (StringUtils.isNotBlank(pathString)) { + fileSystemPath = new Path(pathString); + } + } -public interface ConfigurationProvider { + if (fileSystemPath == null) { + fileSystemPath = new Path("file:///"); + } - default void initialize(Map conf) { - // Do nothing; + return FileSystem.get(fileSystemPath.toUri(), configuration); } - /** - * Get the configuration from the given properties. - * - * @param conf The properties to get the configuration from. - * @return The configuration. - */ - Configuration getConfiguration(Map conf); } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index d3206972680..9709a4833d2 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -681,13 +681,10 @@ public void testFormalizePath() throws IOException { String[] paths = new String[] { - "tmp/catalog", - "/tmp/catalog", - "file:/tmp/catalog", - "file:///tmp/catalog", - "hdfs://localhost:9000/tmp/catalog", - "s3://bucket/tmp/catalog", - "gs://bucket/tmp/catalog" + "tmp/catalog", "/tmp/catalog", "file:/tmp/catalog", "file:///tmp/catalog", + // "hdfs://localhost:9000/tmp/catalog", + // "s3a://bucket/tmp/catalog", + // "gs://bucket/tmp/catalog" }; String[] expected = @@ -696,9 +693,9 @@ public void testFormalizePath() throws IOException { "file:/tmp/catalog", "file:/tmp/catalog", "file:/tmp/catalog", - "hdfs://localhost:9000/tmp/catalog", - "s3://bucket/tmp/catalog", - "gs://bucket/tmp/catalog" + // "hdfs://localhost:9000/tmp/catalog", + // "s3a://bucket/tmp/catalog", + // "gs://bucket/tmp/catalog" }; for (int i = 0; i < paths.length; i++) { @@ -877,6 +874,7 @@ public void testGetFileLocation() throws IOException { try (HadoopCatalogOperations mockOps = Mockito.mock(HadoopCatalogOperations.class)) { mockOps.hadoopConf = new Configuration(); when(mockOps.loadFileset(filesetIdent)).thenReturn(mockFileset); + when(mockOps.getConf()).thenReturn(Maps.newHashMap()); String subPath = "/test/test.parquet"; when(mockOps.getFileLocation(filesetIdent, subPath)).thenCallRealMethod(); String fileLocation = mockOps.getFileLocation(filesetIdent, subPath); From c793582122096b6508f7b8e5147ebd6316154797 Mon Sep 17 00:00:00 2001 From: yuqi Date: Sun, 29 Sep 2024 14:09:52 +0800 Subject: [PATCH 08/89] fix --- .../catalog/hadoop/fs/HDFSFileSystemProvider.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index aa1f88df0cd..4913b4c780d 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -50,8 +50,14 @@ public FileSystem getFileSystem(Configuration configuration, Path path) throws I // Should we call DistributedFileSystem to create file system instance explicitly? If we // explicitly create a HDFS file system here, we can't reuse the file system cache in the // FileSystem class. - if (configuration.get("fs.hdfs.impl") != null) { + String impl = configuration.get("fs.hdfs.impl"); + if (impl == null) { configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); + } else { + if (!impl.equals("org.apache.hadoop.hdfs.DistributedFileSystem")) { + throw new IllegalArgumentException( + "The HDFS file system implementation class should be 'org.apache.hadoop.hdfs.DistributedFileSystem'."); + } } try { From 013f5cb018cf272b27ac0dffa3154857259702b7 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 30 Sep 2024 20:28:04 +0800 Subject: [PATCH 09/89] fix --- build.gradle.kts | 6 +-- bundles/build.gradle.kts | 22 --------- bundles/gcs-bundle/build.gradle.kts | 46 ------------------ .../fileset/gcs/GCSFileSystemProvider.java | 47 ------------------- bundles/s3-bundle/build.gradle.kts | 46 ------------------ .../fileset/s3/S3FileSystemProvider.java | 46 ------------------ .../hadoop/fs/HDFSFileSystemProvider.java | 4 +- gradle/libs.versions.toml | 3 -- settings.gradle.kts | 2 - 9 files changed, 4 insertions(+), 218 deletions(-) delete mode 100644 bundles/build.gradle.kts delete mode 100644 bundles/gcs-bundle/build.gradle.kts delete mode 100644 bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java delete mode 100644 bundles/s3-bundle/build.gradle.kts delete mode 100644 bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3FileSystemProvider.java diff --git a/build.gradle.kts b/build.gradle.kts index 38c8c53ae90..c6ea7f13cda 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -744,8 +744,7 @@ tasks { if (!it.name.startsWith("catalog") && !it.name.startsWith("authorization") && !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && - it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && - it.name != "gcs-bundle" && it.name != "s3-bundle" + it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -764,8 +763,7 @@ tasks { !it.name.startsWith("integration-test") && !it.name.startsWith("flink") && !it.name.startsWith("trino-connector") && - it.name != "hive-metastore-common" && - it.name != "gcs-bundle" && it.name != "s3-bundle" + it.name != "hive-metastore-common" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/bundles/build.gradle.kts b/bundles/build.gradle.kts deleted file mode 100644 index 043fbfec673..00000000000 --- a/bundles/build.gradle.kts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -tasks.all { - enabled = false -} \ No newline at end of file diff --git a/bundles/gcs-bundle/build.gradle.kts b/bundles/gcs-bundle/build.gradle.kts deleted file mode 100644 index 9433a600429..00000000000 --- a/bundles/gcs-bundle/build.gradle.kts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar - -plugins { - `maven-publish` - id("java") - alias(libs.plugins.shadow) -} - -dependencies { - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(libs.hadoop3.common) - implementation(libs.hadoop3.gcs) -} - -tasks.withType(ShadowJar::class.java) { - isZip64 = true - configurations = listOf(project.configurations.runtimeClasspath.get()) - archiveClassifier.set("") -} - -tasks.jar { - dependsOn(tasks.named("shadowJar")) - archiveClassifier.set("empty") -} - -tasks.compileJava { - dependsOn(":catalogs:catalog-hadoop:runtimeJars") -} diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java deleted file mode 100644 index 038bb884708..00000000000 --- a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.fileset.gcs; - -import java.io.IOException; -import java.net.URI; -import org.apache.gravitino.catalog.hadoop.FileSystemProvider; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -public class GCSFileSystemProvider implements FileSystemProvider { - - @Override - public FileSystem getFileSystem(Configuration configuration, Path path) throws IOException { - URI uri = path.toUri(); - if (uri.getScheme() == null || !uri.getScheme().equals("gs")) { - throw new IllegalArgumentException("The path should be a GCS path."); - } - - // TODO Check whether GCS related configurations are set such as filesystem.gs.impl, access key, - // secret key, etc. - - return FileSystem.get(uri, configuration); - } - - @Override - public String getScheme() { - return "gs"; - } -} diff --git a/bundles/s3-bundle/build.gradle.kts b/bundles/s3-bundle/build.gradle.kts deleted file mode 100644 index f01941b27b2..00000000000 --- a/bundles/s3-bundle/build.gradle.kts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar - -plugins { - `maven-publish` - id("java") - alias(libs.plugins.shadow) -} - -dependencies { - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(libs.hadoop3.common) - implementation(libs.hadoop3.s3) -} - -tasks.withType(ShadowJar::class.java) { - isZip64 = true - configurations = listOf(project.configurations.runtimeClasspath.get()) - archiveClassifier.set("") -} - -tasks.jar { - dependsOn(tasks.named("shadowJar")) - archiveClassifier.set("empty") -} - -tasks.compileJava { - dependsOn(":catalogs:catalog-hadoop:runtimeJars") -} diff --git a/bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3FileSystemProvider.java b/bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3FileSystemProvider.java deleted file mode 100644 index d27fca9136c..00000000000 --- a/bundles/s3-bundle/src/main/java/org/apache/gravitino/fileset/s3/S3FileSystemProvider.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.fileset.s3; - -import java.io.IOException; -import java.net.URI; -import org.apache.gravitino.catalog.hadoop.FileSystemProvider; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -public class S3FileSystemProvider implements FileSystemProvider { - - @Override - public FileSystem getFileSystem(Configuration configuration, Path path) throws IOException { - URI uri = path.toUri(); - if (uri.getScheme() == null || !uri.getScheme().equals("s3a")) { - throw new IllegalArgumentException("The path should be a S3 path."); - } - - // TODO Check whether S3 related configurations are set such as filesystem.s3a.impl, access key, - // secret key, etc. - return FileSystem.get(uri, configuration); - } - - @Override - public String getScheme() { - return "s3a"; - } -} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index 4913b4c780d..0cf89583de7 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -42,8 +42,8 @@ public FileSystem getFileSystem(Configuration configuration, Path path) throws I throw new IllegalArgumentException("The path should be specified."); } - URI uri = path.toUri(); - if (uri.getScheme() == null || !uri.getScheme().equals("hdfs")) { + URI uri = fileSystemPath.toUri(); + if (uri.getScheme() != null && !uri.getScheme().equals("hdfs")) { throw new IllegalArgumentException("The path should be a HDFS path."); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 60e47fd7660..255306c983c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -32,7 +32,6 @@ airlift-resolver = "1.6" hive2 = "2.3.9" hadoop2 = "2.10.2" hadoop3 = "3.1.0" -hadoop3-gcs = "3.0.0" hadoop-minikdc = "3.3.6" htrace-core4 = "4.1.0-incubating" httpclient5 = "5.2.1" @@ -152,8 +151,6 @@ hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop3"} hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"} hadoop3-minicluster = { group = "org.apache.hadoop", name = "hadoop-minicluster", version.ref = "hadoop-minikdc"} -hadoop3-s3 = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref = "hadoop3"} -hadoop3-gcs = { group = "com.google.cloud.bigdataoss", name = "gcs-connector", version.ref = "hadoop3-gcs"} htrace-core4 = { group = "org.apache.htrace", name = "htrace-core4", version.ref = "htrace-core4" } airlift-json = { group = "io.airlift", name = "json", version.ref = "airlift-json"} airlift-resolver = { group = "io.airlift.resolver", name = "resolver", version.ref = "airlift-resolver"} diff --git a/settings.gradle.kts b/settings.gradle.kts index de3168b278e..e98f81d39c0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -70,5 +70,3 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") -include("bundles:s3-bundle") -include("bundles:gcs-bundle") From 16dfc73b6f19f63405817878add8539ef8a17a68 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 8 Oct 2024 22:47:08 +0800 Subject: [PATCH 10/89] resolve comments. --- .../catalog/hadoop/FileSystemProvider.java | 15 ++--- .../hadoop/HadoopCatalogOperations.java | 63 +++++++++++++------ .../hadoop/fs/HDFSFileSystemProvider.java | 18 +++--- .../hadoop/fs/LocalFileSystemProvider.java | 24 +++---- .../hadoop/TestHadoopCatalogOperations.java | 15 ++--- 5 files changed, 70 insertions(+), 65 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java index 3332656b51d..1632a161b25 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java @@ -20,9 +20,8 @@ package org.apache.gravitino.catalog.hadoop; import java.io.IOException; -import org.apache.hadoop.conf.Configuration; +import java.util.Map; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; public interface FileSystemProvider { @@ -35,19 +34,17 @@ public interface FileSystemProvider { * *

For example, we can check endpoint configurations for S3AFileSystem, or set the default one. * - * @param configuration The configuration. - * @param path The path. + * @param config The configuration for the FileSystem instance. * @return The FileSystem instance. * @throws IOException If the FileSystem instance cannot be created. */ - FileSystem getFileSystem(Configuration configuration, Path path) throws IOException; + FileSystem getFileSystem(Map config) throws IOException; /** - * Get the scheme of this FileSystem provider. + * Get the scheme of this FileSystem provider. file for LocalFileSystem, hdfs for HDFS, s3a for + * S3AFileSystem, etc. * * @return The scheme of this FileSystem provider. */ - default String getScheme() { - return "file"; - } + String getScheme(); } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 15485b40799..e4a52b6afd2 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -77,6 +77,8 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; private static final String SLASH = "/"; + private static final String DEFAULT_FS = "fs.defaultFS"; + private static final String LOCAL_FILE_SCHEMA = "file"; private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogOperations.class); public static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); @@ -91,6 +93,8 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private Map conf; + Map bypassConfigs; + private CatalogInfo catalogInfo; static { @@ -145,7 +149,9 @@ public void initialize( Map.Entry::getValue)); bypassConfigs.forEach(hadoopConf::set); - initPluginFileSystem(conf); + this.bypassConfigs = bypassConfigs; + + initPluginFileSystem(config); String catalogLocation = (String) @@ -271,9 +277,9 @@ public Fileset createFileset( try { // formalize the path to avoid path without scheme, uri, authority, etc. - filesetPath = formalizePath(filesetPath, hadoopConf); + filesetPath = formalizePath(filesetPath, bypassConfigs); - FileSystem fs = getFileSystem(filesetPath, hadoopConf); + FileSystem fs = getFileSystem(filesetPath, bypassConfigs); if (!fs.exists(filesetPath)) { if (!fs.mkdirs(filesetPath)) { throw new RuntimeException( @@ -376,7 +382,7 @@ public boolean dropFileset(NameIdentifier ident) { // For managed fileset, we should delete the related files. if (filesetEntity.filesetType() == Fileset.Type.MANAGED) { - FileSystem fs = getFileSystem(filesetPath, hadoopConf); + FileSystem fs = getFileSystem(filesetPath, bypassConfigs); if (fs.exists(filesetPath)) { if (!fs.delete(filesetPath, true)) { LOG.warn("Failed to delete fileset {} location {}", ident, filesetPath); @@ -496,7 +502,7 @@ public Schema createSchema(NameIdentifier ident, String comment, Map properties) { } @VisibleForTesting - static Path formalizePath(Path path, Configuration configuration) throws IOException { - FileSystem defaultFs = FileSystem.get(configuration); + static Path formalizePath(Path path, Map configuration) throws IOException { + FileSystem defaultFs = getFileSystem(path, configuration); return path.makeQualified(defaultFs.getUri(), defaultFs.getWorkingDirectory()); } @@ -767,7 +773,7 @@ private boolean hasCallerContext() { private boolean checkSingleFile(Fileset fileset) { try { Path locationPath = new Path(fileset.storageLocation()); - return getFileSystem(locationPath, hadoopConf).getFileStatus(locationPath).isFile(); + return getFileSystem(locationPath, bypassConfigs).getFileStatus(locationPath).isFile(); } catch (FileNotFoundException e) { // We should always return false here, same with the logic in `FileSystem.isFile(Path f)`. return false; @@ -779,19 +785,36 @@ private boolean checkSingleFile(Fileset fileset) { } } - static FileSystem getFileSystem(Path path, Configuration conf) throws IOException { - if (path == null) { - String defaultFS = conf.get("fs.defaultFS"); - if (defaultFS != null) { - path = new Path(defaultFS); + static FileSystem getFileSystem(Path path, Map config) throws IOException { + Map newConfig = Maps.newHashMap(config); + String scheme; + Path fsPath; + if (path != null) { + scheme = path.toUri().getScheme(); + if (scheme == null) { + scheme = LOCAL_FILE_SCHEMA; + } + fsPath = path; + } else { + + String defaultFS = config.get(DEFAULT_FS); + if (defaultFS == null) { + // Should be the local file system. + scheme = LOCAL_FILE_SCHEMA; + fsPath = new Path("file:///"); + } else { + fsPath = new Path(defaultFS); + if (fsPath.toUri().getScheme() == null) { + scheme = LOCAL_FILE_SCHEMA; + } else { + scheme = fsPath.toUri().getScheme(); + } } } - String scheme; - if (path == null || path.toUri().getScheme() == null) { - scheme = "file"; - } else { - scheme = path.toUri().getScheme(); + // For any non-local file system, we need to explicitly set the default FS. + if (!LOCAL_FILE_SCHEMA.equals(scheme)) { + newConfig.put(DEFAULT_FS, fsPath.toString()); } FileSystemProvider provider = FILE_SYSTEM_PROVIDERS.get(scheme); @@ -799,6 +822,6 @@ static FileSystem getFileSystem(Path path, Configuration conf) throws IOExceptio throw new IllegalArgumentException("Unsupported scheme: " + scheme); } - return provider.getFileSystem(conf, path); + return provider.getFileSystem(newConfig); } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index 0cf89583de7..ae2fdff29b5 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.net.URI; -import org.apache.commons.lang3.StringUtils; +import java.util.Map; import org.apache.gravitino.catalog.hadoop.FileSystemProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -29,20 +29,18 @@ public class HDFSFileSystemProvider implements FileSystemProvider { @Override - public FileSystem getFileSystem(Configuration configuration, Path path) throws IOException { - Path fileSystemPath = path; - if (fileSystemPath == null) { - String pathString = configuration.get("fs.defaultFS"); - if (StringUtils.isNotBlank(pathString)) { - fileSystemPath = new Path(pathString); - } + public FileSystem getFileSystem(Map config) throws IOException { + Configuration configuration = new Configuration(); + for (Map.Entry entry : config.entrySet()) { + configuration.set(entry.getKey(), entry.getValue()); } - if (fileSystemPath == null) { + String pathString = configuration.get("fs.defaultFS"); + if (pathString == null) { throw new IllegalArgumentException("The path should be specified."); } - URI uri = fileSystemPath.toUri(); + URI uri = new Path(pathString).toUri(); if (uri.getScheme() != null && !uri.getScheme().equals("hdfs")) { throw new IllegalArgumentException("The path should be a HDFS path."); } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 75840be2def..09c8a6b9fab 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -19,28 +19,22 @@ package org.apache.gravitino.catalog.hadoop.fs; import java.io.IOException; -import org.apache.commons.lang3.StringUtils; +import java.util.Map; import org.apache.gravitino.catalog.hadoop.FileSystemProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; public class LocalFileSystemProvider implements FileSystemProvider { @Override - public FileSystem getFileSystem(Configuration configuration, Path path) throws IOException { - Path fileSystemPath = path; - if (fileSystemPath == null) { - String pathString = configuration.get("fs.defaultFS"); - if (StringUtils.isNotBlank(pathString)) { - fileSystemPath = new Path(pathString); - } - } - - if (fileSystemPath == null) { - fileSystemPath = new Path("file:///"); - } + public FileSystem getFileSystem(Map config) throws IOException { + Configuration configuration = new Configuration(); + config.forEach(configuration::set); + return FileSystem.get(FileSystem.getDefaultUri(configuration), configuration); + } - return FileSystem.get(fileSystemPath.toUri(), configuration); + @Override + public String getScheme() { + return "file"; } } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 9709a4833d2..360d65466cf 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -680,26 +680,18 @@ public void testAlterFilesetProperties() throws IOException { public void testFormalizePath() throws IOException { String[] paths = - new String[] { - "tmp/catalog", "/tmp/catalog", "file:/tmp/catalog", "file:///tmp/catalog", - // "hdfs://localhost:9000/tmp/catalog", - // "s3a://bucket/tmp/catalog", - // "gs://bucket/tmp/catalog" - }; + new String[] {"tmp/catalog", "/tmp/catalog", "file:/tmp/catalog", "file:///tmp/catalog"}; String[] expected = new String[] { "file:" + Paths.get("").toAbsolutePath() + "/tmp/catalog", "file:/tmp/catalog", "file:/tmp/catalog", - "file:/tmp/catalog", - // "hdfs://localhost:9000/tmp/catalog", - // "s3a://bucket/tmp/catalog", - // "gs://bucket/tmp/catalog" + "file:/tmp/catalog" }; for (int i = 0; i < paths.length; i++) { - Path actual = HadoopCatalogOperations.formalizePath(new Path(paths[i]), new Configuration()); + Path actual = HadoopCatalogOperations.formalizePath(new Path(paths[i]), Maps.newHashMap()); Assertions.assertEquals(expected[i], actual.toString()); } } @@ -873,6 +865,7 @@ public void testGetFileLocation() throws IOException { try (HadoopCatalogOperations mockOps = Mockito.mock(HadoopCatalogOperations.class)) { mockOps.hadoopConf = new Configuration(); + mockOps.bypassConfigs = Maps.newHashMap(); when(mockOps.loadFileset(filesetIdent)).thenReturn(mockFileset); when(mockOps.getConf()).thenReturn(Maps.newHashMap()); String subPath = "/test/test.parquet"; From 278fcd87b5737ecb6cbbf4aa5bcf1af265ff2db0 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 9 Oct 2024 11:45:41 +0800 Subject: [PATCH 11/89] Polish code. --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 5 +++-- .../gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java | 4 +--- .../catalog/hadoop/fs/LocalFileSystemProvider.java | 6 +++++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index e4a52b6afd2..5a7d837acea 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -77,8 +77,9 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; private static final String SLASH = "/"; - private static final String DEFAULT_FS = "fs.defaultFS"; + public static final String DEFAULT_FS = "fs.defaultFS"; private static final String LOCAL_FILE_SCHEMA = "file"; + public static final String LOCAL_FILE_PATH = "file:///"; private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogOperations.class); public static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); @@ -801,7 +802,7 @@ static FileSystem getFileSystem(Path path, Map config) throws IO if (defaultFS == null) { // Should be the local file system. scheme = LOCAL_FILE_SCHEMA; - fsPath = new Path("file:///"); + fsPath = new Path(LOCAL_FILE_PATH); } else { fsPath = new Path(defaultFS); if (fsPath.toUri().getScheme() == null) { diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index ae2fdff29b5..5f96a85642b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -31,9 +31,7 @@ public class HDFSFileSystemProvider implements FileSystemProvider { @Override public FileSystem getFileSystem(Map config) throws IOException { Configuration configuration = new Configuration(); - for (Map.Entry entry : config.entrySet()) { - configuration.set(entry.getKey(), entry.getValue()); - } + config.forEach(configuration::set); String pathString = configuration.get("fs.defaultFS"); if (pathString == null) { diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 09c8a6b9fab..a4c99f0a31a 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -18,6 +18,9 @@ */ package org.apache.gravitino.catalog.hadoop.fs; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogOperations.DEFAULT_FS; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogOperations.LOCAL_FILE_PATH; + import java.io.IOException; import java.util.Map; import org.apache.gravitino.catalog.hadoop.FileSystemProvider; @@ -30,7 +33,8 @@ public class LocalFileSystemProvider implements FileSystemProvider { public FileSystem getFileSystem(Map config) throws IOException { Configuration configuration = new Configuration(); config.forEach(configuration::set); - return FileSystem.get(FileSystem.getDefaultUri(configuration), configuration); + config.put(DEFAULT_FS, LOCAL_FILE_PATH); + return FileSystem.get(configuration); } @Override From 3fb55ad4211db30dca143a773d4dcf7142f58541 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 9 Oct 2024 18:54:59 +0800 Subject: [PATCH 12/89] fix --- .../catalog/hadoop/HadoopCatalogPropertiesMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 8637ad5a2db..9cc7c6b7497 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -55,7 +55,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada FILESYSTEM_PROVIDER, PropertyEntry.stringOptionalPropertyEntry( FILESYSTEM_PROVIDER, - "The file system provider class name", + "The file system provider class name, separated by comma", false /* immutable */, null, false /* hidden */)) From cd04666df1b32078ed7b558af8b3f680bcb4e1f2 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 9 Oct 2024 18:58:30 +0800 Subject: [PATCH 13/89] fix --- .../catalog/hadoop/HadoopCatalogPropertiesMetadata.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 9cc7c6b7497..37d6f9ea99e 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -38,6 +38,8 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada * The implementation class name of the {@link FileSystemProvider} to be used by the catalog. * Gravitino supports LocalFileSystem and HDFS by default. Users can implement their own by * extending {@link FileSystemProvider} and specify the class name here. + * + *

The value can be 'xxxx.yyy.FileSystemProvider1, xxxx.yyy.FileSystemProvider2'. */ public static final String FILESYSTEM_PROVIDER = "filesystem.providers"; From d0bf13eac6849db0bc90aa5a4bff5e7c974eed3f Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 9 Oct 2024 21:20:40 +0800 Subject: [PATCH 14/89] Support GCS fileset. --- build.gradle.kts | 4 +- bundles/build.gradle.kts | 22 +++++ bundles/gcs-bundle/build.gradle.kts | 46 +++++++++ .../fileset/gcs/GCSFileSystemProvider.java | 39 ++++++++ catalogs/catalog-hadoop/build.gradle.kts | 2 + .../integration/test/HadoopCatalogIT.java | 63 ++++++------ .../integration/test/HadoopGCPCatalogIT.java | 96 +++++++++++++++++++ gradle/libs.versions.toml | 2 + settings.gradle.kts | 1 + 9 files changed, 242 insertions(+), 33 deletions(-) create mode 100644 bundles/build.gradle.kts create mode 100644 bundles/gcs-bundle/build.gradle.kts create mode 100644 bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java create mode 100644 catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java diff --git a/build.gradle.kts b/build.gradle.kts index 73cbdd80f24..b4841f8c552 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -744,7 +744,7 @@ tasks { if (!it.name.startsWith("catalog") && !it.name.startsWith("authorization") && !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && - it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") + it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && it.name != "gcs-bundle" ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -763,7 +763,7 @@ tasks { !it.name.startsWith("integration-test") && !it.name.startsWith("flink") && !it.name.startsWith("trino-connector") && - it.name != "hive-metastore-common" + it.name != "hive-metastore-common" && it.name != "gcs-bundle" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/bundles/build.gradle.kts b/bundles/build.gradle.kts new file mode 100644 index 00000000000..043fbfec673 --- /dev/null +++ b/bundles/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +tasks.all { + enabled = false +} \ No newline at end of file diff --git a/bundles/gcs-bundle/build.gradle.kts b/bundles/gcs-bundle/build.gradle.kts new file mode 100644 index 00000000000..9433a600429 --- /dev/null +++ b/bundles/gcs-bundle/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(libs.hadoop3.common) + implementation(libs.hadoop3.gcs) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java new file mode 100644 index 00000000000..0bbd772f23c --- /dev/null +++ b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.fileset.gcs; + +import java.io.IOException; +import java.util.Map; +import org.apache.gravitino.catalog.hadoop.FileSystemProvider; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; + +public class GCSFileSystemProvider implements FileSystemProvider { + @Override + public FileSystem getFileSystem(Map config) throws IOException { + Configuration configuration = new Configuration(); + config.forEach(configuration::set); + return FileSystem.get(configuration); + } + + @Override + public String getScheme() { + return "gs"; + } +} diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index ba60a161d8f..07712b50a27 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -71,6 +71,7 @@ dependencies { testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(project(":server")) testImplementation(project(":server-common")) + testImplementation(project(":bundles:gcs-bundle")) testImplementation(libs.minikdc) testImplementation(libs.hadoop3.minicluster) @@ -84,6 +85,7 @@ dependencies { testImplementation(libs.junit.jupiter.params) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) + testImplementation(libs.hadoop3.gcs) testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index 20f9a1eeab8..5adf5e8542c 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -53,32 +53,32 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class HadoopCatalogIT extends AbstractIT { private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - public static final String metalakeName = - GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); - public static final String catalogName = - GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); + public String metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); + public String catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); public static final String SCHEMA_PREFIX = "CatalogFilesetIT_schema"; - public static final String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); - private static final String provider = "hadoop"; - private static GravitinoMetalake metalake; - private static Catalog catalog; - private static FileSystem hdfs; - private static String defaultBaseLocation; + public String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); + protected static final String provider = "hadoop"; + protected static GravitinoMetalake metalake; + protected static Catalog catalog; + protected static FileSystem fileSystem; + protected static String defaultBaseLocation; @BeforeAll - public static void setup() throws IOException { + public void setup() throws IOException { containerSuite.startHiveContainer(); Configuration conf = new Configuration(); conf.set("fs.defaultFS", defaultBaseLocation()); - hdfs = FileSystem.get(conf); + fileSystem = FileSystem.get(conf); createMetalake(); createCatalog(); @@ -86,13 +86,13 @@ public static void setup() throws IOException { } @AfterAll - public static void stop() throws IOException { + public void stop() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); metalake.dropCatalog(catalogName); client.dropMetalake(metalakeName); - if (hdfs != null) { - hdfs.close(); + if (fileSystem != null) { + fileSystem.close(); } try { @@ -102,7 +102,7 @@ public static void stop() throws IOException { } } - private static void createMetalake() { + protected void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); @@ -114,14 +114,14 @@ private static void createMetalake() { metalake = loadMetalake; } - private static void createCatalog() { + protected void createCatalog() { metalake.createCatalog( catalogName, Catalog.Type.FILESET, provider, "comment", ImmutableMap.of()); catalog = metalake.loadCatalog(catalogName); } - private static void createSchema() { + protected void createSchema() { Map properties = Maps.newHashMap(); properties.put("key1", "val1"); properties.put("key2", "val2"); @@ -137,7 +137,7 @@ private static void createSchema() { Assertions.assertNotNull(loadSchema.properties().get("location")); } - private static void dropSchema() { + private void dropSchema() { catalog.asSchemas().dropSchema(schemaName, true); Assertions.assertFalse(catalog.asSchemas().schemaExists(schemaName)); } @@ -171,7 +171,7 @@ public void testCreateFileset() throws IOException { String filesetName = "test_create_fileset"; String storageLocation = storageLocation(filesetName); Assertions.assertFalse( - hdfs.exists(new Path(storageLocation)), "storage location should not exists"); + fileSystem.exists(new Path(storageLocation)), "storage location should not exists"); Fileset fileset = createFileset( filesetName, @@ -242,7 +242,7 @@ public void testCreateFilesetWithChinese() throws IOException { String filesetName = "test_create_fileset_with_chinese"; String storageLocation = storageLocation(filesetName) + "/中文目录test"; Assertions.assertFalse( - hdfs.exists(new Path(storageLocation)), "storage location should not exists"); + fileSystem.exists(new Path(storageLocation)), "storage location should not exists"); Fileset fileset = createFileset( filesetName, @@ -285,7 +285,7 @@ public void testExternalFileset() throws IOException { Assertions.assertEquals(1, fileset.properties().size()); Assertions.assertEquals("v1", fileset.properties().get("k1")); Assertions.assertTrue( - hdfs.exists(new Path(storageLocation)), "storage location should be created"); + fileSystem.exists(new Path(storageLocation)), "storage location should be created"); // create fileset with storage location that not exist String filesetName2 = "test_external_fileset_no_exist"; @@ -349,7 +349,7 @@ public void testDropManagedFileset() throws IOException { String storageLocation = storageLocation(filesetName); Assertions.assertFalse( - hdfs.exists(new Path(storageLocation)), "storage location should not exists"); + fileSystem.exists(new Path(storageLocation)), "storage location should not exists"); createFileset( filesetName, "comment", Fileset.Type.MANAGED, storageLocation, ImmutableMap.of("k1", "v1")); @@ -365,7 +365,7 @@ public void testDropManagedFileset() throws IOException { catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, filesetName)), "fileset should not be exists"); Assertions.assertFalse( - hdfs.exists(new Path(storageLocation)), "storage location should be dropped"); + fileSystem.exists(new Path(storageLocation)), "storage location should be dropped"); } @Test @@ -392,7 +392,7 @@ public void testDropExternalFileset() throws IOException { catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, filesetName)), "fileset should not be exists"); Assertions.assertTrue( - hdfs.exists(new Path(storageLocation)), "storage location should not be dropped"); + fileSystem.exists(new Path(storageLocation)), "storage location should not be dropped"); } @Test @@ -688,7 +688,7 @@ public void testGetFileLocationWithInvalidAuditHeaders() { } } - private static String generateLocation(String filesetName) { + protected String generateLocation(String filesetName) { return String.format( "hdfs://%s:%d/user/hadoop/%s/%s/%s", containerSuite.getHiveContainer().getContainerIpAddress(), @@ -707,7 +707,7 @@ private Fileset createFileset( if (storageLocation != null) { Path location = new Path(storageLocation); try { - hdfs.deleteOnExit(location); + fileSystem.deleteOnExit(location); } catch (IOException e) { LOG.warn("Failed to delete location: {}", location, e); } @@ -724,10 +724,11 @@ private void assertFilesetExists(String filesetName) throws IOException { catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, filesetName)), "fileset should be exists"); Assertions.assertTrue( - hdfs.exists(new Path(storageLocation(filesetName))), "storage location should be exists"); + fileSystem.exists(new Path(storageLocation(filesetName))), + "storage location should be exists"); } - private static String defaultBaseLocation() { + protected String defaultBaseLocation() { if (defaultBaseLocation == null) { defaultBaseLocation = String.format( @@ -739,7 +740,7 @@ private static String defaultBaseLocation() { return defaultBaseLocation; } - private static String storageLocation(String filesetName) { + private String storageLocation(String filesetName) { return defaultBaseLocation() + "/" + filesetName; } } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java new file mode 100644 index 00000000000..13d944f9d1b --- /dev/null +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.hadoop.integration.test; + +import com.google.common.collect.Maps; +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestInstance; + +@Tag("gravitino-docker-test") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Disabled( + "Disabled due to as we don't have a real GCP account to test. If you have a GCP account," + + "please change the configuration(YOUR_KEY_FILE, YOUR_BUCKET) and enable this test.") +public class HadoopGCPCatalogIT extends HadoopCatalogIT { + + @BeforeAll + public void setup() throws IOException { + metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); + catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); + schemaName = GravitinoITUtils.genRandomName("CatalogFilesetIT_schema"); + + schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); + Configuration conf = new Configuration(); + + conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); + conf.set("fs.gs.auth.service.account.enable", "true"); + conf.set("fs.gs.auth.service.account.json.keyfile", "YOUR_KEY_FILE"); + conf.set("fs.defaultFS", "gs:///"); + fileSystem = FileSystem.get(URI.create("gs://YOUR_BUCKET"), conf); + + createMetalake(); + createCatalog(); + createSchema(); + } + + protected String defaultBaseLocation() { + if (defaultBaseLocation == null) { + try { + Path bucket = + new Path("gs://YOUR_BUCKET/" + GravitinoITUtils.genRandomName("CatalogFilesetIT")); + if (!fileSystem.exists(bucket)) { + fileSystem.mkdirs(bucket); + } + + defaultBaseLocation = bucket.toString(); + } catch (IOException e) { + throw new RuntimeException("Failed to create default base location", e); + } + } + + return defaultBaseLocation; + } + + protected void createCatalog() { + Map map = Maps.newHashMap(); + map.put("gravitino.bypass.fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); + map.put("gravitino.bypass.fs.gs.auth.service.account.enable", "true"); + map.put("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", "YOUR_KEY_FILE"); + map.put("gravitino.bypass.fs.defaultFS", "gs:///"); + map.put("filesystem.providers", "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider"); + + metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map); + + catalog = metalake.loadCatalog(catalogName); + } + + protected String generateLocation(String filesetName) { + return String.format("%s/%s", defaultBaseLocation, filesetName); + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 255306c983c..ee1abd3687a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -32,6 +32,7 @@ airlift-resolver = "1.6" hive2 = "2.3.9" hadoop2 = "2.10.2" hadoop3 = "3.1.0" +hadoop3-gcs = "1.9.4-hadoop3" hadoop-minikdc = "3.3.6" htrace-core4 = "4.1.0-incubating" httpclient5 = "5.2.1" @@ -151,6 +152,7 @@ hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop3"} hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"} hadoop3-minicluster = { group = "org.apache.hadoop", name = "hadoop-minicluster", version.ref = "hadoop-minikdc"} +hadoop3-gcs = { group = "com.google.cloud.bigdataoss", name = "gcs-connector", version.ref = "hadoop3-gcs"} htrace-core4 = { group = "org.apache.htrace", name = "htrace-core4", version.ref = "htrace-core4" } airlift-json = { group = "io.airlift", name = "json", version.ref = "airlift-json"} airlift-resolver = { group = "io.airlift.resolver", name = "resolver", version.ref = "airlift-resolver"} diff --git a/settings.gradle.kts b/settings.gradle.kts index e98f81d39c0..dcaa8fbe6f4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -70,3 +70,4 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") +include(":bundles:gcs-bundle") From ffaa064df11a9f35036ece6d32591299cdfb45b4 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 11 Oct 2024 10:51:24 +0800 Subject: [PATCH 15/89] Change gvfs accordingly. --- .../hadoop/fs/HDFSFileSystemProvider.java | 2 +- .../hadoop/fs/LocalFileSystemProvider.java | 2 +- clients/filesystem-hadoop3/build.gradle.kts | 9 +++ .../hadoop/GravitinoVirtualFileSystem.java | 59 ++++++++++++++++++- 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index 5f96a85642b..1f74df599a7 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -66,7 +66,7 @@ public FileSystem getFileSystem(Map config) throws IOException { throw new IllegalArgumentException("The HDFS file system implementation class is not found."); } - return FileSystem.get(uri, configuration); + return FileSystem.newInstance(uri, configuration); } @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index a4c99f0a31a..6fe80b78880 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -34,7 +34,7 @@ public FileSystem getFileSystem(Map config) throws IOException { Configuration configuration = new Configuration(); config.forEach(configuration::set); config.put(DEFAULT_FS, LOCAL_FILE_PATH); - return FileSystem.get(configuration); + return FileSystem.newInstance(configuration); } @Override diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts index d7905cd3b35..aefac5f28b9 100644 --- a/clients/filesystem-hadoop3/build.gradle.kts +++ b/clients/filesystem-hadoop3/build.gradle.kts @@ -26,6 +26,10 @@ plugins { dependencies { compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) compileOnly(libs.hadoop3.common) + implementation(project(":catalogs:catalog-hadoop")) { + exclude(group = "*") + } + implementation(libs.caffeine) testImplementation(project(":api")) @@ -71,6 +75,11 @@ tasks.build { dependsOn("javadoc") } +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:jar") + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} + tasks.test { val skipITs = project.hasProperty("skipITs") if (skipITs) { diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index de0eb758edc..8f4368aa430 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -41,6 +41,9 @@ import org.apache.gravitino.audit.FilesetAuditConstants; import org.apache.gravitino.audit.FilesetDataOperation; import org.apache.gravitino.audit.InternalClientType; +import org.apache.gravitino.catalog.hadoop.FileSystemProvider; +import org.apache.gravitino.catalog.hadoop.fs.HDFSFileSystemProvider; +import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.client.DefaultOAuth2TokenProvider; import org.apache.gravitino.client.GravitinoClient; import org.apache.gravitino.client.KerberosTokenProvider; @@ -81,6 +84,14 @@ public class GravitinoVirtualFileSystem extends FileSystem { private static final Pattern IDENTIFIER_PATTERN = Pattern.compile("^(?:gvfs://fileset)?/([^/]+)/([^/]+)/([^/]+)(?>/[^/]+)*/?$"); private static final String SLASH = "/"; + private static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); + private static final String GRAVITINO_BYPASS_PREFIX = "gravitino.bypass."; + + static { + // Register the default FileSystemProvider + FILE_SYSTEM_PROVIDERS.put("file", new LocalFileSystemProvider()); + FILE_SYSTEM_PROVIDERS.put("hdfs", new HDFSFileSystemProvider()); + } @Override public void initialize(URI name, Configuration configuration) throws IOException { @@ -125,6 +136,8 @@ public void initialize(URI name, Configuration configuration) throws IOException initializeClient(configuration); + initializePluginFileSystem(configuration); + this.workingDirectory = new Path(name); this.uri = URI.create(name.getScheme() + "://" + name.getAuthority()); @@ -132,6 +145,24 @@ public void initialize(URI name, Configuration configuration) throws IOException super.initialize(uri, getConf()); } + private void initializePluginFileSystem(Configuration configuration) { + String fileSystemProviders = configuration.get("fs.gvfs.filesystem.providers"); + if (StringUtils.isNotBlank(fileSystemProviders)) { + String[] providers = fileSystemProviders.split(","); + for (String provider : providers) { + try { + FileSystemProvider fileSystemProvider = + (FileSystemProvider) + Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); + FILE_SYSTEM_PROVIDERS.put(fileSystemProvider.getScheme(), fileSystemProvider); + } catch (Exception e) { + throw new GravitinoRuntimeException( + e, "Failed to initialize file system provider: %s", provider); + } + } + } + } + @VisibleForTesting Cache internalFileSystemCache() { return internalFileSystemCache; @@ -351,7 +382,6 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat Preconditions.checkArgument( filesetCatalog != null, String.format("Loaded fileset catalog: %s is null.", catalogIdent)); - // set the thread local audit info Map contextMap = Maps.newHashMap(); contextMap.put( FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE, @@ -374,7 +404,15 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat scheme, str -> { try { - return FileSystem.newInstance(uri, getConf()); + Map maps = getConfigMap(getConf(), uri); + FileSystemProvider provider = FILE_SYSTEM_PROVIDERS.get(scheme); + if (provider == null) { + throw new GravitinoRuntimeException( + "Unsupported file system scheme: %s for %s.", + scheme, GravitinoVirtualFileSystemConfiguration.GVFS_SCHEME); + } + + return provider.getFileSystem(maps); } catch (IOException ioe) { throw new GravitinoRuntimeException( "Exception occurs when create new FileSystem for actual uri: %s, msg: %s", @@ -385,6 +423,23 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat return new FilesetContextPair(new Path(actualFileLocation), fs); } + private Map getConfigMap(Configuration configuration, URI uri) { + Map maps = Maps.newHashMap(); + configuration.forEach( + entry -> { + String key = entry.getKey(); + if (key.startsWith(GRAVITINO_BYPASS_PREFIX)) { + maps.put(key.substring(GRAVITINO_BYPASS_PREFIX.length()), entry.getValue()); + } else if (!key.startsWith("fs.gvfs.")) { + maps.put(key, entry.getValue()); + } + }); + + maps.put(FS_DEFAULT_NAME_KEY, uri.toString()); + + return maps; + } + private String getSubPathFromVirtualPath(NameIdentifier identifier, String virtualPathString) { return virtualPathString.startsWith(GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX) ? virtualPathString.substring( From d82bf76cbf568b5d974d898e121547907339ce7c Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 11 Oct 2024 16:02:49 +0800 Subject: [PATCH 16/89] Update Java doc for FileSystemProvider --- .../gravitino/catalog/hadoop/FileSystemProvider.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java index 1632a161b25..fe1330c7f72 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java @@ -21,18 +21,20 @@ import java.io.IOException; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; public interface FileSystemProvider { /** - * Get the FileSystem instance according to the configuration and the path. + * Get the FileSystem instance according to the configuration map. * - *

Compared to the FileSystem.get method, this method allows the provider to create a - * FileSystem instance with a specific configuration and path and do further initialization if - * needed. + *

Compared to the {@link FileSystem#get(Configuration)} method, this method allows the + * provider to create a FileSystem instance with a specific configuration and do further + * initialization if needed. * *

For example, we can check endpoint configurations for S3AFileSystem, or set the default one. + * We can also set the default file system for HDFS. * * @param config The configuration for the FileSystem instance. * @return The FileSystem instance. From dfdb77269bc92f7b74769eadf7477df132c3fcaf Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 11 Oct 2024 20:54:19 +0800 Subject: [PATCH 17/89] Fix --- .../hadoop/HadoopCatalogOperations.java | 22 +++++++++---------- .../HadoopCatalogPropertiesMetadata.java | 6 ++--- .../hadoop/GravitinoVirtualFileSystem.java | 20 +++++++++++------ ...avitinoVirtualFileSystemConfiguration.java | 14 ++++++++++++ 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 5a7d837acea..5ac0aabdf4d 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -74,13 +74,13 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchemas, FilesetCatalog { - private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; - private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; - private static final String SLASH = "/"; public static final String DEFAULT_FS = "fs.defaultFS"; private static final String LOCAL_FILE_SCHEMA = "file"; public static final String LOCAL_FILE_PATH = "file:///"; + private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; + private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; + private static final String SLASH = "/"; private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogOperations.class); public static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); @@ -123,7 +123,9 @@ public CatalogInfo getCatalogInfo() { } public Configuration getHadoopConf() { - return hadoopConf; + Configuration configuration = new Configuration(); + bypassConfigs.forEach(configuration::set); + return configuration; } public Map getConf() { @@ -140,19 +142,15 @@ public void initialize( // Initialize Hadoop Configuration. this.conf = config; - hadoopConf = new Configuration(); - Map bypassConfigs = + this.bypassConfigs = conf.entrySet().stream() .filter(e -> e.getKey().startsWith(CATALOG_BYPASS_PREFIX)) .collect( Collectors.toMap( e -> e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), Map.Entry::getValue)); - bypassConfigs.forEach(hadoopConf::set); - - this.bypassConfigs = bypassConfigs; - initPluginFileSystem(config); + initFileSystemProviders(config); String catalogLocation = (String) @@ -165,12 +163,12 @@ public void initialize( : Optional.empty(); } - private void initPluginFileSystem(Map config) { + private void initFileSystemProviders(Map config) { String fileSystemProviders = (String) propertiesMetadata .catalogPropertiesMetadata() - .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDER); + .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); if (StringUtils.isNotBlank(fileSystemProviders)) { String[] providers = fileSystemProviders.split(","); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 37d6f9ea99e..398c11e1ee7 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -41,7 +41,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada * *

The value can be 'xxxx.yyy.FileSystemProvider1, xxxx.yyy.FileSystemProvider2'. */ - public static final String FILESYSTEM_PROVIDER = "filesystem.providers"; + public static final String FILESYSTEM_PROVIDERS = "filesystem-providers"; private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() @@ -54,9 +54,9 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada null, false /* hidden */)) .put( - FILESYSTEM_PROVIDER, + FILESYSTEM_PROVIDERS, PropertyEntry.stringOptionalPropertyEntry( - FILESYSTEM_PROVIDER, + FILESYSTEM_PROVIDERS, "The file system provider class name, separated by comma", false /* immutable */, null, diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index 8f4368aa430..34f6e409ce3 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -18,6 +18,9 @@ */ package org.apache.gravitino.filesystem.hadoop; +import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS; +import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.GVFS_CONFIG_PREFIX; + import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Scheduler; @@ -88,9 +91,12 @@ public class GravitinoVirtualFileSystem extends FileSystem { private static final String GRAVITINO_BYPASS_PREFIX = "gravitino.bypass."; static { - // Register the default FileSystemProvider - FILE_SYSTEM_PROVIDERS.put("file", new LocalFileSystemProvider()); - FILE_SYSTEM_PROVIDERS.put("hdfs", new HDFSFileSystemProvider()); + // Register the default local and HDFS FileSystemProvider + FileSystemProvider localFileSystemProvider = new LocalFileSystemProvider(); + FILE_SYSTEM_PROVIDERS.put(localFileSystemProvider.getScheme(), localFileSystemProvider); + + FileSystemProvider hdfsFileSystemProvider = new HDFSFileSystemProvider(); + FILE_SYSTEM_PROVIDERS.put(hdfsFileSystemProvider.getScheme(), hdfsFileSystemProvider); } @Override @@ -136,7 +142,7 @@ public void initialize(URI name, Configuration configuration) throws IOException initializeClient(configuration); - initializePluginFileSystem(configuration); + initializeFileSystemProviders(configuration); this.workingDirectory = new Path(name); this.uri = URI.create(name.getScheme() + "://" + name.getAuthority()); @@ -145,8 +151,8 @@ public void initialize(URI name, Configuration configuration) throws IOException super.initialize(uri, getConf()); } - private void initializePluginFileSystem(Configuration configuration) { - String fileSystemProviders = configuration.get("fs.gvfs.filesystem.providers"); + private void initializeFileSystemProviders(Configuration configuration) { + String fileSystemProviders = configuration.get(FS_FILESYSTEM_PROVIDERS); if (StringUtils.isNotBlank(fileSystemProviders)) { String[] providers = fileSystemProviders.split(","); for (String provider : providers) { @@ -430,7 +436,7 @@ private Map getConfigMap(Configuration configuration, URI uri) { String key = entry.getKey(); if (key.startsWith(GRAVITINO_BYPASS_PREFIX)) { maps.put(key.substring(GRAVITINO_BYPASS_PREFIX.length()), entry.getValue()); - } else if (!key.startsWith("fs.gvfs.")) { + } else if (!key.startsWith(GVFS_CONFIG_PREFIX)) { maps.put(key, entry.getValue()); } }); diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index 8076c02c36a..bf7e069e313 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -22,6 +22,7 @@ class GravitinoVirtualFileSystemConfiguration { public static final String GVFS_FILESET_PREFIX = "gvfs://fileset"; public static final String GVFS_SCHEME = "gvfs"; + public static final String GVFS_CONFIG_PREFIX = "fs.gvfs."; /** The configuration key for the Gravitino server URI. */ public static final String FS_GRAVITINO_SERVER_URI_KEY = "fs.gravitino.server.uri"; @@ -32,6 +33,19 @@ class GravitinoVirtualFileSystemConfiguration { /** The configuration key for the Gravitino client auth type. */ public static final String FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY = "fs.gravitino.client.authType"; + /** + * Full class name of file systems that implement {@link + * org.apache.gravitino.catalog.hadoop.FileSystemProvider}` spilt by a comma. + * + *

This configuration is used to register file system providers to the gvfs file system. For + * example: + * + * 

+   * fs.gvfs.filesystem.providers=org.apache.gravitino.catalog.hadoop.fs.XFileSystemProvider,org.apache.gravitino.catalog.hadoop.fs.YFileSystemProvider
+   *
+ */ + public static final String FS_FILESYSTEM_PROVIDERS = "fs.gvfs.filesystem.providers"; + public static final String SIMPLE_AUTH_TYPE = "simple"; public static final String OAUTH2_AUTH_TYPE = "oauth2"; public static final String KERBEROS_AUTH_TYPE = "kerberos"; From 8708a8a55160c77c9972e5173d21e18d143c8600 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 11 Oct 2024 21:00:12 +0800 Subject: [PATCH 18/89] Fix --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 5ac0aabdf4d..ba6e89b3e3e 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -75,9 +75,9 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchemas, FilesetCatalog { public static final String DEFAULT_FS = "fs.defaultFS"; - private static final String LOCAL_FILE_SCHEMA = "file"; public static final String LOCAL_FILE_PATH = "file:///"; + private static final String LOCAL_FILE_SCHEMA = "file"; private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; private static final String SLASH = "/"; From ba9f8fa8dc8e2411d5c9bc8f793069a02326e167 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 11 Oct 2024 21:22:50 +0800 Subject: [PATCH 19/89] Fix test error. --- .../gravitino/catalog/hadoop/TestHadoopCatalogOperations.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 360d65466cf..2ec61754ec2 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -230,13 +230,13 @@ public void testHadoopCatalogConfiguration() { CatalogInfo catalogInfo = randomCatalogInfo(); ops.initialize(emptyProps, catalogInfo, HADOOP_PROPERTIES_METADATA); - Configuration conf = ops.hadoopConf; + Configuration conf = ops.getHadoopConf(); String value = conf.get("fs.defaultFS"); Assertions.assertEquals("file:///", value); emptyProps.put(CATALOG_BYPASS_PREFIX + "fs.defaultFS", "hdfs://localhost:9000"); ops.initialize(emptyProps, catalogInfo, HADOOP_PROPERTIES_METADATA); - Configuration conf1 = ops.hadoopConf; + Configuration conf1 = ops.getHadoopConf(); String value1 = conf1.get("fs.defaultFS"); Assertions.assertEquals("hdfs://localhost:9000", value1); From dae99f7f1bd871c0715fc325dcea3620e50de2b4 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 11 Oct 2024 21:52:04 +0800 Subject: [PATCH 20/89] Polish. --- .../catalog/hadoop/HadoopCatalogOperations.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index ba6e89b3e3e..b20532cf34b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -791,12 +791,19 @@ static FileSystem getFileSystem(Path path, Map config) throws IO if (path != null) { scheme = path.toUri().getScheme(); if (scheme == null) { - scheme = LOCAL_FILE_SCHEMA; + // If the schema of the path is not set, we need to get the default FS from the + // configuration. + String defaultFS = config.get(DEFAULT_FS); + if (defaultFS == null) { + scheme = LOCAL_FILE_SCHEMA; + } else { + String schemaFromDefaultFS = new Path(defaultFS).toUri().getScheme(); + scheme = schemaFromDefaultFS == null ? LOCAL_FILE_SCHEMA : schemaFromDefaultFS; + } } fsPath = path; } else { - - String defaultFS = config.get(DEFAULT_FS); + String defaultFS = newConfig.get(DEFAULT_FS); if (defaultFS == null) { // Should be the local file system. scheme = LOCAL_FILE_SCHEMA; @@ -812,7 +819,7 @@ static FileSystem getFileSystem(Path path, Map config) throws IO } // For any non-local file system, we need to explicitly set the default FS. - if (!LOCAL_FILE_SCHEMA.equals(scheme)) { + if (!LOCAL_FILE_SCHEMA.equals(scheme) && !newConfig.containsKey(DEFAULT_FS)) { newConfig.put(DEFAULT_FS, fsPath.toString()); } From e22053b25261068ee63e6738b256f59b64edd73a Mon Sep 17 00:00:00 2001 From: yuqi Date: Sat, 12 Oct 2024 11:18:05 +0800 Subject: [PATCH 21/89] Polish --- .../hadoop/HadoopCatalogOperations.java | 37 +++++++++----- .../HadoopCatalogPropertiesMetadata.java | 1 + .../hadoop/{ => fs}/FileSystemProvider.java | 15 ++++-- .../catalog/hadoop/fs/FileSystemUtils.java | 48 +++++++++++++++++++ .../hadoop/fs/HDFSFileSystemProvider.java | 1 - .../hadoop/fs/LocalFileSystemProvider.java | 1 - .../hadoop/GravitinoVirtualFileSystem.java | 24 ++-------- ...avitinoVirtualFileSystemConfiguration.java | 5 +- 8 files changed, 90 insertions(+), 42 deletions(-) rename catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/{ => fs}/FileSystemProvider.java (73%) create mode 100644 catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index b20532cf34b..a0027d8787c 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -44,6 +44,8 @@ import org.apache.gravitino.audit.CallerContext; import org.apache.gravitino.audit.FilesetAuditConstants; import org.apache.gravitino.audit.FilesetDataOperation; +import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; +import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.catalog.hadoop.fs.HDFSFileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.connector.CatalogInfo; @@ -94,6 +96,7 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private Map conf; + // The bypassConfigs are the configurations that are used to initialize the Hadoop Configuration. Map bypassConfigs; private CatalogInfo catalogInfo; @@ -150,7 +153,13 @@ public void initialize( e -> e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), Map.Entry::getValue)); - initFileSystemProviders(config); + String fileSystemProviders = + (String) + propertiesMetadata + .catalogPropertiesMetadata() + .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); + + FileSystemUtils.initFileSystemProviders(fileSystemProviders, FILE_SYSTEM_PROVIDERS); String catalogLocation = (String) @@ -170,18 +179,20 @@ private void initFileSystemProviders(Map config) { .catalogPropertiesMetadata() .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); - if (StringUtils.isNotBlank(fileSystemProviders)) { - String[] providers = fileSystemProviders.split(","); - for (String provider : providers) { - try { - FileSystemProvider fileSystemProvider = - (FileSystemProvider) - Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); - FILE_SYSTEM_PROVIDERS.put(fileSystemProvider.getScheme(), fileSystemProvider); - } catch (Exception e) { - throw new GravitinoRuntimeException( - e, "Failed to initialize file system provider: %s", provider); - } + if (StringUtils.isBlank(fileSystemProviders)) { + return; + } + + String[] providers = fileSystemProviders.split(","); + for (String provider : providers) { + try { + FileSystemProvider fileSystemProvider = + (FileSystemProvider) + Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); + FILE_SYSTEM_PROVIDERS.put(fileSystemProvider.getScheme(), fileSystemProvider); + } catch (Exception e) { + throw new GravitinoRuntimeException( + e, "Failed to initialize file system provider: %s", provider); } } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 398c11e1ee7..58496b492ba 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.gravitino.catalog.hadoop.authentication.AuthenticationConfig; import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; +import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java similarity index 73% rename from catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java rename to catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java index fe1330c7f72..1e5bbf1b80a 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/FileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java @@ -17,13 +17,17 @@ * under the License. */ -package org.apache.gravitino.catalog.hadoop; +package org.apache.gravitino.catalog.hadoop.fs; import java.io.IOException; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +/** + * FileSystemProvider is an interface for providing FileSystem instances. It is used by the + * HadoopCatalog to create FileSystem instances for accessing Hadoop compatible file systems. + */ public interface FileSystemProvider { /** @@ -33,8 +37,9 @@ public interface FileSystemProvider { * provider to create a FileSystem instance with a specific configuration and do further * initialization if needed. * - *

For example, we can check endpoint configurations for S3AFileSystem, or set the default one. - * We can also set the default file system for HDFS. + *

For example: 1. We can check the endpoint value validity for S3AFileSystem then do further + * actions. 2. We can also change some default behavior of the FileSystem initialization process + * 3. More... * * @param config The configuration for the FileSystem instance. * @return The FileSystem instance. @@ -43,8 +48,8 @@ public interface FileSystemProvider { FileSystem getFileSystem(Map config) throws IOException; /** - * Get the scheme of this FileSystem provider. file for LocalFileSystem, hdfs for HDFS, s3a for - * S3AFileSystem, etc. + * Get the scheme of this FileSystem provider. The value is 'file' for LocalFileSystem, 'hdfs' for + * HDFS, 's3a' for S3AFileSystem, etc. * * @return The scheme of this FileSystem provider. */ diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java new file mode 100644 index 00000000000..341ba8fc28f --- /dev/null +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.hadoop.fs; + +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.exceptions.GravitinoRuntimeException; + +public class FileSystemUtils { + + private FileSystemUtils() {} + + public static void initFileSystemProviders( + String fileSystemProviders, Map fileProviders) { + if (StringUtils.isBlank(fileSystemProviders)) { + return; + } + + String[] providers = fileSystemProviders.split(","); + for (String provider : providers) { + try { + FileSystemProvider fileSystemProvider = + (FileSystemProvider) + Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); + fileProviders.put(fileSystemProvider.getScheme(), fileSystemProvider); + } catch (Exception e) { + throw new GravitinoRuntimeException( + e, "Failed to initialize file system provider: %s", provider); + } + } + } +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index 1f74df599a7..a5e52afd040 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.net.URI; import java.util.Map; -import org.apache.gravitino.catalog.hadoop.FileSystemProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 6fe80b78880..fb8f7615201 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.Map; -import org.apache.gravitino.catalog.hadoop.FileSystemProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index 34f6e409ce3..f6d26f9eacf 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -44,7 +44,8 @@ import org.apache.gravitino.audit.FilesetAuditConstants; import org.apache.gravitino.audit.FilesetDataOperation; import org.apache.gravitino.audit.InternalClientType; -import org.apache.gravitino.catalog.hadoop.FileSystemProvider; +import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; +import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.catalog.hadoop.fs.HDFSFileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.client.DefaultOAuth2TokenProvider; @@ -142,7 +143,8 @@ public void initialize(URI name, Configuration configuration) throws IOException initializeClient(configuration); - initializeFileSystemProviders(configuration); + String fileSystemProviders = configuration.get(FS_FILESYSTEM_PROVIDERS); + FileSystemUtils.initFileSystemProviders(fileSystemProviders, FILE_SYSTEM_PROVIDERS); this.workingDirectory = new Path(name); this.uri = URI.create(name.getScheme() + "://" + name.getAuthority()); @@ -151,24 +153,6 @@ public void initialize(URI name, Configuration configuration) throws IOException super.initialize(uri, getConf()); } - private void initializeFileSystemProviders(Configuration configuration) { - String fileSystemProviders = configuration.get(FS_FILESYSTEM_PROVIDERS); - if (StringUtils.isNotBlank(fileSystemProviders)) { - String[] providers = fileSystemProviders.split(","); - for (String provider : providers) { - try { - FileSystemProvider fileSystemProvider = - (FileSystemProvider) - Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); - FILE_SYSTEM_PROVIDERS.put(fileSystemProvider.getScheme(), fileSystemProvider); - } catch (Exception e) { - throw new GravitinoRuntimeException( - e, "Failed to initialize file system provider: %s", provider); - } - } - } - } - @VisibleForTesting Cache internalFileSystemCache() { return internalFileSystemCache; diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index bf7e069e313..1ac91220fa0 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.filesystem.hadoop; +import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; + /** Configuration class for Gravitino Virtual File System. */ class GravitinoVirtualFileSystemConfiguration { public static final String GVFS_FILESET_PREFIX = "gvfs://fileset"; @@ -34,8 +36,7 @@ class GravitinoVirtualFileSystemConfiguration { public static final String FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY = "fs.gravitino.client.authType"; /** - * Full class name of file systems that implement {@link - * org.apache.gravitino.catalog.hadoop.FileSystemProvider}` spilt by a comma. + * Full class name of file systems that implement {@link FileSystemProvider}` spilt by a comma. * *

This configuration is used to register file system providers to the gvfs file system. For * example: From 4fb89e040abf29ca13e78ac98b0404e5df27626f Mon Sep 17 00:00:00 2001 From: yuqi Date: Sat, 12 Oct 2024 11:25:25 +0800 Subject: [PATCH 22/89] Polish --- .../hadoop/HadoopCatalogOperations.java | 25 ------------------- ...avitinoVirtualFileSystemConfiguration.java | 3 ++- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index a0027d8787c..8621464ebae 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -172,31 +172,6 @@ public void initialize( : Optional.empty(); } - private void initFileSystemProviders(Map config) { - String fileSystemProviders = - (String) - propertiesMetadata - .catalogPropertiesMetadata() - .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); - - if (StringUtils.isBlank(fileSystemProviders)) { - return; - } - - String[] providers = fileSystemProviders.split(","); - for (String provider : providers) { - try { - FileSystemProvider fileSystemProvider = - (FileSystemProvider) - Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); - FILE_SYSTEM_PROVIDERS.put(fileSystemProvider.getScheme(), fileSystemProvider); - } catch (Exception e) { - throw new GravitinoRuntimeException( - e, "Failed to initialize file system provider: %s", provider); - } - } - } - @Override public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaException { try { diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index 1ac91220fa0..7a365a98920 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -42,7 +42,8 @@ class GravitinoVirtualFileSystemConfiguration { * example: * * 

-   * fs.gvfs.filesystem.providers=org.apache.gravitino.catalog.hadoop.fs.XFileSystemProvider,org.apache.gravitino.catalog.hadoop.fs.YFileSystemProvider
+   * fs.gvfs.filesystem.providers=org.apache.gravitino.catalog.hadoop.fs.XFileSystemProvider,
+   * org.apache.gravitino.catalog.hadoop.fs.YFileSystemProvider
    *
*/ public static final String FS_FILESYSTEM_PROVIDERS = "fs.gvfs.filesystem.providers"; From e5746c06ce1fd5563d5ac1510f4ee4921f269cc8 Mon Sep 17 00:00:00 2001 From: yuqi Date: Sat, 12 Oct 2024 15:33:38 +0800 Subject: [PATCH 23/89] Rename `AbstractIT` to `BaseIT` --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 8621464ebae..31c4ae1ad36 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -78,13 +78,13 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem public static final String DEFAULT_FS = "fs.defaultFS"; public static final String LOCAL_FILE_PATH = "file:///"; + public static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); private static final String LOCAL_FILE_SCHEMA = "file"; private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; private static final String SLASH = "/"; private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogOperations.class); - public static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); private final EntityStore store; From b2d7bed929ab852c1d0b71c9e844eda3f3960dff Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 19:40:12 +0800 Subject: [PATCH 24/89] Fix --- .../hadoop/HadoopCatalogOperations.java | 120 +++++++++--------- .../HadoopCatalogPropertiesMetadata.java | 16 +++ .../catalog/hadoop/fs/FileSystemUtils.java | 9 +- .../hadoop/fs/HDFSFileSystemProvider.java | 5 +- .../hadoop/fs/LocalFileSystemProvider.java | 10 +- .../hadoop/TestHadoopCatalogOperations.java | 75 +++++++++-- .../tests/integration/test_catalog.py | 4 +- .../tests/integration/test_fileset_catalog.py | 2 +- .../tests/integration/test_schema.py | 2 +- .../hadoop/GravitinoVirtualFileSystem.java | 18 +-- docs/hadoop-catalog.md | 23 ++-- 11 files changed, 180 insertions(+), 104 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 31c4ae1ad36..e10e708df19 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -30,7 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; @@ -46,8 +45,6 @@ import org.apache.gravitino.audit.FilesetDataOperation; import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; -import org.apache.gravitino.catalog.hadoop.fs.HDFSFileSystemProvider; -import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.connector.CatalogInfo; import org.apache.gravitino.connector.CatalogOperations; import org.apache.gravitino.connector.HasPropertyMetadata; @@ -78,7 +75,6 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem public static final String DEFAULT_FS = "fs.defaultFS"; public static final String LOCAL_FILE_PATH = "file:///"; - public static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); private static final String LOCAL_FILE_SCHEMA = "file"; private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; @@ -97,17 +93,11 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private Map conf; // The bypassConfigs are the configurations that are used to initialize the Hadoop Configuration. - Map bypassConfigs; + Map bypassConfigs = Maps.newHashMap(); private CatalogInfo catalogInfo; - static { - FileSystemProvider localFileSystemProvider = new LocalFileSystemProvider(); - FileSystemProvider hdfsFileSystemProvider = new HDFSFileSystemProvider(); - - FILE_SYSTEM_PROVIDERS.put(localFileSystemProvider.getScheme(), localFileSystemProvider); - FILE_SYSTEM_PROVIDERS.put(hdfsFileSystemProvider.getScheme(), hdfsFileSystemProvider); - } + private final Map fileSystemProvidersMap = Maps.newHashMap(); HadoopCatalogOperations(EntityStore store) { this.store = store; @@ -142,24 +132,25 @@ public void initialize( this.propertiesMetadata = propertiesMetadata; this.catalogInfo = info; - // Initialize Hadoop Configuration. this.conf = config; - this.bypassConfigs = - conf.entrySet().stream() - .filter(e -> e.getKey().startsWith(CATALOG_BYPASS_PREFIX)) - .collect( - Collectors.toMap( - e -> e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), - Map.Entry::getValue)); + // conf.entrySet().stream() + // .filter(e -> e.getKey().startsWith(CATALOG_BYPASS_PREFIX)) + // .forEach(e -> bypassConfigs.put(e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), + // e.getValue())); + // + // String defaultFS = (String) propertiesMetadata.catalogPropertiesMetadata() + // .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS); + // if (StringUtils.isNotBlank(defaultFS)) { + // bypassConfigs.put(DEFAULT_FS, defaultFS); + // } String fileSystemProviders = (String) propertiesMetadata .catalogPropertiesMetadata() .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); - - FileSystemUtils.initFileSystemProviders(fileSystemProviders, FILE_SYSTEM_PROVIDERS); + FileSystemUtils.initFileSystemProviders(fileSystemProviders, fileSystemProvidersMap); String catalogLocation = (String) @@ -262,9 +253,9 @@ public Fileset createFileset( try { // formalize the path to avoid path without scheme, uri, authority, etc. - filesetPath = formalizePath(filesetPath, bypassConfigs); + filesetPath = formalizePath(filesetPath, conf); - FileSystem fs = getFileSystem(filesetPath, bypassConfigs); + FileSystem fs = getFileSystem(filesetPath, conf); if (!fs.exists(filesetPath)) { if (!fs.mkdirs(filesetPath)) { throw new RuntimeException( @@ -367,7 +358,7 @@ public boolean dropFileset(NameIdentifier ident) { // For managed fileset, we should delete the related files. if (filesetEntity.filesetType() == Fileset.Type.MANAGED) { - FileSystem fs = getFileSystem(filesetPath, bypassConfigs); + FileSystem fs = getFileSystem(filesetPath, conf); if (fs.exists(filesetPath)) { if (!fs.delete(filesetPath, true)) { LOG.warn("Failed to delete fileset {} location {}", ident, filesetPath); @@ -487,7 +478,7 @@ public Schema createSchema(NameIdentifier ident, String comment, Map properties) { } @VisibleForTesting - static Path formalizePath(Path path, Map configuration) throws IOException { + Path formalizePath(Path path, Map configuration) throws IOException { FileSystem defaultFs = getFileSystem(path, configuration); return path.makeQualified(defaultFs.getUri(), defaultFs.getWorkingDirectory()); } @@ -758,7 +749,7 @@ private boolean hasCallerContext() { private boolean checkSingleFile(Fileset fileset) { try { Path locationPath = new Path(fileset.storageLocation()); - return getFileSystem(locationPath, bypassConfigs).getFileStatus(locationPath).isFile(); + return getFileSystem(locationPath, conf).getFileStatus(locationPath).isFile(); } catch (FileNotFoundException e) { // We should always return false here, same with the logic in `FileSystem.isFile(Path f)`. return false; @@ -770,48 +761,61 @@ private boolean checkSingleFile(Fileset fileset) { } } - static FileSystem getFileSystem(Path path, Map config) throws IOException { - Map newConfig = Maps.newHashMap(config); - String scheme; + FileSystem getFileSystem(Path path, Map config) throws IOException { + // Set by catalog properties 'defaultFS' explicitly. + String defaultFSSetByUsers = + (String) + propertiesMetadata + .catalogPropertiesMetadata() + .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS); + + // Set by properties 'gravitino.bypass.fs.defaultFS'. + String defaultFSfromByPass = config.get(CATALOG_BYPASS_PREFIX + DEFAULT_FS); + String schema; Path fsPath; - if (path != null) { - scheme = path.toUri().getScheme(); - if (scheme == null) { - // If the schema of the path is not set, we need to get the default FS from the - // configuration. - String defaultFS = config.get(DEFAULT_FS); - if (defaultFS == null) { - scheme = LOCAL_FILE_SCHEMA; - } else { - String schemaFromDefaultFS = new Path(defaultFS).toUri().getScheme(); - scheme = schemaFromDefaultFS == null ? LOCAL_FILE_SCHEMA : schemaFromDefaultFS; - } - } + + Map newConfig = Maps.newHashMap(config); + if (path != null && path.toUri().getScheme() != null) { + schema = path.toUri().getScheme(); fsPath = path; } else { - String defaultFS = newConfig.get(DEFAULT_FS); - if (defaultFS == null) { - // Should be the local file system. - scheme = LOCAL_FILE_SCHEMA; - fsPath = new Path(LOCAL_FILE_PATH); + if (defaultFSSetByUsers == null && defaultFSfromByPass == null) { + throw new IllegalArgumentException( + String.format( + "Can't get the schema from the path: %s, and the `defaultFS` and" + + " `gravitino.bypass.fs.defaultFS` is not set.", + path)); + } + + if (defaultFSSetByUsers != null) { + fsPath = new Path(defaultFSSetByUsers); + schema = fsPath.toUri().getScheme(); + if (schema == null) { + throw new IllegalArgumentException( + String.format( + "Can't get the schema from the path: %s, and can't get schema from `defaultFS`.", + path)); + } } else { - fsPath = new Path(defaultFS); - if (fsPath.toUri().getScheme() == null) { - scheme = LOCAL_FILE_SCHEMA; - } else { - scheme = fsPath.toUri().getScheme(); + fsPath = new Path(defaultFSfromByPass); + schema = fsPath.toUri().getScheme(); + if (schema == null) { + throw new IllegalArgumentException( + String.format( + "Can't get the schema from the path: %s, and can't get schema from `gravitino.bypass.fs.defaultFS`.", + path)); } } } // For any non-local file system, we need to explicitly set the default FS. - if (!LOCAL_FILE_SCHEMA.equals(scheme) && !newConfig.containsKey(DEFAULT_FS)) { + if (!newConfig.containsKey(DEFAULT_FS) && !LOCAL_FILE_SCHEMA.equals(schema)) { newConfig.put(DEFAULT_FS, fsPath.toString()); } - FileSystemProvider provider = FILE_SYSTEM_PROVIDERS.get(scheme); + FileSystemProvider provider = fileSystemProvidersMap.get(schema); if (provider == null) { - throw new IllegalArgumentException("Unsupported scheme: " + scheme); + throw new IllegalArgumentException("Unsupported scheme: " + schema); } return provider.getFileSystem(newConfig); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 58496b492ba..64f099dc633 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -44,6 +44,13 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada */ public static final String FILESYSTEM_PROVIDERS = "filesystem-providers"; + /** + * The default file system URI. It is used to create the default file system instance; if not + * specified, the default file system instance will be created with the schema prefix in the file + * path. + */ + public static final String DEFAULT_FS = "defaultFS"; + private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() .put( @@ -62,6 +69,15 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada false /* immutable */, null, false /* hidden */)) + .put( + DEFAULT_FS, + PropertyEntry.stringOptionalPropertyEntry( + DEFAULT_FS, + "Default file system URI, used to create the default file system " + + "instance like hdfs:///, gs://bucket-name", + false /* immutable */, + null, + false /* hidden */)) // The following two are about authentication. .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES) .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 341ba8fc28f..1ea86caca2a 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -27,7 +27,12 @@ public class FileSystemUtils { private FileSystemUtils() {} public static void initFileSystemProviders( - String fileSystemProviders, Map fileProviders) { + String fileSystemProviders, Map fileProvidersMap) { + FileSystemProvider localFileSystemProvider = new LocalFileSystemProvider(); + FileSystemProvider hdfsFileSystemProvider = new HDFSFileSystemProvider(); + fileProvidersMap.put(localFileSystemProvider.getScheme(), localFileSystemProvider); + fileProvidersMap.put(hdfsFileSystemProvider.getScheme(), hdfsFileSystemProvider); + if (StringUtils.isBlank(fileSystemProviders)) { return; } @@ -38,7 +43,7 @@ public static void initFileSystemProviders( FileSystemProvider fileSystemProvider = (FileSystemProvider) Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); - fileProviders.put(fileSystemProvider.getScheme(), fileSystemProvider); + fileProvidersMap.put(fileSystemProvider.getScheme(), fileSystemProvider); } catch (Exception e) { throw new GravitinoRuntimeException( e, "Failed to initialize file system provider: %s", provider); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index a5e52afd040..d2cb63cff18 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -30,7 +30,10 @@ public class HDFSFileSystemProvider implements FileSystemProvider { @Override public FileSystem getFileSystem(Map config) throws IOException { Configuration configuration = new Configuration(); - config.forEach(configuration::set); + config.forEach( + (k, v) -> { + configuration.set(k.replace("gravitino.bypass.", ""), v); + }); String pathString = configuration.get("fs.defaultFS"); if (pathString == null) { diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index fb8f7615201..45d7bd5e9d6 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -18,9 +18,6 @@ */ package org.apache.gravitino.catalog.hadoop.fs; -import static org.apache.gravitino.catalog.hadoop.HadoopCatalogOperations.DEFAULT_FS; -import static org.apache.gravitino.catalog.hadoop.HadoopCatalogOperations.LOCAL_FILE_PATH; - import java.io.IOException; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -31,8 +28,11 @@ public class LocalFileSystemProvider implements FileSystemProvider { @Override public FileSystem getFileSystem(Map config) throws IOException { Configuration configuration = new Configuration(); - config.forEach(configuration::set); - config.put(DEFAULT_FS, LOCAL_FILE_PATH); + config.forEach( + (k, v) -> { + configuration.set(k.replace("gravitino.bypass.", ""), v); + }); + return FileSystem.newInstance(configuration); } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 2ec61754ec2..5e21d577118 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -34,7 +34,6 @@ import static org.apache.gravitino.catalog.hadoop.HadoopCatalog.CATALOG_PROPERTIES_META; import static org.apache.gravitino.catalog.hadoop.HadoopCatalog.FILESET_PROPERTIES_META; import static org.apache.gravitino.catalog.hadoop.HadoopCatalog.SCHEMA_PROPERTIES_META; -import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.when; @@ -50,10 +49,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Config; import org.apache.gravitino.EntityStore; import org.apache.gravitino.EntityStoreFactory; +import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.Schema; @@ -65,6 +66,7 @@ import org.apache.gravitino.connector.CatalogInfo; import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.connector.PropertyEntry; import org.apache.gravitino.exceptions.GravitinoRuntimeException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; @@ -74,6 +76,7 @@ import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.RandomIdGenerator; +import org.apache.gravitino.storage.relational.RelationalEntityStore; import org.apache.gravitino.storage.relational.service.CatalogMetaService; import org.apache.gravitino.storage.relational.service.MetalakeMetaService; import org.apache.gravitino.utils.NameIdentifierUtil; @@ -234,11 +237,11 @@ public void testHadoopCatalogConfiguration() { String value = conf.get("fs.defaultFS"); Assertions.assertEquals("file:///", value); - emptyProps.put(CATALOG_BYPASS_PREFIX + "fs.defaultFS", "hdfs://localhost:9000"); - ops.initialize(emptyProps, catalogInfo, HADOOP_PROPERTIES_METADATA); - Configuration conf1 = ops.getHadoopConf(); - String value1 = conf1.get("fs.defaultFS"); - Assertions.assertEquals("hdfs://localhost:9000", value1); + // emptyProps.put(CATALOG_BYPASS_PREFIX + "fs.defaultFS", "hdfs://localhost:9000"); + // ops.initialize(emptyProps, catalogInfo, HADOOP_PROPERTIES_METADATA); + // Configuration conf1 = ops.getHadoopConf(); + // String value1 = conf1.get("fs.defaultFS"); + // Assertions.assertEquals("hdfs://localhost:9000", value1); Assertions.assertFalse(ops.catalogStorageLocation.isPresent()); @@ -483,6 +486,7 @@ public void testCreateLoadAndDeleteFilesetWithLocations( if (catalogPath != null) { catalogProps.put(HadoopCatalogPropertiesMetadata.LOCATION, catalogPath); } + catalogProps.put(HadoopCatalogPropertiesMetadata.DEFAULT_FS, "file:///"); NameIdentifier schemaIdent = NameIdentifierUtil.ofSchema("m1", "c1", schemaName); try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { @@ -677,7 +681,7 @@ public void testAlterFilesetProperties() throws IOException { } @Test - public void testFormalizePath() throws IOException { + public void testFormalizePath() throws IOException, IllegalAccessException { String[] paths = new String[] {"tmp/catalog", "/tmp/catalog", "file:/tmp/catalog", "file:///tmp/catalog"}; @@ -690,9 +694,55 @@ public void testFormalizePath() throws IOException { "file:/tmp/catalog" }; - for (int i = 0; i < paths.length; i++) { - Path actual = HadoopCatalogOperations.formalizePath(new Path(paths[i]), Maps.newHashMap()); - Assertions.assertEquals(expected[i], actual.toString()); + HasPropertyMetadata hasPropertyMetadata = + new HasPropertyMetadata() { + @Override + public PropertiesMetadata tablePropertiesMetadata() throws UnsupportedOperationException { + return null; + } + + @Override + public PropertiesMetadata catalogPropertiesMetadata() + throws UnsupportedOperationException { + return new PropertiesMetadata() { + @Override + public Map> propertyEntries() { + return new HadoopCatalogPropertiesMetadata().propertyEntries(); + } + }; + } + + @Override + public PropertiesMetadata schemaPropertiesMetadata() + throws UnsupportedOperationException { + return null; + } + + @Override + public PropertiesMetadata filesetPropertiesMetadata() + throws UnsupportedOperationException { + return null; + } + + @Override + public PropertiesMetadata topicPropertiesMetadata() throws UnsupportedOperationException { + return null; + } + }; + + try { + FieldUtils.writeField( + GravitinoEnv.getInstance(), "entityStore", new RelationalEntityStore(), true); + try (HadoopCatalogOperations hadoopCatalogOperations = new HadoopCatalogOperations()) { + Map map = ImmutableMap.of("defaultFS", "file:///"); + hadoopCatalogOperations.initialize(map, null, hasPropertyMetadata); + for (int i = 0; i < paths.length; i++) { + Path actual = hadoopCatalogOperations.formalizePath(new Path(paths[i]), map); + Assertions.assertEquals(expected[i], actual.toString()); + } + } + } finally { + FieldUtils.writeField(GravitinoEnv.getInstance(), "entityStore", null, true); } } @@ -870,6 +920,8 @@ public void testGetFileLocation() throws IOException { when(mockOps.getConf()).thenReturn(Maps.newHashMap()); String subPath = "/test/test.parquet"; when(mockOps.getFileLocation(filesetIdent, subPath)).thenCallRealMethod(); + when(mockOps.getFileSystem(Mockito.any(), Mockito.any())) + .thenReturn(FileSystem.getLocal(new Configuration())); String fileLocation = mockOps.getFileLocation(filesetIdent, subPath); Assertions.assertEquals( String.format("%s%s", mockFileset.storageLocation(), subPath.substring(1)), fileLocation); @@ -1116,6 +1168,8 @@ private Schema createSchema(String name, String comment, String catalogPath, Str props.put(HadoopCatalogPropertiesMetadata.LOCATION, catalogPath); } + props.put(HadoopCatalogPropertiesMetadata.DEFAULT_FS, "file:///"); + try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { ops.initialize(props, randomCatalogInfo("m1", "c1"), HADOOP_PROPERTIES_METADATA); @@ -1144,6 +1198,7 @@ private Fileset createFileset( if (catalogPath != null) { props.put(HadoopCatalogPropertiesMetadata.LOCATION, catalogPath); } + props.put(HadoopCatalogPropertiesMetadata.DEFAULT_FS, "file:///"); try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { ops.initialize(props, randomCatalogInfo("m1", "c1"), HADOOP_PROPERTIES_METADATA); diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 71caafbc206..7ade9e4f735 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -71,7 +71,7 @@ def create_catalog(self, catalog_name) -> Catalog: catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment=self.catalog_comment, - properties={self.catalog_location_prop: "/tmp/test_schema"}, + properties={self.catalog_location_prop: "file:///tmp/test_schema"}, ) def clean_test_data(self): @@ -154,7 +154,7 @@ def test_load_catalog(self): self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual(catalog.comment(), self.catalog_comment) self.assertEqual( - catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} + catalog.properties(), {self.catalog_location_prop: "file:///tmp/test_schema"} ) self.assertEqual(catalog.audit_info().creator(), "anonymous") diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 0e92ec1b090..62c4ad9aebd 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -51,7 +51,7 @@ class TestFilesetCatalog(IntegrationTestEnv): fileset_alter_name: str = fileset_name + "Alter" fileset_comment: str = "fileset_comment" - fileset_location: str = "/tmp/TestFilesetCatalog" + fileset_location: str = "file:///tmp/TestFilesetCatalog" fileset_properties_key1: str = "fileset_properties_key1" fileset_properties_value1: str = "fileset_properties_value1" fileset_properties_key2: str = "fileset_properties_key2" diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index e57e6676b00..17bb41d7b88 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -89,7 +89,7 @@ def init_test_env(self): catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", - properties={self.catalog_location_prop: "/tmp/test_schema"}, + properties={self.catalog_location_prop: "file:///tmp/test_schema"}, ) def clean_test_data(self): diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index f6d26f9eacf..2f58394fab6 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -46,8 +46,6 @@ import org.apache.gravitino.audit.InternalClientType; import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; -import org.apache.gravitino.catalog.hadoop.fs.HDFSFileSystemProvider; -import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.client.DefaultOAuth2TokenProvider; import org.apache.gravitino.client.GravitinoClient; import org.apache.gravitino.client.KerberosTokenProvider; @@ -88,18 +86,9 @@ public class GravitinoVirtualFileSystem extends FileSystem { private static final Pattern IDENTIFIER_PATTERN = Pattern.compile("^(?:gvfs://fileset)?/([^/]+)/([^/]+)/([^/]+)(?>/[^/]+)*/?$"); private static final String SLASH = "/"; - private static final Map FILE_SYSTEM_PROVIDERS = Maps.newHashMap(); + private final Map fileSystemProvidersMap = Maps.newHashMap(); private static final String GRAVITINO_BYPASS_PREFIX = "gravitino.bypass."; - static { - // Register the default local and HDFS FileSystemProvider - FileSystemProvider localFileSystemProvider = new LocalFileSystemProvider(); - FILE_SYSTEM_PROVIDERS.put(localFileSystemProvider.getScheme(), localFileSystemProvider); - - FileSystemProvider hdfsFileSystemProvider = new HDFSFileSystemProvider(); - FILE_SYSTEM_PROVIDERS.put(hdfsFileSystemProvider.getScheme(), hdfsFileSystemProvider); - } - @Override public void initialize(URI name, Configuration configuration) throws IOException { if (!name.toString().startsWith(GravitinoVirtualFileSystemConfiguration.GVFS_FILESET_PREFIX)) { @@ -143,8 +132,9 @@ public void initialize(URI name, Configuration configuration) throws IOException initializeClient(configuration); + // Register the default local and HDFS FileSystemProvider String fileSystemProviders = configuration.get(FS_FILESYSTEM_PROVIDERS); - FileSystemUtils.initFileSystemProviders(fileSystemProviders, FILE_SYSTEM_PROVIDERS); + FileSystemUtils.initFileSystemProviders(fileSystemProviders, fileSystemProvidersMap); this.workingDirectory = new Path(name); this.uri = URI.create(name.getScheme() + "://" + name.getAuthority()); @@ -395,7 +385,7 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat str -> { try { Map maps = getConfigMap(getConf(), uri); - FileSystemProvider provider = FILE_SYSTEM_PROVIDERS.get(scheme); + FileSystemProvider provider = fileSystemProvidersMap.get(scheme); if (provider == null) { throw new GravitinoRuntimeException( "Unsupported file system scheme: %s for %s.", diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index ca552091c7e..dd82d0a42aa 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -25,16 +25,19 @@ Hadoop 3. If there's any compatibility issue, please create an [issue](https://g Besides the [common catalog properties](./gravitino-server-config.md#gravitino-catalog-properties-configuration), the Hadoop catalog has the following properties: -| Property Name | Description | Default Value | Required | Since Version | -|----------------------------------------------------|------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|---------------| -| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | -| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | -| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | - +| Property Name | Description | Default Value | Required | Since Version | +|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|---------------| +| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | +| `filesystem-providers` | The filesystem providers for the Hadoop catalog. Gravitino already support `local file` and `hdfs`, if you want to support other file system, you can implement `FileSystemProvider` and set this value | (none) | No | 0.7.0 | +| `defaultFS` | The default file system of this Hadoop catalog. | (none) | No | 0.7.0 | +| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | +| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | + +For more about `filesystem-providers`, please refer to `HadoopFileSystemProvider` or `LocalFileSystemProvider` in the source code. Furthermore, you also need to place the jar of the file system provider into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory. ### Authentication for Hadoop Catalog From f4041ecc2133b47a7663bbb1972e3f0e48958471 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 20:17:09 +0800 Subject: [PATCH 25/89] Fix python ut error again. --- .../tests/integration/auth/test_auth_common.py | 4 ++-- clients/client-python/tests/integration/test_catalog.py | 7 ++++--- .../tests/integration/test_fileset_catalog.py | 8 +++++--- clients/client-python/tests/integration/test_schema.py | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/clients/client-python/tests/integration/auth/test_auth_common.py b/clients/client-python/tests/integration/auth/test_auth_common.py index a8cdc7c1e2d..592b51ce80d 100644 --- a/clients/client-python/tests/integration/auth/test_auth_common.py +++ b/clients/client-python/tests/integration/auth/test_auth_common.py @@ -48,7 +48,7 @@ class TestCommonAuth: fileset_name: str = "test_client_fileset" fileset_comment: str = "fileset_comment" - fileset_location: str = "/tmp/TestFilesetCatalog" + fileset_location: str = "file:///tmp/TestFilesetCatalog" fileset_properties_key1: str = "fileset_properties_key1" fileset_properties_value1: str = "fileset_properties_value1" fileset_properties_key2: str = "fileset_properties_key2" @@ -115,7 +115,7 @@ def init_test_env(self): catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", - properties={self.catalog_location_prop: "/tmp/test1"}, + properties={self.catalog_location_prop: "file:/tmp/test1"}, ) catalog.as_schemas().create_schema( schema_name=self.schema_name, comment="", properties={} diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 7ade9e4f735..1d9e66a4d49 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -71,7 +71,7 @@ def create_catalog(self, catalog_name) -> Catalog: catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment=self.catalog_comment, - properties={self.catalog_location_prop: "file:///tmp/test_schema"}, + properties={self.catalog_location_prop: "file:/tmp/test_schema"}, ) def clean_test_data(self): @@ -105,7 +105,7 @@ def test_create_catalog(self): catalog = self.create_catalog(self.catalog_name) self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual( - catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} + catalog.properties(), {self.catalog_location_prop: "file:/tmp/test_schema"} ) def test_failed_create_catalog(self): @@ -154,7 +154,8 @@ def test_load_catalog(self): self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual(catalog.comment(), self.catalog_comment) self.assertEqual( - catalog.properties(), {self.catalog_location_prop: "file:///tmp/test_schema"} + catalog.properties(), + {self.catalog_location_prop: "file:/tmp/test_schema"}, ) self.assertEqual(catalog.audit_info().creator(), "anonymous") diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 62c4ad9aebd..ff3fb0417bb 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -145,7 +145,7 @@ def init_test_env(self): catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", - properties={self.catalog_location_prop: "/tmp/test1"}, + properties={self.catalog_location_prop: "file:///tmp/test1"}, ) catalog.as_schemas().create_schema( schema_name=self.schema_name, comment="", properties={} @@ -246,7 +246,7 @@ def test_get_file_location(self): fileset_ident: NameIdentifier = NameIdentifier.of( self.schema_name, "test_get_file_location" ) - fileset_location = "/tmp/test_get_file_location" + fileset_location = "file:/tmp/test_get_file_location" self.create_custom_fileset(fileset_ident, fileset_location) actual_file_location = ( self.gravitino_client.load_catalog(name=self.catalog_name) @@ -254,7 +254,9 @@ def test_get_file_location(self): .get_file_location(fileset_ident, "/test/test.txt") ) - self.assertEqual(actual_file_location, f"file:{fileset_location}/test/test.txt") + self.assertEqual( + actual_file_location, f"file:/tmp/test_get_file_location/test/test.txt" + ) # test rename without sub path should throw an exception caller_context = CallerContext( diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index 17bb41d7b88..5169f400558 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -89,7 +89,7 @@ def init_test_env(self): catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", - properties={self.catalog_location_prop: "file:///tmp/test_schema"}, + properties={self.catalog_location_prop: "file:/tmp/test_schema"}, ) def clean_test_data(self): From 3cfb94fe1b48c477401acdebe1080ae5d28b942f Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 20:39:40 +0800 Subject: [PATCH 26/89] Fix test error again. --- .../catalog/hadoop/HadoopCatalogOperations.java | 12 ------------ .../hadoop/HadoopCatalogPropertiesMetadata.java | 2 +- .../catalog/hadoop/TestHadoopCatalogOperations.java | 2 +- .../test/authorization/AccessControlIT.java | 7 ++++++- .../test/authorization/CheckCurrentUserIT.java | 7 ++++++- .../integration/test/authorization/OwnerIT.java | 7 ++++++- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index e10e708df19..b55db91c392 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -74,7 +74,6 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchemas, FilesetCatalog { public static final String DEFAULT_FS = "fs.defaultFS"; - public static final String LOCAL_FILE_PATH = "file:///"; private static final String LOCAL_FILE_SCHEMA = "file"; private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; @@ -134,17 +133,6 @@ public void initialize( this.conf = config; - // conf.entrySet().stream() - // .filter(e -> e.getKey().startsWith(CATALOG_BYPASS_PREFIX)) - // .forEach(e -> bypassConfigs.put(e.getKey().substring(CATALOG_BYPASS_PREFIX.length()), - // e.getValue())); - // - // String defaultFS = (String) propertiesMetadata.catalogPropertiesMetadata() - // .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS); - // if (StringUtils.isNotBlank(defaultFS)) { - // bypassConfigs.put(DEFAULT_FS, defaultFS); - // } - String fileSystemProviders = (String) propertiesMetadata diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 64f099dc633..5709336cd2b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -49,7 +49,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada * specified, the default file system instance will be created with the schema prefix in the file * path. */ - public static final String DEFAULT_FS = "defaultFS"; + public static final String DEFAULT_FS = "default-filesystem"; private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 5e21d577118..15c81782401 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -734,7 +734,7 @@ public PropertiesMetadata topicPropertiesMetadata() throws UnsupportedOperationE FieldUtils.writeField( GravitinoEnv.getInstance(), "entityStore", new RelationalEntityStore(), true); try (HadoopCatalogOperations hadoopCatalogOperations = new HadoopCatalogOperations()) { - Map map = ImmutableMap.of("defaultFS", "file:///"); + Map map = ImmutableMap.of("default-filesystem", "file:///"); hadoopCatalogOperations.initialize(map, null, hasPropertyMetadata); for (int i = 0; i < paths.length; i++) { Path actual = hadoopCatalogOperations.formalizePath(new Path(paths[i]), map); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java index e62cebcfdbd..7c73aa1e8f0 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.client.integration.test.authorization; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.util.Arrays; @@ -71,7 +72,11 @@ public static void startIntegrationTest() throws Exception { Catalog filesetCatalog = metalake.createCatalog( - "fileset_catalog", Catalog.Type.FILESET, "hadoop", "comment", Collections.emptyMap()); + "fileset_catalog", + Catalog.Type.FILESET, + "hadoop", + "comment", + ImmutableMap.of("default-filesystem", "file:///")); NameIdentifier fileIdent = NameIdentifier.of("fileset_schema", "fileset"); filesetCatalog.asSchemas().createSchema("fileset_schema", "comment", Collections.emptyMap()); filesetCatalog diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java index 2f80a310231..414055297c0 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java @@ -20,6 +20,7 @@ import static org.apache.gravitino.server.GravitinoServer.WEBSERVER_CONF_PREFIX; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.util.Collections; @@ -161,7 +162,11 @@ public void testCreateFileset() { Catalog catalog = metalake.createCatalog( - catalogName, Catalog.Type.FILESET, "hadoop", "comment", Collections.emptyMap()); + catalogName, + Catalog.Type.FILESET, + "hadoop", + "comment", + ImmutableMap.of("default-filesystem", "file:///")); // Test to create a schema with a not-existed user Catalog anotherCatalog = anotherMetalake.loadCatalog(catalogName); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java index ca9d96b8b10..5339bc28711 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.client.integration.test.authorization; +import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.util.Collections; @@ -105,7 +106,11 @@ public void testCreateFileset() { String catalogNameA = RandomNameUtils.genRandomName("catalogA"); Catalog catalog = metalake.createCatalog( - catalogNameA, Catalog.Type.FILESET, "hadoop", "comment", Collections.emptyMap()); + catalogNameA, + Catalog.Type.FILESET, + "hadoop", + "comment", + ImmutableBiMap.of("default-filesystem", "file:///")); NameIdentifier fileIdent = NameIdentifier.of("schema_owner", "fileset_owner"); catalog.asSchemas().createSchema("schema_owner", "comment", Collections.emptyMap()); catalog From 7d1150f438d16b9c94bc9deb19d928e3aa4b5584 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 20:46:33 +0800 Subject: [PATCH 27/89] Fix minor. --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 3 +-- .../catalog/hadoop/TestHadoopCatalogOperations.java | 8 -------- docs/hadoop-catalog.md | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index b55db91c392..d3bde802f9b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -74,7 +74,6 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchemas, FilesetCatalog { public static final String DEFAULT_FS = "fs.defaultFS"; - private static final String LOCAL_FILE_SCHEMA = "file"; private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; @@ -750,7 +749,7 @@ private boolean checkSingleFile(Fileset fileset) { } FileSystem getFileSystem(Path path, Map config) throws IOException { - // Set by catalog properties 'defaultFS' explicitly. + // Set by catalog properties 'default-filesystem' explicitly. String defaultFSSetByUsers = (String) propertiesMetadata diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 15c81782401..7e1f5390025 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -237,14 +237,6 @@ public void testHadoopCatalogConfiguration() { String value = conf.get("fs.defaultFS"); Assertions.assertEquals("file:///", value); - // emptyProps.put(CATALOG_BYPASS_PREFIX + "fs.defaultFS", "hdfs://localhost:9000"); - // ops.initialize(emptyProps, catalogInfo, HADOOP_PROPERTIES_METADATA); - // Configuration conf1 = ops.getHadoopConf(); - // String value1 = conf1.get("fs.defaultFS"); - // Assertions.assertEquals("hdfs://localhost:9000", value1); - - Assertions.assertFalse(ops.catalogStorageLocation.isPresent()); - emptyProps.put(HadoopCatalogPropertiesMetadata.LOCATION, "file:///tmp/catalog"); ops.initialize(emptyProps, catalogInfo, HADOOP_PROPERTIES_METADATA); Assertions.assertTrue(ops.catalogStorageLocation.isPresent()); diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index ef40c32d42d..f50803b7587 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -29,7 +29,7 @@ Besides the [common catalog properties](./gravitino-server-config.md#gravitino-c |----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|---------------| | `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | | `filesystem-providers` | The filesystem providers for the Hadoop catalog. Gravitino already support `local file` and `hdfs`, if you want to support other file system, you can implement `FileSystemProvider` and set this value | (none) | No | 0.7.0 | -| `defaultFS` | The default file system of this Hadoop catalog. | (none) | No | 0.7.0 | +| `default-filesystem` | The default file system of this Hadoop catalog. This configuration is equivalent to Hadoop `fs.defaultFS` | (none) | No | 0.7.0 | | `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | | `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | | `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | From 608081b28909a3a9b751a049ac8b900338ad3522 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 20:56:29 +0800 Subject: [PATCH 28/89] fix --- .../gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java | 4 +++- .../gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java | 4 +++- .../client-python/tests/integration/test_fileset_catalog.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index d2cb63cff18..5642b9d763f 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.catalog.hadoop.fs; +import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; + import java.io.IOException; import java.net.URI; import java.util.Map; @@ -32,7 +34,7 @@ public FileSystem getFileSystem(Map config) throws IOException { Configuration configuration = new Configuration(); config.forEach( (k, v) -> { - configuration.set(k.replace("gravitino.bypass.", ""), v); + configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v); }); String pathString = configuration.get("fs.defaultFS"); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 45d7bd5e9d6..42709940969 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.catalog.hadoop.fs; +import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; + import java.io.IOException; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -30,7 +32,7 @@ public FileSystem getFileSystem(Map config) throws IOException { Configuration configuration = new Configuration(); config.forEach( (k, v) -> { - configuration.set(k.replace("gravitino.bypass.", ""), v); + configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v); }); return FileSystem.newInstance(configuration); diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index ff3fb0417bb..d813aef6509 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -255,7 +255,7 @@ def test_get_file_location(self): ) self.assertEqual( - actual_file_location, f"file:/tmp/test_get_file_location/test/test.txt" + actual_file_location, "file:/tmp/test_get_file_location/test/test.txt" ) # test rename without sub path should throw an exception From 9edfe82dde9c6b7f933a7b42da94076940c93f84 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 21:55:45 +0800 Subject: [PATCH 29/89] Fix --- .../hadoop/HadoopCatalogOperations.java | 66 +++++++------------ .../HadoopCatalogPropertiesMetadata.java | 9 +-- .../hadoop/TestHadoopCatalogOperations.java | 3 - .../test/authorization/AccessControlIT.java | 7 +- .../authorization/CheckCurrentUserIT.java | 7 +- .../test/authorization/OwnerIT.java | 7 +- .../integration/auth/test_auth_common.py | 4 +- .../tests/integration/test_catalog.py | 6 +- .../tests/integration/test_fileset_catalog.py | 8 +-- .../tests/integration/test_schema.py | 2 +- docs/hadoop-catalog.md | 2 +- 11 files changed, 41 insertions(+), 80 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index d3bde802f9b..6df76cf69d4 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -18,8 +18,6 @@ */ package org.apache.gravitino.catalog.hadoop; -import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; @@ -749,62 +747,42 @@ private boolean checkSingleFile(Fileset fileset) { } FileSystem getFileSystem(Path path, Map config) throws IOException { - // Set by catalog properties 'default-filesystem' explicitly. - String defaultFSSetByUsers = + String defaultFilesystemProvider = (String) propertiesMetadata .catalogPropertiesMetadata() .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS); - // Set by properties 'gravitino.bypass.fs.defaultFS'. - String defaultFSfromByPass = config.get(CATALOG_BYPASS_PREFIX + DEFAULT_FS); - String schema; - Path fsPath; - Map newConfig = Maps.newHashMap(config); - if (path != null && path.toUri().getScheme() != null) { - schema = path.toUri().getScheme(); - fsPath = path; - } else { - if (defaultFSSetByUsers == null && defaultFSfromByPass == null) { - throw new IllegalArgumentException( - String.format( - "Can't get the schema from the path: %s, and the `defaultFS` and" - + " `gravitino.bypass.fs.defaultFS` is not set.", - path)); - } - - if (defaultFSSetByUsers != null) { - fsPath = new Path(defaultFSSetByUsers); - schema = fsPath.toUri().getScheme(); - if (schema == null) { - throw new IllegalArgumentException( - String.format( - "Can't get the schema from the path: %s, and can't get schema from `defaultFS`.", - path)); - } + if (path == null) { + if (defaultFilesystemProvider != null) { + return getByFileSystemByScheme(defaultFilesystemProvider, newConfig); } else { - fsPath = new Path(defaultFSfromByPass); - schema = fsPath.toUri().getScheme(); - if (schema == null) { - throw new IllegalArgumentException( - String.format( - "Can't get the schema from the path: %s, and can't get schema from `gravitino.bypass.fs.defaultFS`.", - path)); - } + LOG.warn("The path and default filesystem provider are both null, using local file system"); + return getByFileSystemByScheme(LOCAL_FILE_SCHEMA, newConfig); } } - // For any non-local file system, we need to explicitly set the default FS. - if (!newConfig.containsKey(DEFAULT_FS) && !LOCAL_FILE_SCHEMA.equals(schema)) { - newConfig.put(DEFAULT_FS, fsPath.toString()); + // Path is not null; + if (path.toUri().getScheme() == null) { + LOG.warn( + "Can't get schema from path: {} and default filesystem provider are both null, using" + + " local file system", + path); + return getByFileSystemByScheme(LOCAL_FILE_SCHEMA, newConfig); + } else { + newConfig.put(DEFAULT_FS, path.toUri().toString()); + return getByFileSystemByScheme(path.toUri().getScheme(), newConfig); } + } - FileSystemProvider provider = fileSystemProvidersMap.get(schema); + private FileSystem getByFileSystemByScheme(String scheme, Map config) + throws IOException { + FileSystemProvider provider = fileSystemProvidersMap.get(scheme); if (provider == null) { - throw new IllegalArgumentException("Unsupported scheme: " + schema); + throw new IllegalArgumentException("Unsupported scheme: " + scheme); } - return provider.getFileSystem(newConfig); + return provider.getFileSystem(config); } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 5709336cd2b..80883062fef 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -45,11 +45,12 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada public static final String FILESYSTEM_PROVIDERS = "filesystem-providers"; /** - * The default file system URI. It is used to create the default file system instance; if not - * specified, the default file system instance will be created with the schema prefix in the file - * path. + * The default file system provider. It is used to create the default file system instance; if not + * specified, file system instance will be created with the schema prefix in the file path like + * 'file:/tmp/'. If there is no schema prefix, the default file system provider will be local file + * system. */ - public static final String DEFAULT_FS = "default-filesystem"; + public static final String DEFAULT_FS = "default-filesystem-provider"; private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 7e1f5390025..53ef0f7cfae 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -478,7 +478,6 @@ public void testCreateLoadAndDeleteFilesetWithLocations( if (catalogPath != null) { catalogProps.put(HadoopCatalogPropertiesMetadata.LOCATION, catalogPath); } - catalogProps.put(HadoopCatalogPropertiesMetadata.DEFAULT_FS, "file:///"); NameIdentifier schemaIdent = NameIdentifierUtil.ofSchema("m1", "c1", schemaName); try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { @@ -1160,8 +1159,6 @@ private Schema createSchema(String name, String comment, String catalogPath, Str props.put(HadoopCatalogPropertiesMetadata.LOCATION, catalogPath); } - props.put(HadoopCatalogPropertiesMetadata.DEFAULT_FS, "file:///"); - try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { ops.initialize(props, randomCatalogInfo("m1", "c1"), HADOOP_PROPERTIES_METADATA); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java index 7c73aa1e8f0..e62cebcfdbd 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.client.integration.test.authorization; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.util.Arrays; @@ -72,11 +71,7 @@ public static void startIntegrationTest() throws Exception { Catalog filesetCatalog = metalake.createCatalog( - "fileset_catalog", - Catalog.Type.FILESET, - "hadoop", - "comment", - ImmutableMap.of("default-filesystem", "file:///")); + "fileset_catalog", Catalog.Type.FILESET, "hadoop", "comment", Collections.emptyMap()); NameIdentifier fileIdent = NameIdentifier.of("fileset_schema", "fileset"); filesetCatalog.asSchemas().createSchema("fileset_schema", "comment", Collections.emptyMap()); filesetCatalog diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java index 414055297c0..2f80a310231 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java @@ -20,7 +20,6 @@ import static org.apache.gravitino.server.GravitinoServer.WEBSERVER_CONF_PREFIX; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.util.Collections; @@ -162,11 +161,7 @@ public void testCreateFileset() { Catalog catalog = metalake.createCatalog( - catalogName, - Catalog.Type.FILESET, - "hadoop", - "comment", - ImmutableMap.of("default-filesystem", "file:///")); + catalogName, Catalog.Type.FILESET, "hadoop", "comment", Collections.emptyMap()); // Test to create a schema with a not-existed user Catalog anotherCatalog = anotherMetalake.loadCatalog(catalogName); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java index 5339bc28711..ca9d96b8b10 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.client.integration.test.authorization; -import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.util.Collections; @@ -106,11 +105,7 @@ public void testCreateFileset() { String catalogNameA = RandomNameUtils.genRandomName("catalogA"); Catalog catalog = metalake.createCatalog( - catalogNameA, - Catalog.Type.FILESET, - "hadoop", - "comment", - ImmutableBiMap.of("default-filesystem", "file:///")); + catalogNameA, Catalog.Type.FILESET, "hadoop", "comment", Collections.emptyMap()); NameIdentifier fileIdent = NameIdentifier.of("schema_owner", "fileset_owner"); catalog.asSchemas().createSchema("schema_owner", "comment", Collections.emptyMap()); catalog diff --git a/clients/client-python/tests/integration/auth/test_auth_common.py b/clients/client-python/tests/integration/auth/test_auth_common.py index 592b51ce80d..a8cdc7c1e2d 100644 --- a/clients/client-python/tests/integration/auth/test_auth_common.py +++ b/clients/client-python/tests/integration/auth/test_auth_common.py @@ -48,7 +48,7 @@ class TestCommonAuth: fileset_name: str = "test_client_fileset" fileset_comment: str = "fileset_comment" - fileset_location: str = "file:///tmp/TestFilesetCatalog" + fileset_location: str = "/tmp/TestFilesetCatalog" fileset_properties_key1: str = "fileset_properties_key1" fileset_properties_value1: str = "fileset_properties_value1" fileset_properties_key2: str = "fileset_properties_key2" @@ -115,7 +115,7 @@ def init_test_env(self): catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", - properties={self.catalog_location_prop: "file:/tmp/test1"}, + properties={self.catalog_location_prop: "/tmp/test1"}, ) catalog.as_schemas().create_schema( schema_name=self.schema_name, comment="", properties={} diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 1d9e66a4d49..0403b41757e 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -71,7 +71,7 @@ def create_catalog(self, catalog_name) -> Catalog: catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment=self.catalog_comment, - properties={self.catalog_location_prop: "file:/tmp/test_schema"}, + properties={self.catalog_location_prop: "/tmp/test_schema"}, ) def clean_test_data(self): @@ -105,7 +105,7 @@ def test_create_catalog(self): catalog = self.create_catalog(self.catalog_name) self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual( - catalog.properties(), {self.catalog_location_prop: "file:/tmp/test_schema"} + catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} ) def test_failed_create_catalog(self): @@ -155,7 +155,7 @@ def test_load_catalog(self): self.assertEqual(catalog.comment(), self.catalog_comment) self.assertEqual( catalog.properties(), - {self.catalog_location_prop: "file:/tmp/test_schema"}, + {self.catalog_location_prop: "/tmp/test_schema"}, ) self.assertEqual(catalog.audit_info().creator(), "anonymous") diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index d813aef6509..f5fd654fe57 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -51,7 +51,7 @@ class TestFilesetCatalog(IntegrationTestEnv): fileset_alter_name: str = fileset_name + "Alter" fileset_comment: str = "fileset_comment" - fileset_location: str = "file:///tmp/TestFilesetCatalog" + fileset_location: str = "/tmp/TestFilesetCatalog" fileset_properties_key1: str = "fileset_properties_key1" fileset_properties_value1: str = "fileset_properties_value1" fileset_properties_key2: str = "fileset_properties_key2" @@ -145,7 +145,7 @@ def init_test_env(self): catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", - properties={self.catalog_location_prop: "file:///tmp/test1"}, + properties={self.catalog_location_prop: "/tmp/test1"}, ) catalog.as_schemas().create_schema( schema_name=self.schema_name, comment="", properties={} @@ -246,7 +246,7 @@ def test_get_file_location(self): fileset_ident: NameIdentifier = NameIdentifier.of( self.schema_name, "test_get_file_location" ) - fileset_location = "file:/tmp/test_get_file_location" + fileset_location = "/tmp/test_get_file_location" self.create_custom_fileset(fileset_ident, fileset_location) actual_file_location = ( self.gravitino_client.load_catalog(name=self.catalog_name) @@ -255,7 +255,7 @@ def test_get_file_location(self): ) self.assertEqual( - actual_file_location, "file:/tmp/test_get_file_location/test/test.txt" + actual_file_location, "/tmp/test_get_file_location/test/test.txt" ) # test rename without sub path should throw an exception diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index 5169f400558..e57e6676b00 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -89,7 +89,7 @@ def init_test_env(self): catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", - properties={self.catalog_location_prop: "file:/tmp/test_schema"}, + properties={self.catalog_location_prop: "/tmp/test_schema"}, ) def clean_test_data(self): diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index f50803b7587..fba90ecb70d 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -29,7 +29,7 @@ Besides the [common catalog properties](./gravitino-server-config.md#gravitino-c |----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|---------------| | `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | | `filesystem-providers` | The filesystem providers for the Hadoop catalog. Gravitino already support `local file` and `hdfs`, if you want to support other file system, you can implement `FileSystemProvider` and set this value | (none) | No | 0.7.0 | -| `default-filesystem` | The default file system of this Hadoop catalog. This configuration is equivalent to Hadoop `fs.defaultFS` | (none) | No | 0.7.0 | +| `default-filesystem-provider` | The default file system of this Hadoop catalog. The value of this can be 'file', 'hdfs' currently. | (none) | No | 0.7.0 | | `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | | `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | | `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | From 3079bf061f765b0934783a75486178298872e1da Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 22:01:12 +0800 Subject: [PATCH 30/89] Fix --- .../gravitino/catalog/hadoop/TestHadoopCatalogOperations.java | 1 - clients/client-python/tests/integration/test_catalog.py | 3 +-- .../client-python/tests/integration/test_fileset_catalog.py | 4 +--- docs/hadoop-catalog.md | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 53ef0f7cfae..f40826c60de 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -1187,7 +1187,6 @@ private Fileset createFileset( if (catalogPath != null) { props.put(HadoopCatalogPropertiesMetadata.LOCATION, catalogPath); } - props.put(HadoopCatalogPropertiesMetadata.DEFAULT_FS, "file:///"); try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { ops.initialize(props, randomCatalogInfo("m1", "c1"), HADOOP_PROPERTIES_METADATA); diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 0403b41757e..71caafbc206 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -154,8 +154,7 @@ def test_load_catalog(self): self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual(catalog.comment(), self.catalog_comment) self.assertEqual( - catalog.properties(), - {self.catalog_location_prop: "/tmp/test_schema"}, + catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} ) self.assertEqual(catalog.audit_info().creator(), "anonymous") diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index f5fd654fe57..0e92ec1b090 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -254,9 +254,7 @@ def test_get_file_location(self): .get_file_location(fileset_ident, "/test/test.txt") ) - self.assertEqual( - actual_file_location, "/tmp/test_get_file_location/test/test.txt" - ) + self.assertEqual(actual_file_location, f"file:{fileset_location}/test/test.txt") # test rename without sub path should throw an exception caller_context = CallerContext( diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index fba90ecb70d..698cb187e45 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -29,7 +29,7 @@ Besides the [common catalog properties](./gravitino-server-config.md#gravitino-c |----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|---------------| | `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | | `filesystem-providers` | The filesystem providers for the Hadoop catalog. Gravitino already support `local file` and `hdfs`, if you want to support other file system, you can implement `FileSystemProvider` and set this value | (none) | No | 0.7.0 | -| `default-filesystem-provider` | The default file system of this Hadoop catalog. The value of this can be 'file', 'hdfs' currently. | (none) | No | 0.7.0 | +| `default-filesystem-provider` | The default file system provider this Hadoop catalog. The value of this can be 'file', 'hdfs' currently, more information please refer to `filesystem-providers` | (none) | No | 0.7.0 | | `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | | `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | | `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | From da49e602731c04ab71c92ec691d8fac594edce0a Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 22:04:05 +0800 Subject: [PATCH 31/89] Fix --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 5 +---- .../catalog/hadoop/TestHadoopCatalogOperations.java | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 6df76cf69d4..b68b755eb62 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -88,9 +88,6 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private Map conf; - // The bypassConfigs are the configurations that are used to initialize the Hadoop Configuration. - Map bypassConfigs = Maps.newHashMap(); - private CatalogInfo catalogInfo; private final Map fileSystemProvidersMap = Maps.newHashMap(); @@ -113,7 +110,7 @@ public CatalogInfo getCatalogInfo() { public Configuration getHadoopConf() { Configuration configuration = new Configuration(); - bypassConfigs.forEach(configuration::set); + conf.forEach(configuration::set); return configuration; } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index f40826c60de..2b89180a8d1 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -906,7 +906,6 @@ public void testGetFileLocation() throws IOException { try (HadoopCatalogOperations mockOps = Mockito.mock(HadoopCatalogOperations.class)) { mockOps.hadoopConf = new Configuration(); - mockOps.bypassConfigs = Maps.newHashMap(); when(mockOps.loadFileset(filesetIdent)).thenReturn(mockFileset); when(mockOps.getConf()).thenReturn(Maps.newHashMap()); String subPath = "/test/test.parquet"; From b621d8960ab484672a66c7abada44bc80f8d9bdb Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 14 Oct 2024 22:07:29 +0800 Subject: [PATCH 32/89] Fix --- .../catalog/hadoop/HadoopCatalogOperations.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index b68b755eb62..c4259a9a244 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -753,10 +753,10 @@ FileSystem getFileSystem(Path path, Map config) throws IOExcepti Map newConfig = Maps.newHashMap(config); if (path == null) { if (defaultFilesystemProvider != null) { - return getByFileSystemByScheme(defaultFilesystemProvider, newConfig); + return getFileSystemByScheme(defaultFilesystemProvider, newConfig); } else { LOG.warn("The path and default filesystem provider are both null, using local file system"); - return getByFileSystemByScheme(LOCAL_FILE_SCHEMA, newConfig); + return getFileSystemByScheme(LOCAL_FILE_SCHEMA, newConfig); } } @@ -766,14 +766,14 @@ FileSystem getFileSystem(Path path, Map config) throws IOExcepti "Can't get schema from path: {} and default filesystem provider are both null, using" + " local file system", path); - return getByFileSystemByScheme(LOCAL_FILE_SCHEMA, newConfig); + return getFileSystemByScheme(LOCAL_FILE_SCHEMA, newConfig); } else { newConfig.put(DEFAULT_FS, path.toUri().toString()); - return getByFileSystemByScheme(path.toUri().getScheme(), newConfig); + return getFileSystemByScheme(path.toUri().getScheme(), newConfig); } } - private FileSystem getByFileSystemByScheme(String scheme, Map config) + private FileSystem getFileSystemByScheme(String scheme, Map config) throws IOException { FileSystemProvider provider = fileSystemProvidersMap.get(scheme); if (provider == null) { From 9d5b8dc10b9475db6bc006b2211648e0e7c9be8d Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 11:13:35 +0800 Subject: [PATCH 33/89] rebase issue_5019 --- .../fileset/gcs/GCSFileSystemProvider.java | 8 +++++-- .../integration/test/HadoopGCPCatalogIT.java | 21 ++++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java index 0bbd772f23c..daa286535b1 100644 --- a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java +++ b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.Map; -import org.apache.gravitino.catalog.hadoop.FileSystemProvider; +import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -28,7 +28,11 @@ public class GCSFileSystemProvider implements FileSystemProvider { @Override public FileSystem getFileSystem(Map config) throws IOException { Configuration configuration = new Configuration(); - config.forEach(configuration::set); + config.forEach( + (k, v) -> { + configuration.set(k.replace("gravitino.bypass.", ""), v); + }); + return FileSystem.get(configuration); } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java index 13d944f9d1b..ba1fdf8ac96 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.catalog.hadoop.integration.test; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; + import com.google.common.collect.Maps; import java.io.IOException; import java.net.URI; @@ -39,6 +41,9 @@ + "please change the configuration(YOUR_KEY_FILE, YOUR_BUCKET) and enable this test.") public class HadoopGCPCatalogIT extends HadoopCatalogIT { + public static final String BUCKET_NAME = "YOUR_BUCKET"; + public static final String SERVICE_ACCOUNT_FILE = "YOUR_KEY_FILE"; + @BeforeAll public void setup() throws IOException { metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); @@ -50,9 +55,9 @@ public void setup() throws IOException { conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); conf.set("fs.gs.auth.service.account.enable", "true"); - conf.set("fs.gs.auth.service.account.json.keyfile", "YOUR_KEY_FILE"); - conf.set("fs.defaultFS", "gs:///"); - fileSystem = FileSystem.get(URI.create("gs://YOUR_BUCKET"), conf); + conf.set("fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); + conf.set("fs.defaultFS", "gs:///" + BUCKET_NAME); + fileSystem = FileSystem.get(URI.create(String.format("gs://%s", BUCKET_NAME)), conf); createMetalake(); createCatalog(); @@ -63,7 +68,9 @@ protected String defaultBaseLocation() { if (defaultBaseLocation == null) { try { Path bucket = - new Path("gs://YOUR_BUCKET/" + GravitinoITUtils.genRandomName("CatalogFilesetIT")); + new Path( + String.format( + "gs://%s/%s", BUCKET_NAME, GravitinoITUtils.genRandomName("CatalogFilesetIT"))); if (!fileSystem.exists(bucket)) { fileSystem.mkdirs(bucket); } @@ -81,9 +88,9 @@ protected void createCatalog() { Map map = Maps.newHashMap(); map.put("gravitino.bypass.fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); map.put("gravitino.bypass.fs.gs.auth.service.account.enable", "true"); - map.put("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", "YOUR_KEY_FILE"); - map.put("gravitino.bypass.fs.defaultFS", "gs:///"); - map.put("filesystem.providers", "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider"); + map.put("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); + map.put("gravitino.bypass.fs.defaultFS", "gs://" + BUCKET_NAME); + map.put(FILESYSTEM_PROVIDERS, "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider"); metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map); From e58f9a0c887932c272f3c6be81235f450d0a3306 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 11:18:33 +0800 Subject: [PATCH 34/89] Fix --- .../catalog/hadoop/HadoopCatalogOperations.java | 2 +- .../hadoop/HadoopCatalogPropertiesMetadata.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index c4259a9a244..c0b4270aa80 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -748,7 +748,7 @@ FileSystem getFileSystem(Path path, Map config) throws IOExcepti (String) propertiesMetadata .catalogPropertiesMetadata() - .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS); + .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS_PROVIDER); Map newConfig = Maps.newHashMap(config); if (path == null) { diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 80883062fef..1a2cb2f11b2 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -50,7 +50,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada * 'file:/tmp/'. If there is no schema prefix, the default file system provider will be local file * system. */ - public static final String DEFAULT_FS = "default-filesystem-provider"; + public static final String DEFAULT_FS_PROVIDER = "default-filesystem-provider"; private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() @@ -71,11 +71,11 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada null, false /* hidden */)) .put( - DEFAULT_FS, + DEFAULT_FS_PROVIDER, PropertyEntry.stringOptionalPropertyEntry( - DEFAULT_FS, - "Default file system URI, used to create the default file system " - + "instance like hdfs:///, gs://bucket-name", + DEFAULT_FS_PROVIDER, + "Default file system provider, used to create the default file system " + + "candidate value is 'local' or 'hdfs'", false /* immutable */, null, false /* hidden */)) From c521daf4a94e4650dd386a0fd4e0b0a92c5f7b9f Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 14:00:49 +0800 Subject: [PATCH 35/89] resolve comments --- .../hadoop/HadoopCatalogOperations.java | 25 +++++++++++-------- .../hadoop/fs/LocalFileSystemProvider.java | 3 ++- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index c0b4270aa80..9552e68e5a9 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -72,7 +72,7 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchemas, FilesetCatalog { public static final String DEFAULT_FS = "fs.defaultFS"; - private static final String LOCAL_FILE_SCHEMA = "file"; + private static final String LOCAL_FILE_SCHEME = "file"; private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; private static final String SLASH = "/"; @@ -92,6 +92,8 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private final Map fileSystemProvidersMap = Maps.newHashMap(); + private String defaultFilesystemProvider; + HadoopCatalogOperations(EntityStore store) { this.store = store; } @@ -134,6 +136,12 @@ public void initialize( .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); FileSystemUtils.initFileSystemProviders(fileSystemProviders, fileSystemProvidersMap); + this.defaultFilesystemProvider = + (String) + propertiesMetadata + .catalogPropertiesMetadata() + .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS_PROVIDER); + String catalogLocation = (String) propertiesMetadata @@ -744,29 +752,26 @@ private boolean checkSingleFile(Fileset fileset) { } FileSystem getFileSystem(Path path, Map config) throws IOException { - String defaultFilesystemProvider = - (String) - propertiesMetadata - .catalogPropertiesMetadata() - .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS_PROVIDER); - Map newConfig = Maps.newHashMap(config); if (path == null) { if (defaultFilesystemProvider != null) { return getFileSystemByScheme(defaultFilesystemProvider, newConfig); } else { LOG.warn("The path and default filesystem provider are both null, using local file system"); - return getFileSystemByScheme(LOCAL_FILE_SCHEMA, newConfig); + return getFileSystemByScheme(LOCAL_FILE_SCHEME, newConfig); } } // Path is not null; if (path.toUri().getScheme() == null) { + if (defaultFilesystemProvider != null) { + return getFileSystemByScheme(defaultFilesystemProvider, newConfig); + } LOG.warn( - "Can't get schema from path: {} and default filesystem provider are both null, using" + "Can't get schema from path: {} and default filesystem provider is null, using" + " local file system", path); - return getFileSystemByScheme(LOCAL_FILE_SCHEMA, newConfig); + return getFileSystemByScheme(LOCAL_FILE_SCHEME, newConfig); } else { newConfig.put(DEFAULT_FS, path.toUri().toString()); return getFileSystemByScheme(path.toUri().getScheme(), newConfig); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 42709940969..32d319048d9 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; public class LocalFileSystemProvider implements FileSystemProvider { @@ -35,7 +36,7 @@ public FileSystem getFileSystem(Map config) throws IOException { configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v); }); - return FileSystem.newInstance(configuration); + return LocalFileSystem.getLocal(configuration); } @Override From 46e996aaccff63a65dec8034e6f9edd7b3032e77 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 14:53:41 +0800 Subject: [PATCH 36/89] Resolve comments again. --- .../hadoop/HadoopCatalogOperations.java | 26 ++++-------- .../catalog/hadoop/fs/FileSystemProvider.java | 8 +++- .../hadoop/fs/HDFSFileSystemProvider.java | 42 +++---------------- .../hadoop/fs/LocalFileSystemProvider.java | 5 ++- .../hadoop/GravitinoVirtualFileSystem.java | 12 +++--- 5 files changed, 28 insertions(+), 65 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 9552e68e5a9..ba843887238 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -70,8 +70,6 @@ import org.slf4j.LoggerFactory; public class HadoopCatalogOperations implements CatalogOperations, SupportsSchemas, FilesetCatalog { - - public static final String DEFAULT_FS = "fs.defaultFS"; private static final String LOCAL_FILE_SCHEME = "file"; private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; @@ -752,39 +750,33 @@ private boolean checkSingleFile(Fileset fileset) { } FileSystem getFileSystem(Path path, Map config) throws IOException { - Map newConfig = Maps.newHashMap(config); if (path == null) { - if (defaultFilesystemProvider != null) { - return getFileSystemByScheme(defaultFilesystemProvider, newConfig); - } else { - LOG.warn("The path and default filesystem provider are both null, using local file system"); - return getFileSystemByScheme(LOCAL_FILE_SCHEME, newConfig); - } + throw new IllegalArgumentException("Path should not be null"); } - // Path is not null; + // Can't get the scheme from the path like '/path/to/file', use the default filesystem provider. if (path.toUri().getScheme() == null) { if (defaultFilesystemProvider != null) { - return getFileSystemByScheme(defaultFilesystemProvider, newConfig); + return getFileSystemByScheme(defaultFilesystemProvider, config, path); } + LOG.warn( "Can't get schema from path: {} and default filesystem provider is null, using" + " local file system", path); - return getFileSystemByScheme(LOCAL_FILE_SCHEME, newConfig); - } else { - newConfig.put(DEFAULT_FS, path.toUri().toString()); - return getFileSystemByScheme(path.toUri().getScheme(), newConfig); + return getFileSystemByScheme(LOCAL_FILE_SCHEME, config, path); } + + return getFileSystemByScheme(path.toUri().getScheme(), config, path); } - private FileSystem getFileSystemByScheme(String scheme, Map config) + private FileSystem getFileSystemByScheme(String scheme, Map config, Path path) throws IOException { FileSystemProvider provider = fileSystemProvidersMap.get(scheme); if (provider == null) { throw new IllegalArgumentException("Unsupported scheme: " + scheme); } - return provider.getFileSystem(config); + return provider.getFileSystem(path, config); } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java index 1e5bbf1b80a..bbfa60571af 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java @@ -21,8 +21,10 @@ import java.io.IOException; import java.util.Map; +import javax.annotation.Nonnull; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; /** * FileSystemProvider is an interface for providing FileSystem instances. It is used by the @@ -31,7 +33,7 @@ public interface FileSystemProvider { /** - * Get the FileSystem instance according to the configuration map. + * Get the FileSystem instance according to the configuration map and file path. * *

Compared to the {@link FileSystem#get(Configuration)} method, this method allows the * provider to create a FileSystem instance with a specific configuration and do further @@ -42,10 +44,12 @@ public interface FileSystemProvider { * 3. More... * * @param config The configuration for the FileSystem instance. + * @param path The path to the file system. * @return The FileSystem instance. * @throws IOException If the FileSystem instance cannot be created. */ - FileSystem getFileSystem(Map config) throws IOException; + FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map config) + throws IOException; /** * Get the scheme of this FileSystem provider. The value is 'file' for LocalFileSystem, 'hdfs' for diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index 5642b9d763f..af679d19a79 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -21,56 +21,24 @@ import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; import java.io.IOException; -import java.net.URI; import java.util.Map; +import javax.annotation.Nonnull; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; public class HDFSFileSystemProvider implements FileSystemProvider { @Override - public FileSystem getFileSystem(Map config) throws IOException { + public FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map config) + throws IOException { Configuration configuration = new Configuration(); config.forEach( (k, v) -> { configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v); }); - - String pathString = configuration.get("fs.defaultFS"); - if (pathString == null) { - throw new IllegalArgumentException("The path should be specified."); - } - - URI uri = new Path(pathString).toUri(); - if (uri.getScheme() != null && !uri.getScheme().equals("hdfs")) { - throw new IllegalArgumentException("The path should be a HDFS path."); - } - - // Should we call DistributedFileSystem to create file system instance explicitly? If we - // explicitly create a HDFS file system here, we can't reuse the file system cache in the - // FileSystem class. - String impl = configuration.get("fs.hdfs.impl"); - if (impl == null) { - configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); - } else { - if (!impl.equals("org.apache.hadoop.hdfs.DistributedFileSystem")) { - throw new IllegalArgumentException( - "The HDFS file system implementation class should be 'org.apache.hadoop.hdfs.DistributedFileSystem'."); - } - } - - try { - if (HDFSFileSystemProvider.class.getClassLoader().loadClass(configuration.get("fs.hdfs.impl")) - == null) { - throw new IllegalArgumentException( - "The HDFS file system implementation class is not found."); - } - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("The HDFS file system implementation class is not found."); - } - - return FileSystem.newInstance(uri, configuration); + return DistributedFileSystem.newInstance(path.toUri(), configuration); } @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 32d319048d9..29ded3782af 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -25,18 +25,19 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; public class LocalFileSystemProvider implements FileSystemProvider { @Override - public FileSystem getFileSystem(Map config) throws IOException { + public FileSystem getFileSystem(Path path, Map config) throws IOException { Configuration configuration = new Configuration(); config.forEach( (k, v) -> { configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v); }); - return LocalFileSystem.getLocal(configuration); + return LocalFileSystem.newInstance(path.toUri(), configuration); } @Override diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index 2f58394fab6..d4e3722c755 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -374,7 +374,8 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat filesetCatalog.getFileLocation( NameIdentifier.of(identifier.namespace().level(2), identifier.name()), subPath); - URI uri = new Path(actualFileLocation).toUri(); + Path filePath = new Path(actualFileLocation); + URI uri = filePath.toUri(); // we cache the fs for the same scheme, so we can reuse it String scheme = uri.getScheme(); Preconditions.checkArgument( @@ -384,15 +385,14 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat scheme, str -> { try { - Map maps = getConfigMap(getConf(), uri); + Map maps = getConfigMap(getConf()); FileSystemProvider provider = fileSystemProvidersMap.get(scheme); if (provider == null) { throw new GravitinoRuntimeException( "Unsupported file system scheme: %s for %s.", scheme, GravitinoVirtualFileSystemConfiguration.GVFS_SCHEME); } - - return provider.getFileSystem(maps); + return provider.getFileSystem(filePath, maps); } catch (IOException ioe) { throw new GravitinoRuntimeException( "Exception occurs when create new FileSystem for actual uri: %s, msg: %s", @@ -403,7 +403,7 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat return new FilesetContextPair(new Path(actualFileLocation), fs); } - private Map getConfigMap(Configuration configuration, URI uri) { + private Map getConfigMap(Configuration configuration) { Map maps = Maps.newHashMap(); configuration.forEach( entry -> { @@ -415,8 +415,6 @@ private Map getConfigMap(Configuration configuration, URI uri) { } }); - maps.put(FS_DEFAULT_NAME_KEY, uri.toString()); - return maps; } From da0b7caa42a7011a63e4ec6c778a3ad8096cd38d Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 15:08:09 +0800 Subject: [PATCH 37/89] Polish again. --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 4 +++- .../catalog/hadoop/HadoopCatalogPropertiesMetadata.java | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index ba843887238..7a014bf6c7f 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.catalog.hadoop; +import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; @@ -110,7 +112,7 @@ public CatalogInfo getCatalogInfo() { public Configuration getHadoopConf() { Configuration configuration = new Configuration(); - conf.forEach(configuration::set); + conf.forEach((k, v) -> configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v)); return configuration; } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 1a2cb2f11b2..54bd57b945c 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -48,7 +48,8 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada * The default file system provider. It is used to create the default file system instance; if not * specified, file system instance will be created with the schema prefix in the file path like * 'file:/tmp/'. If there is no schema prefix, the default file system provider will be local file - * system. + * system. The candidate value is 'local' or 'hdfs' or others specified in the {@link + * FileSystemProvider#getScheme()} */ public static final String DEFAULT_FS_PROVIDER = "default-filesystem-provider"; @@ -75,7 +76,8 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada PropertyEntry.stringOptionalPropertyEntry( DEFAULT_FS_PROVIDER, "Default file system provider, used to create the default file system " - + "candidate value is 'local' or 'hdfs'", + + "candidate value is 'local', 'hdfs' or others specified in the " + + "FileSystemProvider#getScheme()", false /* immutable */, null, false /* hidden */)) From ba1fe5fdd13500e0252ad7c07e4a6f5fb6766a64 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 15:22:22 +0800 Subject: [PATCH 38/89] Rebase branch issue_5019 --- .../apache/gravitino/fileset/gcs/GCSFileSystemProvider.java | 6 ++++-- .../catalog/hadoop/integration/test/HadoopGCPCatalogIT.java | 4 ---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java index daa286535b1..5a5b6edd5c0 100644 --- a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java +++ b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java @@ -18,22 +18,24 @@ */ package org.apache.gravitino.fileset.gcs; +import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; import java.io.IOException; import java.util.Map; import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; public class GCSFileSystemProvider implements FileSystemProvider { @Override - public FileSystem getFileSystem(Map config) throws IOException { + public FileSystem getFileSystem(Path path, Map config) throws IOException { Configuration configuration = new Configuration(); config.forEach( (k, v) -> { configuration.set(k.replace("gravitino.bypass.", ""), v); }); - return FileSystem.get(configuration); + return GoogleHadoopFileSystem.newInstance(path.toUri(), configuration); } @Override diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java index ba1fdf8ac96..e512f74d5c4 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java @@ -53,10 +53,8 @@ public void setup() throws IOException { schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); Configuration conf = new Configuration(); - conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); conf.set("fs.gs.auth.service.account.enable", "true"); conf.set("fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); - conf.set("fs.defaultFS", "gs:///" + BUCKET_NAME); fileSystem = FileSystem.get(URI.create(String.format("gs://%s", BUCKET_NAME)), conf); createMetalake(); @@ -86,10 +84,8 @@ protected String defaultBaseLocation() { protected void createCatalog() { Map map = Maps.newHashMap(); - map.put("gravitino.bypass.fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); map.put("gravitino.bypass.fs.gs.auth.service.account.enable", "true"); map.put("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); - map.put("gravitino.bypass.fs.defaultFS", "gs://" + BUCKET_NAME); map.put(FILESYSTEM_PROVIDERS, "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider"); metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map); From 7c44a57b435e5cb858c233829206387c725cfb89 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 15:58:31 +0800 Subject: [PATCH 39/89] Support python gvfs --- .../gravitino/filesystem/gvfs.py | 95 ++++++++++- .../gravitino/filesystem/gvfs_config.py | 5 + clients/client-python/requirements.txt | 3 +- .../tests/integration/test_gvfs_with_gcs.py | 93 +++++++++++ .../tests/integration/test_gvfs_with_hdfs.py | 154 ++++++++++-------- 5 files changed, 276 insertions(+), 74 deletions(-) create mode 100644 clients/client-python/tests/integration/test_gvfs_with_gcs.py diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index e5a565ce0d6..bba4e41fbff 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import os from enum import Enum from pathlib import PurePosixPath from typing import Dict, Tuple @@ -27,6 +27,9 @@ from fsspec.implementations.arrow import ArrowFSWrapper from fsspec.utils import infer_storage_options from pyarrow.fs import HadoopFileSystem +from pyarrow.fs import GcsFileSystem +from pyarrow.fs import S3FileSystem + from readerwriterlock import rwlock from gravitino.audit.caller_context import CallerContext, CallerContextHolder from gravitino.audit.fileset_audit_constants import FilesetAuditConstants @@ -46,7 +49,9 @@ class StorageType(Enum): HDFS = "hdfs" - LOCAL = "file" + LOCAL = ("file",) + GCS = ("gs",) + S3 = ("s3",) class FilesetContextPair: @@ -66,7 +71,7 @@ def filesystem(self): class GravitinoVirtualFileSystem(fsspec.AbstractFileSystem): - """This is a virtual file system which users can access `fileset` and + """This is a virtual file system that users can access `fileset` and other resources. It obtains the actual storage location corresponding to the resource from the @@ -149,6 +154,7 @@ def __init__( self._cache_lock = rwlock.RWLockFair() self._catalog_cache = LRUCache(maxsize=100) self._catalog_cache_lock = rwlock.RWLockFair() + self._options = options super().__init__(**kwargs) @@ -309,7 +315,9 @@ def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): ) dst_actual_path = dst_context_pair.actual_file_location() - if storage_type == StorageType.HDFS: + # convert the following to in + + if storage_type in [StorageType.HDFS, StorageType.GCS, StorageType.S3]: src_context_pair.filesystem().mv( self._strip_storage_protocol(storage_type, src_actual_path), self._strip_storage_protocol(storage_type, dst_actual_path), @@ -540,7 +548,13 @@ def _convert_actual_path( :param virtual_location: Virtual location :return A virtual path """ - if storage_location.startswith(f"{StorageType.HDFS.value}://"): + # If the storage path start with hdfs, gcs, s3, s3a or s3n, we should use the path as the prefix. + + if ( + storage_location.startswith(f"{StorageType.HDFS.value}://") + or storage_location.startswith(f"{StorageType.GCS.value}://") + or storage_location.startswith(f"{StorageType.S3.value}://") + ): actual_prefix = infer_storage_options(storage_location)["path"] elif storage_location.startswith(f"{StorageType.LOCAL.value}:/"): actual_prefix = storage_location[len(f"{StorageType.LOCAL.value}:") :] @@ -681,6 +695,10 @@ def _recognize_storage_type(path: str): return StorageType.HDFS if path.startswith(f"{StorageType.LOCAL.value}:/"): return StorageType.LOCAL + if path.startswith(f"{StorageType.GCS.value}://"): + return StorageType.GCS + if path.startswith(f"{StorageType.S3.value}://"): + return StorageType.S3 raise GravitinoRuntimeException( f"Storage type doesn't support now. Path:{path}" ) @@ -777,6 +795,10 @@ def _get_filesystem(self, actual_file_location: str): fs = ArrowFSWrapper(HadoopFileSystem.from_uri(actual_file_location)) elif storage_type == StorageType.LOCAL: fs = LocalFileSystem() + elif storage_type == StorageType.GCS: + fs = ArrowFSWrapper(self._get_gcs_filesystem()) + elif storage_type == StorageType.S3: + fs = ArrowFSWrapper(self._get_s3_filesystem()) else: raise GravitinoRuntimeException( f"Storage type: `{storage_type}` doesn't support now." @@ -786,5 +808,68 @@ def _get_filesystem(self, actual_file_location: str): finally: write_lock.release() + def _get_gcs_filesystem(self): + # get All keys from the options that start with 'gravitino.bypass.gcs.' and remove the prefix + gcs_options = { + key[len(GVFSConfig.GVFS_FILESYSTEM_BY_PASS_GCS) :]: value + for key, value in self._options.items() + if key.startswith(GVFSConfig.GVFS_FILESYSTEM_BY_PASS_GCS) + } + + # get 'service-account-key' from gcs_options, if the key is not found, throw an exception + service_account_key_path = gcs_options.get("service-account-key-path") + if service_account_key_path is None: + raise GravitinoRuntimeException( + "Service account key is not found in the options." + ) + + # scopes = ["https://www.googleapis.com/auth/cloud-platform"] + # credentials = service_account.Credentials.from_service_account_file( + # service_account_key_path, scopes=scopes) + # credentials.refresh(Request()) + + # access_token = credentials.token + # expiration = credentials.expiry + + # return GcsFileSystem(access_token=access_token, + # credential_token_expiration=expiration) + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_path + return GcsFileSystem() + + def _get_s3_filesystem(self): + # get All keys from the options that start with 'gravitino.bypass.s3.' and remove the prefix + s3_options = { + key[len(GVFSConfig.GVFS_FILESYSTEM_BY_PASS_S3) :]: value + for key, value in self._options.items() + if key.startswith(GVFSConfig.GVFS_FILESYSTEM_BY_PASS_S3) + } + + # get 'aws_access_key_id' from s3_options, if the key is not found, throw an exception + aws_access_key_id = s3_options.get("aws_access_key_id") + if aws_access_key_id is None: + raise GravitinoRuntimeException( + "AWS access key id is not found in the options." + ) + + # get 'aws_secret_access_key' from s3_options, if the key is not found, throw an exception + aws_secret_access_key = s3_options.get("aws_secret_access_key") + if aws_secret_access_key is None: + raise GravitinoRuntimeException( + "AWS secret access key is not found in the options." + ) + + # get 'aws_endpoint_url' from s3_options, if the key is not found, throw an exception + aws_endpoint_url = s3_options.get("aws_endpoint_url") + if aws_endpoint_url is None: + raise GravitinoRuntimeException( + "AWS endpoint url is not found in the options." + ) + + return S3FileSystem( + key=aws_access_key_id, + secret=aws_secret_access_key, + endpoint_override=aws_endpoint_url, + ) + fsspec.register_implementation(PROTOCOL_NAME, GravitinoVirtualFileSystem) diff --git a/clients/client-python/gravitino/filesystem/gvfs_config.py b/clients/client-python/gravitino/filesystem/gvfs_config.py index eb5733b56be..743216d7ed3 100644 --- a/clients/client-python/gravitino/filesystem/gvfs_config.py +++ b/clients/client-python/gravitino/filesystem/gvfs_config.py @@ -31,3 +31,8 @@ class GVFSConfig: OAUTH2_CREDENTIAL = "oauth2_credential" OAUTH2_PATH = "oauth2_path" OAUTH2_SCOPE = "oauth2_scope" + DEFAULT_AUTH_TYPE = "simple" + + GVFS_FILESYSTEM_BY_PASS = "gravitino.bypass" + GVFS_FILESYSTEM_BY_PASS_GCS = "gravitino.bypass.gcs." + GVFS_FILESYSTEM_BY_PASS_S3 = "gravitino.bypass.s3." diff --git a/clients/client-python/requirements.txt b/clients/client-python/requirements.txt index 7242082b77c..a330f738a1f 100644 --- a/clients/client-python/requirements.txt +++ b/clients/client-python/requirements.txt @@ -22,4 +22,5 @@ dataclasses-json==0.6.6 readerwriterlock==1.0.9 fsspec==2024.3.1 pyarrow==15.0.2 -cachetools==5.3.3 \ No newline at end of file +cachetools==5.3.3 +google-auth==2.35.0 \ No newline at end of file diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py new file mode 100644 index 00000000000..021d6f8510e --- /dev/null +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +from fsspec.implementations.arrow import ArrowFSWrapper +from pyarrow.fs import GcsFileSystem + +from tests.integration.test_gvfs_with_hdfs import TestGvfsWithHDFS +from gravitino import ( + GravitinoClient, + Catalog, + Fileset, +) + + +class TestGvfsWithGCS(TestGvfsWithHDFS): + + @classmethod + def setUpClass(cls): + cls._get_gravitino_home() + + # init gcs config + cls.config = { + "gravitino.bypass.gcs.service-account-key-path": "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" + } + + cls.bucket_name = "gravitino-gcs-test" + cls.options = { + "gravitino.bypass.gcs.service-account-key-path": "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" + } + + cls.hadoop_conf_path = f"{cls.gravitino_home}/catalogs/hadoop/conf/hadoop.conf" + + # append the hadoop conf to server + cls._append_conf(cls.config, cls.hadoop_conf_path) + # restart the server + cls.restart_server() + # create entity + cls._init_test_entities() + + @classmethod + def _init_test_entities(cls): + cls.gravitino_admin_client.create_metalake( + name=cls.metalake_name, comment="", properties={} + ) + cls.gravitino_client = GravitinoClient( + uri="http://localhost:8090", metalake_name=cls.metalake_name + ) + catalog = cls.gravitino_client.create_catalog( + name=cls.catalog_name, + catalog_type=Catalog.Type.FILESET, + provider=cls.catalog_provider, + comment="", + properties={}, + ) + catalog.as_schemas().create_schema( + schema_name=cls.schema_name, comment="", properties={} + ) + + cls.fileset_storage_location: str = ( + f"gs://{cls.bucket_name}/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}" + ) + cls.fileset_gvfs_location = ( + f"gvfs://fileset/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}" + ) + catalog.as_fileset_catalog().create_fileset( + ident=cls.fileset_ident, + fileset_type=Fileset.Type.MANAGED, + comment=cls.fileset_comment, + storage_location=cls.fileset_storage_location, + properties=cls.fileset_properties, + ) + + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.config[ + "gravitino.bypass.gcs.service-account-key-path" + ] + arrow_gcs_fs = GcsFileSystem() + cls.fs = ArrowFSWrapper(arrow_gcs_fs) diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 9116005b840..8793f570d1a 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -159,7 +159,7 @@ def _init_test_entities(cls): properties=cls.fileset_properties, ) arrow_hadoop_fs = HadoopFileSystem(host=cls.hdfs_container.get_ip(), port=9000) - cls.hdfs = ArrowFSWrapper(arrow_hadoop_fs) + cls.fs = ArrowFSWrapper(arrow_hadoop_fs) cls.conf: Dict = {"fs.defaultFS": f"hdfs://{cls.hdfs_container.get_ip()}:9000/"} @classmethod @@ -208,7 +208,6 @@ def _clean_test_data(cls): logger.warning("Failed to drop metalake %s", cls.metalake_name) def test_simple_auth(self): - options = {"auth_type": "simple"} current_user = ( None if os.environ.get("user.name") is None else os.environ["user.name"] ) @@ -217,7 +216,7 @@ def test_simple_auth(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=options, + options=self.options, ) token = fs._client._rest_client.auth_data_provider.get_token_data() token_string = base64.b64decode( @@ -234,15 +233,16 @@ def test_ls(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(ls_actual_dir) - self.assertTrue(self.hdfs.exists(ls_actual_dir)) + self.fs.mkdir(ls_actual_dir) + self.assertTrue(self.fs.exists(ls_actual_dir)) ls_file = self.fileset_gvfs_location + "/test_ls/test.file" ls_actual_file = self.fileset_storage_location + "/test_ls/test.file" - self.hdfs.touch(ls_actual_file) - self.assertTrue(self.hdfs.exists(ls_actual_file)) + self.fs.touch(ls_actual_file) + self.assertTrue(self.fs.exists(ls_actual_file)) # test detail = false file_list_without_detail = fs.ls(ls_dir, detail=False) @@ -260,15 +260,16 @@ def test_info(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(info_actual_dir) - self.assertTrue(self.hdfs.exists(info_actual_dir)) + self.fs.mkdir(info_actual_dir) + self.assertTrue(self.fs.exists(info_actual_dir)) info_file = self.fileset_gvfs_location + "/test_info/test.file" info_actual_file = self.fileset_storage_location + "/test_info/test.file" - self.hdfs.touch(info_actual_file) - self.assertTrue(self.hdfs.exists(info_actual_file)) + self.fs.touch(info_actual_file) + self.assertTrue(self.fs.exists(info_actual_file)) dir_info = fs.info(info_dir) self.assertEqual(dir_info["name"], info_dir[len("gvfs://") :]) @@ -282,16 +283,17 @@ def test_exist(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(exist_actual_dir) - self.assertTrue(self.hdfs.exists(exist_actual_dir)) + self.fs.mkdir(exist_actual_dir) + self.assertTrue(self.fs.exists(exist_actual_dir)) self.assertTrue(fs.exists(exist_dir)) exist_file = self.fileset_gvfs_location + "/test_exist/test.file" exist_actual_file = self.fileset_storage_location + "/test_exist/test.file" - self.hdfs.touch(exist_actual_file) - self.assertTrue(self.hdfs.exists(exist_actual_file)) + self.fs.touch(exist_actual_file) + self.assertTrue(self.fs.exists(exist_actual_file)) self.assertTrue(fs.exists(exist_file)) def test_cp_file(self): @@ -300,19 +302,20 @@ def test_cp_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(cp_file_actual_dir) - self.assertTrue(self.hdfs.exists(cp_file_actual_dir)) + self.fs.mkdir(cp_file_actual_dir) + self.assertTrue(self.fs.exists(cp_file_actual_dir)) self.assertTrue(fs.exists(cp_file_dir)) cp_file_file = self.fileset_gvfs_location + "/test_cp_file/test.file" cp_file_actual_file = self.fileset_storage_location + "/test_cp_file/test.file" - self.hdfs.touch(cp_file_actual_file) - self.assertTrue(self.hdfs.exists(cp_file_actual_file)) + self.fs.touch(cp_file_actual_file) + self.assertTrue(self.fs.exists(cp_file_actual_file)) self.assertTrue(fs.exists(cp_file_file)) - with self.hdfs.open(cp_file_actual_file, "wb") as f: + with self.fs.open(cp_file_actual_file, "wb") as f: f.write(b"test_file_1") cp_file_new_file = self.fileset_gvfs_location + "/test_cp_file/test_cp.file" @@ -322,7 +325,7 @@ def test_cp_file(self): fs.cp_file(cp_file_file, cp_file_new_file) self.assertTrue(fs.exists(cp_file_new_file)) - with self.hdfs.open(cp_file_new_actual_file, "rb") as f: + with self.fs.open(cp_file_new_actual_file, "rb") as f: result = f.read() self.assertEqual(b"test_file_1", result) @@ -332,10 +335,11 @@ def test_mv(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(mv_actual_dir) - self.assertTrue(self.hdfs.exists(mv_actual_dir)) + self.fs.mkdir(mv_actual_dir) + self.assertTrue(self.fs.exists(mv_actual_dir)) self.assertTrue(fs.exists(mv_dir)) mv_new_dir = self.fileset_gvfs_location + "/test_mv_new" @@ -343,16 +347,17 @@ def test_mv(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(mv_new_actual_dir) - self.assertTrue(self.hdfs.exists(mv_new_actual_dir)) + self.fs.mkdir(mv_new_actual_dir) + self.assertTrue(self.fs.exists(mv_new_actual_dir)) self.assertTrue(fs.exists(mv_new_dir)) mv_file = self.fileset_gvfs_location + "/test_mv/test.file" mv_actual_file = self.fileset_storage_location + "/test_mv/test.file" - self.hdfs.touch(mv_actual_file) - self.assertTrue(self.hdfs.exists(mv_actual_file)) + self.fs.touch(mv_actual_file) + self.assertTrue(self.fs.exists(mv_actual_file)) self.assertTrue(fs.exists(mv_file)) mv_new_file = self.fileset_gvfs_location + "/test_mv_new/test_new.file" @@ -362,7 +367,7 @@ def test_mv(self): fs.mv(mv_file, mv_new_file) self.assertTrue(fs.exists(mv_new_file)) - self.assertTrue(self.hdfs.exists(mv_new_actual_file)) + self.assertTrue(self.fs.exists(mv_new_actual_file)) # test rename without sub path, which should throw an exception with self.assertRaises(GravitinoRuntimeException): @@ -374,16 +379,17 @@ def test_rm(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(rm_actual_dir) - self.assertTrue(self.hdfs.exists(rm_actual_dir)) + self.fs.mkdir(rm_actual_dir) + self.assertTrue(self.fs.exists(rm_actual_dir)) self.assertTrue(fs.exists(rm_dir)) rm_file = self.fileset_gvfs_location + "/test_rm/test.file" rm_actual_file = self.fileset_storage_location + "/test_rm/test.file" - self.hdfs.touch(rm_file) - self.assertTrue(self.hdfs.exists(rm_actual_file)) + self.fs.touch(rm_file) + self.assertTrue(self.fs.exists(rm_actual_file)) self.assertTrue(fs.exists(rm_file)) # test delete file @@ -393,8 +399,8 @@ def test_rm(self): # test delete dir with recursive = false rm_new_file = self.fileset_gvfs_location + "/test_rm/test_new.file" rm_new_actual_file = self.fileset_storage_location + "/test_rm/test_new.file" - self.hdfs.touch(rm_new_actual_file) - self.assertTrue(self.hdfs.exists(rm_new_actual_file)) + self.fs.touch(rm_new_actual_file) + self.assertTrue(self.fs.exists(rm_new_actual_file)) self.assertTrue(fs.exists(rm_new_file)) with self.assertRaises(ValueError): fs.rm(rm_dir, recursive=False) @@ -409,16 +415,17 @@ def test_rm_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(rm_file_actual_dir) - self.assertTrue(self.hdfs.exists(rm_file_actual_dir)) + self.fs.mkdir(rm_file_actual_dir) + self.assertTrue(self.fs.exists(rm_file_actual_dir)) self.assertTrue(fs.exists(rm_file_dir)) rm_file_file = self.fileset_gvfs_location + "/test_rm_file/test.file" rm_file_actual_file = self.fileset_storage_location + "/test_rm_file/test.file" - self.hdfs.touch(rm_file_actual_file) - self.assertTrue(self.hdfs.exists(rm_file_actual_file)) + self.fs.touch(rm_file_actual_file) + self.assertTrue(self.fs.exists(rm_file_actual_file)) self.assertTrue(fs.exists(rm_file_file)) # test delete file @@ -435,16 +442,17 @@ def test_rmdir(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(rmdir_actual_dir) - self.assertTrue(self.hdfs.exists(rmdir_actual_dir)) + self.fs.mkdir(rmdir_actual_dir) + self.assertTrue(self.fs.exists(rmdir_actual_dir)) self.assertTrue(fs.exists(rmdir_dir)) rmdir_file = self.fileset_gvfs_location + "/test_rmdir/test.file" rmdir_actual_file = self.fileset_storage_location + "/test_rmdir/test.file" - self.hdfs.touch(rmdir_actual_file) - self.assertTrue(self.hdfs.exists(rmdir_actual_file)) + self.fs.touch(rmdir_actual_file) + self.assertTrue(self.fs.exists(rmdir_actual_file)) self.assertTrue(fs.exists(rmdir_file)) # test delete file @@ -461,16 +469,17 @@ def test_open(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(open_actual_dir) - self.assertTrue(self.hdfs.exists(open_actual_dir)) + self.fs.mkdir(open_actual_dir) + self.assertTrue(self.fs.exists(open_actual_dir)) self.assertTrue(fs.exists(open_dir)) open_file = self.fileset_gvfs_location + "/test_open/test.file" open_actual_file = self.fileset_storage_location + "/test_open/test.file" - self.hdfs.touch(open_actual_file) - self.assertTrue(self.hdfs.exists(open_actual_file)) + self.fs.touch(open_actual_file) + self.assertTrue(self.fs.exists(open_actual_file)) self.assertTrue(fs.exists(open_file)) # test open and write file @@ -488,11 +497,12 @@ def test_mkdir(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) fs.mkdir(mkdir_dir) self.assertTrue(fs.exists(mkdir_dir)) - self.assertTrue(self.hdfs.exists(mkdir_actual_dir)) + self.assertTrue(self.fs.exists(mkdir_actual_dir)) # test mkdir dir with create_parents = false parent_not_exist_virtual_path = mkdir_dir + "/not_exist/sub_dir" @@ -514,11 +524,12 @@ def test_makedirs(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) fs.makedirs(makedirs_dir) self.assertTrue(fs.exists(makedirs_dir)) - self.assertTrue(self.hdfs.exists(makedirs_actual_dir)) + self.assertTrue(self.fs.exists(makedirs_actual_dir)) # test mkdir dir not exist parent_not_exist_virtual_path = makedirs_dir + "/not_exist/sub_dir" @@ -532,10 +543,11 @@ def test_created(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(created_actual_dir) - self.assertTrue(self.hdfs.exists(created_actual_dir)) + self.fs.mkdir(created_actual_dir) + self.assertTrue(self.fs.exists(created_actual_dir)) self.assertTrue(fs.exists(created_dir)) with self.assertRaises(GravitinoRuntimeException): @@ -547,10 +559,11 @@ def test_modified(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(modified_actual_dir) - self.assertTrue(self.hdfs.exists(modified_actual_dir)) + self.fs.mkdir(modified_actual_dir) + self.assertTrue(self.fs.exists(modified_actual_dir)) self.assertTrue(fs.exists(modified_dir)) # test mkdir dir which exists @@ -562,16 +575,17 @@ def test_cat_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(cat_actual_dir) - self.assertTrue(self.hdfs.exists(cat_actual_dir)) + self.fs.mkdir(cat_actual_dir) + self.assertTrue(self.fs.exists(cat_actual_dir)) self.assertTrue(fs.exists(cat_dir)) cat_file = self.fileset_gvfs_location + "/test_cat/test.file" cat_actual_file = self.fileset_storage_location + "/test_cat/test.file" - self.hdfs.touch(cat_actual_file) - self.assertTrue(self.hdfs.exists(cat_actual_file)) + self.fs.touch(cat_actual_file) + self.assertTrue(self.fs.exists(cat_actual_file)) self.assertTrue(fs.exists(cat_file)) # test open and write file @@ -589,16 +603,17 @@ def test_get_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(get_actual_dir) - self.assertTrue(self.hdfs.exists(get_actual_dir)) + self.fs.mkdir(get_actual_dir) + self.assertTrue(self.fs.exists(get_actual_dir)) self.assertTrue(fs.exists(get_dir)) get_file = self.fileset_gvfs_location + "/test_get/test.file" get_actual_file = self.fileset_storage_location + "/test_get/test.file" - self.hdfs.touch(get_actual_file) - self.assertTrue(self.hdfs.exists(get_actual_file)) + self.fs.touch(get_actual_file) + self.assertTrue(self.fs.exists(get_actual_file)) self.assertTrue(fs.exists(get_file)) # test open and write file @@ -628,10 +643,11 @@ def test_pandas(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(pands_actual_dir) - self.assertTrue(self.hdfs.exists(pands_actual_dir)) + self.fs.mkdir(pands_actual_dir) + self.assertTrue(self.fs.exists(pands_actual_dir)) self.assertTrue(fs.exists(pands_dir)) data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) @@ -642,7 +658,7 @@ def test_pandas(self): ) data.to_parquet(parquet_file, filesystem=fs) self.assertTrue(fs.exists(parquet_file)) - self.assertTrue(self.hdfs.exists(parquet_actual_file)) + self.assertTrue(self.fs.exists(parquet_actual_file)) # read parquet ds1 = pandas.read_parquet(path=parquet_file, filesystem=fs) @@ -660,7 +676,7 @@ def test_pandas(self): storage_options=storage_options, ) self.assertTrue(fs.exists(csv_file)) - self.assertTrue(self.hdfs.exists(csv_actual_file)) + self.assertTrue(self.fs.exists(csv_actual_file)) # read csv ds2 = pandas.read_csv(csv_file, storage_options=storage_options) @@ -672,10 +688,11 @@ def test_pyarrow(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(pyarrow_actual_dir) - self.assertTrue(self.hdfs.exists(pyarrow_actual_dir)) + self.fs.mkdir(pyarrow_actual_dir) + self.assertTrue(self.fs.exists(pyarrow_actual_dir)) self.assertTrue(fs.exists(pyarrow_dir)) data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) @@ -701,10 +718,11 @@ def test_llama_index(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(llama_actual_dir) - self.assertTrue(self.hdfs.exists(llama_actual_dir)) + self.fs.mkdir(llama_actual_dir) + self.assertTrue(self.fs.exists(llama_actual_dir)) self.assertTrue(fs.exists(llama_dir)) data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) From 992ba0ac58998019ad6e221e7f487b1ba6ceab7e Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 16:15:31 +0800 Subject: [PATCH 40/89] fix --- clients/client-python/gravitino/filesystem/gvfs.py | 8 ++++---- .../client-python/tests/integration/test_gvfs_with_gcs.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index bba4e41fbff..91d4eeb994f 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -48,10 +48,10 @@ class StorageType(Enum): - HDFS = "hdfs" - LOCAL = ("file",) - GCS = ("gs",) - S3 = ("s3",) + HDFS = "hdfs", + LOCAL = "file", + GCS = "gs", + S3 = "s3" class FilesetContextPair: diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 021d6f8510e..206ce09bcfe 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -66,7 +66,9 @@ def _init_test_entities(cls): catalog_type=Catalog.Type.FILESET, provider=cls.catalog_provider, comment="", - properties={}, + properties={ + "filesystem.providers": "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider" + }, ) catalog.as_schemas().create_schema( schema_name=cls.schema_name, comment="", properties={} From 5dbca5f0c93b5b1f342fd024ad8f7848c7f6a4f7 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 17:15:18 +0800 Subject: [PATCH 41/89] fix --- .../hadoop/HadoopCatalogOperations.java | 38 +++++++++---------- .../HadoopCatalogPropertiesMetadata.java | 32 +++++++--------- .../catalog/hadoop/fs/FileSystemUtils.java | 11 ++++++ docs/hadoop-catalog.md | 26 ++++++------- 4 files changed, 55 insertions(+), 52 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 7a014bf6c7f..fbd4001a8b9 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -92,7 +92,7 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private final Map fileSystemProvidersMap = Maps.newHashMap(); - private String defaultFilesystemProvider; + private String defaultFileSystemProviderScheme; HadoopCatalogOperations(EntityStore store) { this.store = store; @@ -136,11 +136,17 @@ public void initialize( .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); FileSystemUtils.initFileSystemProviders(fileSystemProviders, fileSystemProvidersMap); - this.defaultFilesystemProvider = + String defaultFileSystemProviderClassName = (String) propertiesMetadata .catalogPropertiesMetadata() - .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS_PROVIDER); + .getOrDefault( + config, HadoopCatalogPropertiesMetadata.DEFAULT_FS_PROVIDER_CLASSNAME); + this.defaultFileSystemProviderScheme = + StringUtils.isNotBlank(defaultFileSystemProviderClassName) + ? FileSystemUtils.getSchemeByProvider( + defaultFileSystemProviderClassName, fileSystemProvidersMap) + : LOCAL_FILE_SCHEME; String catalogLocation = (String) @@ -756,27 +762,17 @@ FileSystem getFileSystem(Path path, Map config) throws IOExcepti throw new IllegalArgumentException("Path should not be null"); } - // Can't get the scheme from the path like '/path/to/file', use the default filesystem provider. - if (path.toUri().getScheme() == null) { - if (defaultFilesystemProvider != null) { - return getFileSystemByScheme(defaultFilesystemProvider, config, path); - } - - LOG.warn( - "Can't get schema from path: {} and default filesystem provider is null, using" - + " local file system", - path); - return getFileSystemByScheme(LOCAL_FILE_SCHEME, config, path); - } + String scheme = + path.toUri().getScheme() != null + ? path.toUri().getScheme() + : defaultFileSystemProviderScheme; - return getFileSystemByScheme(path.toUri().getScheme(), config, path); - } - - private FileSystem getFileSystemByScheme(String scheme, Map config, Path path) - throws IOException { FileSystemProvider provider = fileSystemProvidersMap.get(scheme); if (provider == null) { - throw new IllegalArgumentException("Unsupported scheme: " + scheme); + throw new IllegalArgumentException( + String.format( + "Unsupported scheme: %s, path: %s, supported schemas: %s", + scheme, path, fileSystemProvidersMap.keySet())); } return provider.getFileSystem(path, config); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 54bd57b945c..4db6898c1b7 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -23,6 +23,7 @@ import org.apache.gravitino.catalog.hadoop.authentication.AuthenticationConfig; import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; +import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; @@ -36,22 +37,19 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada public static final String LOCATION = "location"; /** - * The implementation class name of the {@link FileSystemProvider} to be used by the catalog. - * Gravitino supports LocalFileSystem and HDFS by default. Users can implement their own by - * extending {@link FileSystemProvider} and specify the class name here. - * - *

The value can be 'xxxx.yyy.FileSystemProvider1, xxxx.yyy.FileSystemProvider2'. + * The class names of {@link FileSystemProvider} to be added to the catalog. Except built-in + * FileSystemProvider like LocalFileSystemProvider and HDFSFileSystemProvider, users can add their + * own FileSystemProvider by specifying the class name here. The value can be + * 'xxxx.yyy.FileSystemProvider1,xxxx.yyy.FileSystemProvider2'. */ - public static final String FILESYSTEM_PROVIDERS = "filesystem-providers"; + public static final String FILESYSTEM_PROVIDERS = "filesystem-providers-classnames"; /** - * The default file system provider. It is used to create the default file system instance; if not - * specified, file system instance will be created with the schema prefix in the file path like - * 'file:/tmp/'. If there is no schema prefix, the default file system provider will be local file - * system. The candidate value is 'local' or 'hdfs' or others specified in the {@link - * FileSystemProvider#getScheme()} + * The default file system provider class name, used to create the default file system. If not + * specified, the default file system provider will be {@link LocalFileSystemProvider}. */ - public static final String DEFAULT_FS_PROVIDER = "default-filesystem-provider"; + public static final String DEFAULT_FS_PROVIDER_CLASSNAME = + "default-filesystem-provider-classname"; private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() @@ -72,14 +70,12 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada null, false /* hidden */)) .put( - DEFAULT_FS_PROVIDER, + DEFAULT_FS_PROVIDER_CLASSNAME, PropertyEntry.stringOptionalPropertyEntry( - DEFAULT_FS_PROVIDER, - "Default file system provider, used to create the default file system " - + "candidate value is 'local', 'hdfs' or others specified in the " - + "FileSystemProvider#getScheme()", + DEFAULT_FS_PROVIDER_CLASSNAME, + "Default file system provider, used to create the default file system", false /* immutable */, - null, + LocalFileSystemProvider.class.getCanonicalName(), false /* hidden */)) // The following two are about authentication. .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 1ea86caca2a..3c6c3efc4ab 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -50,4 +50,15 @@ public static void initFileSystemProviders( } } } + + public static String getSchemeByProvider( + String providerClassName, Map fileProvidersMap) { + for (Map.Entry entry : fileProvidersMap.entrySet()) { + if (entry.getValue().getClass().getName().equals(providerClassName)) { + return entry.getKey(); + } + } + + throw new UnsupportedOperationException("Provider class name not found: " + providerClassName); + } } diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index 698cb187e45..a195c615fc8 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -25,19 +25,19 @@ Hadoop 3. If there's any compatibility issue, please create an [issue](https://g Besides the [common catalog properties](./gravitino-server-config.md#gravitino-catalog-properties-configuration), the Hadoop catalog has the following properties: -| Property Name | Description | Default Value | Required | Since Version | -|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------------------|---------------| -| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | -| `filesystem-providers` | The filesystem providers for the Hadoop catalog. Gravitino already support `local file` and `hdfs`, if you want to support other file system, you can implement `FileSystemProvider` and set this value | (none) | No | 0.7.0 | -| `default-filesystem-provider` | The default file system provider this Hadoop catalog. The value of this can be 'file', 'hdfs' currently, more information please refer to `filesystem-providers` | (none) | No | 0.7.0 | -| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | -| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | - -For more about `filesystem-providers`, please refer to `HadoopFileSystemProvider` or `LocalFileSystemProvider` in the source code. Furthermore, you also need to place the jar of the file system provider into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory. +| Property Name | Description | Default Value | Required | Since Version | +|-----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------|-------------------------------------------------------------|---------------| +| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | +| `filesystem-providers-classnames` | The fully qualified classnames of filesystem providers for the Hadoop catalog. Gravitino already support built-in `LocalFileSystemProvider`(`local file`) and `HDFSFileSystemProvider`(`hdfs`). If you want to support more file system and add it to Gravitino, you can implement `FileSystemProvider` and set this value | (none) | No | 0.7.0 | +| `default-filesystem-provider-classname` | The fully qualified classnames of default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider` | `org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider` | No | 0.7.0 | +| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | +| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | + +For more about `filesystem-providers-classnames`, please refer to `HadoopFileSystemProvider` or `LocalFileSystemProvider` in the source code. Furthermore, you also need to place the jar of the file system provider into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory if it's not in the classpath. ### Authentication for Hadoop Catalog From f27520adafcedd17e19148ca40aedcf6289951a9 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 17:25:38 +0800 Subject: [PATCH 42/89] Update code. --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 7 ++++--- .../catalog/hadoop/HadoopCatalogPropertiesMetadata.java | 6 +++--- .../gravitino/catalog/hadoop/fs/FileSystemUtils.java | 7 +++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index fbd4001a8b9..ba8becbe90c 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -133,7 +133,8 @@ public void initialize( (String) propertiesMetadata .catalogPropertiesMetadata() - .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); + .getOrDefault( + config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS_CLASSNAMES); FileSystemUtils.initFileSystemProviders(fileSystemProviders, fileSystemProvidersMap); String defaultFileSystemProviderClassName = @@ -144,7 +145,7 @@ public void initialize( config, HadoopCatalogPropertiesMetadata.DEFAULT_FS_PROVIDER_CLASSNAME); this.defaultFileSystemProviderScheme = StringUtils.isNotBlank(defaultFileSystemProviderClassName) - ? FileSystemUtils.getSchemeByProvider( + ? FileSystemUtils.getSchemeByFileSystemProvider( defaultFileSystemProviderClassName, fileSystemProvidersMap) : LOCAL_FILE_SCHEME; @@ -771,7 +772,7 @@ FileSystem getFileSystem(Path path, Map config) throws IOExcepti if (provider == null) { throw new IllegalArgumentException( String.format( - "Unsupported scheme: %s, path: %s, supported schemas: %s", + "Unsupported scheme: %s, path: %s, all supported scheme: %s", scheme, path, fileSystemProvidersMap.keySet())); } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 4db6898c1b7..06e4c579789 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -42,7 +42,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada * own FileSystemProvider by specifying the class name here. The value can be * 'xxxx.yyy.FileSystemProvider1,xxxx.yyy.FileSystemProvider2'. */ - public static final String FILESYSTEM_PROVIDERS = "filesystem-providers-classnames"; + public static final String FILESYSTEM_PROVIDERS_CLASSNAMES = "filesystem-providers-classnames"; /** * The default file system provider class name, used to create the default file system. If not @@ -62,9 +62,9 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada null, false /* hidden */)) .put( - FILESYSTEM_PROVIDERS, + FILESYSTEM_PROVIDERS_CLASSNAMES, PropertyEntry.stringOptionalPropertyEntry( - FILESYSTEM_PROVIDERS, + FILESYSTEM_PROVIDERS_CLASSNAMES, "The file system provider class name, separated by comma", false /* immutable */, null, diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 3c6c3efc4ab..cf68e005be9 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -51,7 +51,7 @@ public static void initFileSystemProviders( } } - public static String getSchemeByProvider( + public static String getSchemeByFileSystemProvider( String providerClassName, Map fileProvidersMap) { for (Map.Entry entry : fileProvidersMap.entrySet()) { if (entry.getValue().getClass().getName().equals(providerClassName)) { @@ -59,6 +59,9 @@ public static String getSchemeByProvider( } } - throw new UnsupportedOperationException("Provider class name not found: " + providerClassName); + throw new UnsupportedOperationException( + String.format( + "File system provider class name '%s' not found. Supported file system providers: %s", + providerClassName, fileProvidersMap.values())); } } From bc1e76fc09d43102bde1d262108caa18e38d4ae3 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 17:34:44 +0800 Subject: [PATCH 43/89] Rebase branch issue_5019 --- .../catalog/hadoop/integration/test/HadoopGCPCatalogIT.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java index e512f74d5c4..5f61d96049e 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java @@ -18,7 +18,7 @@ */ package org.apache.gravitino.catalog.hadoop.integration.test; -import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS_CLASSNAMES; import com.google.common.collect.Maps; import java.io.IOException; @@ -86,7 +86,8 @@ protected void createCatalog() { Map map = Maps.newHashMap(); map.put("gravitino.bypass.fs.gs.auth.service.account.enable", "true"); map.put("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); - map.put(FILESYSTEM_PROVIDERS, "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider"); + map.put( + FILESYSTEM_PROVIDERS_CLASSNAMES, "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider"); metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map); From 2115e310ab6fdff8261d712e5256041e00073567 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 17:37:51 +0800 Subject: [PATCH 44/89] fix --- clients/client-python/gravitino/filesystem/gvfs.py | 6 +++--- .../client-python/tests/integration/test_gvfs_with_gcs.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index 91d4eeb994f..e9a860c75ac 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -48,9 +48,9 @@ class StorageType(Enum): - HDFS = "hdfs", - LOCAL = "file", - GCS = "gs", + HDFS = ("hdfs",) + LOCAL = ("file",) + GCS = ("gs",) S3 = "s3" diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 206ce09bcfe..0a49ad7e432 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -39,7 +39,7 @@ def setUpClass(cls): "gravitino.bypass.gcs.service-account-key-path": "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" } - cls.bucket_name = "gravitino-gcs-test" + cls.bucket_name = "example_qazwsx" cls.options = { "gravitino.bypass.gcs.service-account-key-path": "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" } @@ -67,7 +67,7 @@ def _init_test_entities(cls): provider=cls.catalog_provider, comment="", properties={ - "filesystem.providers": "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider" + "filesystem-providers-classnames": "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider" }, ) catalog.as_schemas().create_schema( From c2e55d42382926e772064a7962c2f91e40013ca5 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 18:19:40 +0800 Subject: [PATCH 45/89] fix --- .../gravitino/filesystem/gvfs.py | 6 +++--- .../tests/integration/test_gvfs_with_gcs.py | 9 +++++---- .../tests/integration/test_gvfs_with_hdfs.py | 20 ------------------- 3 files changed, 8 insertions(+), 27 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index e9a860c75ac..0c10e9f4298 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -48,9 +48,9 @@ class StorageType(Enum): - HDFS = ("hdfs",) - LOCAL = ("file",) - GCS = ("gs",) + HDFS = "hdfs" + LOCAL = "file" + GCS = "gs" S3 = "s3" diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 0a49ad7e432..92582270a9a 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -40,9 +40,9 @@ def setUpClass(cls): } cls.bucket_name = "example_qazwsx" - cls.options = { - "gravitino.bypass.gcs.service-account-key-path": "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" - } + # cls.options = { + # "gravitino.bypass.gcs.service-account-key-path": "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" + # } cls.hadoop_conf_path = f"{cls.gravitino_home}/catalogs/hadoop/conf/hadoop.conf" @@ -67,7 +67,8 @@ def _init_test_entities(cls): provider=cls.catalog_provider, comment="", properties={ - "filesystem-providers-classnames": "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider" + "filesystem-providers-classnames": "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider", + "gravitino.bypass.fs.gs.auth.service.account.enable": "true", }, ) catalog.as_schemas().create_schema( diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 8793f570d1a..3fc1090ab1f 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -216,7 +216,6 @@ def test_simple_auth(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, ) token = fs._client._rest_client.auth_data_provider.get_token_data() token_string = base64.b64decode( @@ -233,7 +232,6 @@ def test_ls(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(ls_actual_dir) @@ -260,7 +258,6 @@ def test_info(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(info_actual_dir) @@ -283,7 +280,6 @@ def test_exist(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(exist_actual_dir) @@ -302,7 +298,6 @@ def test_cp_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(cp_file_actual_dir) @@ -335,7 +330,6 @@ def test_mv(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(mv_actual_dir) @@ -347,7 +341,6 @@ def test_mv(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(mv_new_actual_dir) @@ -379,7 +372,6 @@ def test_rm(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(rm_actual_dir) @@ -415,7 +407,6 @@ def test_rm_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(rm_file_actual_dir) @@ -442,7 +433,6 @@ def test_rmdir(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(rmdir_actual_dir) @@ -469,7 +459,6 @@ def test_open(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(open_actual_dir) @@ -497,7 +486,6 @@ def test_mkdir(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) fs.mkdir(mkdir_dir) @@ -524,7 +512,6 @@ def test_makedirs(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) fs.makedirs(makedirs_dir) @@ -543,7 +530,6 @@ def test_created(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(created_actual_dir) @@ -559,7 +545,6 @@ def test_modified(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(modified_actual_dir) @@ -575,7 +560,6 @@ def test_cat_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(cat_actual_dir) @@ -603,7 +587,6 @@ def test_get_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(get_actual_dir) @@ -643,7 +626,6 @@ def test_pandas(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(pands_actual_dir) @@ -688,7 +670,6 @@ def test_pyarrow(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(pyarrow_actual_dir) @@ -718,7 +699,6 @@ def test_llama_index(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, **self.conf, ) self.fs.mkdir(llama_actual_dir) From 557aa02446354a453955e137ffb813e3f3d6ce7d Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 18:44:03 +0800 Subject: [PATCH 46/89] fix --- clients/client-python/tests/integration/test_gvfs_with_gcs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 92582270a9a..fa56f431ba0 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -40,9 +40,6 @@ def setUpClass(cls): } cls.bucket_name = "example_qazwsx" - # cls.options = { - # "gravitino.bypass.gcs.service-account-key-path": "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" - # } cls.hadoop_conf_path = f"{cls.gravitino_home}/catalogs/hadoop/conf/hadoop.conf" From 5c3fa5c8677b1258c2ee09a584802181e16f09ca Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 19:52:46 +0800 Subject: [PATCH 47/89] fix --- .../fileset/gcs/GCSFileSystemProvider.java | 5 ++++ .../tests/integration/test_gvfs_with_gcs.py | 17 +++++++------ .../tests/integration/test_gvfs_with_hdfs.py | 24 +++++++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java index 5a5b6edd5c0..ecdc2ef51b1 100644 --- a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java +++ b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java @@ -25,8 +25,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GCSFileSystemProvider implements FileSystemProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(GCSFileSystemProvider.class); + @Override public FileSystem getFileSystem(Path path, Map config) throws IOException { Configuration configuration = new Configuration(); @@ -35,6 +39,7 @@ public FileSystem getFileSystem(Path path, Map config) throws IO configuration.set(k.replace("gravitino.bypass.", ""), v); }); + LOGGER.info("Creating GCS file system with config: {}", config); return GoogleHadoopFileSystem.newInstance(path.toUri(), configuration); } diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index fa56f431ba0..97b38ea2c57 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -29,22 +29,22 @@ class TestGvfsWithGCS(TestGvfsWithHDFS): + key_file = "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" + + def __init__(self): + super().__init__() + self.options = {"gravitino.bypass.gcs.service-account-key-path": self.key_file} @classmethod def setUpClass(cls): cls._get_gravitino_home() - - # init gcs config - cls.config = { - "gravitino.bypass.gcs.service-account-key-path": "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" - } + cls.key_file = "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" cls.bucket_name = "example_qazwsx" cls.hadoop_conf_path = f"{cls.gravitino_home}/catalogs/hadoop/conf/hadoop.conf" # append the hadoop conf to server - cls._append_conf(cls.config, cls.hadoop_conf_path) # restart the server cls.restart_server() # create entity @@ -66,6 +66,7 @@ def _init_test_entities(cls): properties={ "filesystem-providers-classnames": "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider", "gravitino.bypass.fs.gs.auth.service.account.enable": "true", + "gravitino.bypass.fs.gs.auth.service.account.json.keyfile": cls.key_file, }, ) catalog.as_schemas().create_schema( @@ -86,8 +87,6 @@ def _init_test_entities(cls): properties=cls.fileset_properties, ) - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.config[ - "gravitino.bypass.gcs.service-account-key-path" - ] + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.key_file arrow_gcs_fs = GcsFileSystem() cls.fs = ArrowFSWrapper(arrow_gcs_fs) diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 3fc1090ab1f..c642ca2815a 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -90,10 +90,15 @@ class TestGvfsWithHDFS(IntegrationTestEnv): ) gravitino_client: GravitinoClient = None + def __init__(self): + super().__init__() + self.options = {} + @classmethod def setUpClass(cls): cls._get_gravitino_home() + cls.options = {} cls.hdfs_container = HDFSContainer() hdfs_container_ip = cls.hdfs_container.get_ip() @@ -232,6 +237,7 @@ def test_ls(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(ls_actual_dir) @@ -258,6 +264,7 @@ def test_info(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(info_actual_dir) @@ -280,6 +287,7 @@ def test_exist(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(exist_actual_dir) @@ -298,6 +306,7 @@ def test_cp_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(cp_file_actual_dir) @@ -330,6 +339,7 @@ def test_mv(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(mv_actual_dir) @@ -341,6 +351,7 @@ def test_mv(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(mv_new_actual_dir) @@ -372,6 +383,7 @@ def test_rm(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(rm_actual_dir) @@ -407,6 +419,7 @@ def test_rm_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(rm_file_actual_dir) @@ -433,6 +446,7 @@ def test_rmdir(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(rmdir_actual_dir) @@ -459,6 +473,7 @@ def test_open(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(open_actual_dir) @@ -486,6 +501,7 @@ def test_mkdir(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) fs.mkdir(mkdir_dir) @@ -512,6 +528,7 @@ def test_makedirs(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) fs.makedirs(makedirs_dir) @@ -530,6 +547,7 @@ def test_created(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(created_actual_dir) @@ -545,6 +563,7 @@ def test_modified(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(modified_actual_dir) @@ -560,6 +579,7 @@ def test_cat_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(cat_actual_dir) @@ -587,6 +607,7 @@ def test_get_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(get_actual_dir) @@ -626,6 +647,7 @@ def test_pandas(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(pands_actual_dir) @@ -670,6 +692,7 @@ def test_pyarrow(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(pyarrow_actual_dir) @@ -699,6 +722,7 @@ def test_llama_index(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) self.fs.mkdir(llama_actual_dir) From 408eca75c8b1a0c93d88422eda369c689532945a Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 21:01:49 +0800 Subject: [PATCH 48/89] fix --- clients/client-python/gravitino/filesystem/gvfs.py | 3 ++- .../tests/integration/test_gvfs_with_gcs.py | 9 ++++----- .../tests/integration/test_gvfs_with_hdfs.py | 7 +++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index 0c10e9f4298..0c7f4dfedba 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -723,10 +723,11 @@ def _strip_storage_protocol(storage_type: StorageType, path: str): :param path: The path :return: The stripped path """ - if storage_type == StorageType.HDFS: + if storage_type in (StorageType.HDFS, StorageType.S3, StorageType.GCS): return path if storage_type == StorageType.LOCAL: return path[len(f"{StorageType.LOCAL.value}:") :] + raise GravitinoRuntimeException( f"Storage type:{storage_type} doesn't support now." ) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 97b38ea2c57..c497b655e9d 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -30,17 +30,15 @@ class TestGvfsWithGCS(TestGvfsWithHDFS): key_file = "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" + bucket_name = "example_qazwsx" - def __init__(self): - super().__init__() + def setUp(self): + super().setUp() self.options = {"gravitino.bypass.gcs.service-account-key-path": self.key_file} @classmethod def setUpClass(cls): cls._get_gravitino_home() - cls.key_file = "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" - - cls.bucket_name = "example_qazwsx" cls.hadoop_conf_path = f"{cls.gravitino_home}/catalogs/hadoop/conf/hadoop.conf" @@ -58,6 +56,7 @@ def _init_test_entities(cls): cls.gravitino_client = GravitinoClient( uri="http://localhost:8090", metalake_name=cls.metalake_name ) + cls.conf = {} catalog = cls.gravitino_client.create_catalog( name=cls.catalog_name, catalog_type=Catalog.Type.FILESET, diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index c642ca2815a..0b3db37845f 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -90,15 +90,13 @@ class TestGvfsWithHDFS(IntegrationTestEnv): ) gravitino_client: GravitinoClient = None - def __init__(self): - super().__init__() + def setUp(self): self.options = {} @classmethod def setUpClass(cls): cls._get_gravitino_home() - cls.options = {} cls.hdfs_container = HDFSContainer() hdfs_container_ip = cls.hdfs_container.get_ip() @@ -129,7 +127,8 @@ def tearDownClass(cls): BaseHadoopEnvironment.clear_hadoop_env() finally: # close hdfs container - cls.hdfs_container.close() + if cls.hdfs_container is not None: + cls.hdfs_container.close() @classmethod def _init_test_entities(cls): From a02065db2220931843350cc52ca69ee123d5a9e2 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 22:22:06 +0800 Subject: [PATCH 49/89] fix --- clients/client-python/gravitino/filesystem/gvfs.py | 2 +- .../client-python/tests/integration/test_gvfs_with_gcs.py | 7 +++++-- .../client-python/tests/integration/test_gvfs_with_hdfs.py | 4 +--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index 0c7f4dfedba..803e667f7ba 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -148,7 +148,7 @@ def __init__( if options is None else options.get( GVFSConfig.CACHE_EXPIRED_TIME, GVFSConfig.DEFAULT_CACHE_EXPIRED_TIME - ) + test_gvfs_with_hdfs.py) ) self._cache = TTLCache(maxsize=cache_size, ttl=cache_expired_time) self._cache_lock = rwlock.RWLockFair() diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index c497b655e9d..bbe1ae099a2 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -33,9 +33,11 @@ class TestGvfsWithGCS(TestGvfsWithHDFS): bucket_name = "example_qazwsx" def setUp(self): - super().setUp() self.options = {"gravitino.bypass.gcs.service-account-key-path": self.key_file} + def tearDown(self): + self.options = {} + @classmethod def setUpClass(cls): cls._get_gravitino_home() @@ -56,7 +58,8 @@ def _init_test_entities(cls): cls.gravitino_client = GravitinoClient( uri="http://localhost:8090", metalake_name=cls.metalake_name ) - cls.conf = {} + + cls.config = {} catalog = cls.gravitino_client.create_catalog( name=cls.catalog_name, catalog_type=Catalog.Type.FILESET, diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 0b3db37845f..e18b7ff1c0a 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -89,9 +89,7 @@ class TestGvfsWithHDFS(IntegrationTestEnv): uri="http://localhost:8090" ) gravitino_client: GravitinoClient = None - - def setUp(self): - self.options = {} + options = {} @classmethod def setUpClass(cls): From 8762bae648f64344860b5197dcc00a9ab5acd480 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 22:25:13 +0800 Subject: [PATCH 50/89] fix --- clients/client-python/gravitino/filesystem/gvfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index 803e667f7ba..0c7f4dfedba 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -148,7 +148,7 @@ def __init__( if options is None else options.get( GVFSConfig.CACHE_EXPIRED_TIME, GVFSConfig.DEFAULT_CACHE_EXPIRED_TIME - test_gvfs_with_hdfs.py) + ) ) self._cache = TTLCache(maxsize=cache_size, ttl=cache_expired_time) self._cache_lock = rwlock.RWLockFair() From dc7a9152cf6ee7a6c78aca3a3948e6f897589230 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 22:32:56 +0800 Subject: [PATCH 51/89] fix --- clients/client-python/tests/integration/test_gvfs_with_gcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index bbe1ae099a2..5bf857e11a5 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -60,6 +60,7 @@ def _init_test_entities(cls): ) cls.config = {} + cls.conf = {} catalog = cls.gravitino_client.create_catalog( name=cls.catalog_name, catalog_type=Catalog.Type.FILESET, From c23099115a33de1730a07b95dd44bce8cab218a8 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 15 Oct 2024 22:57:01 +0800 Subject: [PATCH 52/89] fix --- clients/client-python/tests/integration/test_gvfs_with_hdfs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index e18b7ff1c0a..217ef5221c0 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -218,6 +218,7 @@ def test_simple_auth(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, ) token = fs._client._rest_client.auth_data_provider.get_token_data() token_string = base64.b64decode( @@ -730,6 +731,7 @@ def test_llama_index(self): storage_options = { "server_uri": "http://localhost:8090", "metalake_name": self.metalake_name, + "options": self.options, } csv_file = llama_dir + "/test.csv" # to csv From dc548803211ec5241b0872e0d0e52c962741fe8a Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 00:03:09 +0800 Subject: [PATCH 53/89] skip some test. --- .../client-python/tests/integration/test_gvfs_with_hdfs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 217ef5221c0..a91d7f93dcf 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -375,6 +375,7 @@ def test_mv(self): with self.assertRaises(GravitinoRuntimeException): fs.mv(self.fileset_gvfs_location, self.fileset_gvfs_location + "/test_mv") + @unittest.skip("Not implemented") def test_rm(self): rm_dir = self.fileset_gvfs_location + "/test_rm" rm_actual_dir = self.fileset_storage_location + "/test_rm" @@ -639,6 +640,7 @@ def test_get_file(self): with self.assertRaises(GravitinoRuntimeException): fs.get_file(get_file, remote_path) + @unittest.skip("Not implemented") def test_pandas(self): pands_dir = self.fileset_gvfs_location + "/test_pandas" pands_actual_dir = self.fileset_storage_location + "/test_pandas" @@ -668,6 +670,7 @@ def test_pandas(self): storage_options = { "server_uri": "http://localhost:8090", "metalake_name": self.metalake_name, + "options": self.options } # to csv csv_file = self.fileset_gvfs_location + "/test_pandas/test.csv" @@ -684,6 +687,7 @@ def test_pandas(self): ds2 = pandas.read_csv(csv_file, storage_options=storage_options) self.assertTrue(data.equals(ds2)) + @unittest.skip("Not implemented") def test_pyarrow(self): pyarrow_dir = self.fileset_gvfs_location + "/test_pyarrow" pyarrow_actual_dir = self.fileset_storage_location + "/test_pyarrow" From 7ecc04038bc2bf200036f351c51c09f41f11e0f9 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 00:07:12 +0800 Subject: [PATCH 54/89] fix --- .../client-python/tests/integration/test_gvfs_with_gcs.py | 8 ++++++++ .../tests/integration/test_gvfs_with_hdfs.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 5bf857e11a5..d694941bd63 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -50,6 +50,14 @@ def setUpClass(cls): # create entity cls._init_test_entities() + @classmethod + def tearDownClass(cls): + cls._clean_test_data() + # reset server conf + cls._reset_conf(cls.config, cls.hadoop_conf_path) + # restart server + cls.restart_server() + @classmethod def _init_test_entities(cls): cls.gravitino_admin_client.create_metalake( diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index a91d7f93dcf..9ee498f340e 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -642,7 +642,7 @@ def test_get_file(self): @unittest.skip("Not implemented") def test_pandas(self): - pands_dir = self.fileset_gvfs_location + "/test_pandas" + pands_dirclear_hadoop_env = self.fileset_gvfs_location + "/test_pandas" pands_actual_dir = self.fileset_storage_location + "/test_pandas" fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", From da4632156db0b9d3508b7e397bbc02c635f33141 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 00:22:59 +0800 Subject: [PATCH 55/89] fix --- clients/client-python/tests/integration/test_gvfs_with_hdfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 9ee498f340e..a91d7f93dcf 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -642,7 +642,7 @@ def test_get_file(self): @unittest.skip("Not implemented") def test_pandas(self): - pands_dirclear_hadoop_env = self.fileset_gvfs_location + "/test_pandas" + pands_dir = self.fileset_gvfs_location + "/test_pandas" pands_actual_dir = self.fileset_storage_location + "/test_pandas" fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", From 27bc2ab28a60b7ef6f9d0cbb53ce8cab4fb8bc9d Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 10:49:18 +0800 Subject: [PATCH 56/89] Fix --- .../hadoop/HadoopCatalogOperations.java | 27 +++---- .../HadoopCatalogPropertiesMetadata.java | 32 ++++---- .../catalog/hadoop/fs/FileSystemProvider.java | 16 +++- .../catalog/hadoop/fs/FileSystemUtils.java | 77 ++++++++++--------- .../hadoop/fs/HDFSFileSystemProvider.java | 7 +- .../hadoop/fs/LocalFileSystemProvider.java | 7 +- ...itino.catalog.hadoop.fs.FileSystemProvider | 21 +++++ .../hadoop/GravitinoVirtualFileSystem.java | 2 +- ...avitinoVirtualFileSystemConfiguration.java | 3 +- docs/hadoop-catalog.md | 26 +++---- 10 files changed, 129 insertions(+), 89 deletions(-) create mode 100644 catalogs/catalog-hadoop/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index ba8becbe90c..dedc050e5c1 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -72,7 +72,6 @@ import org.slf4j.LoggerFactory; public class HadoopCatalogOperations implements CatalogOperations, SupportsSchemas, FilesetCatalog { - private static final String LOCAL_FILE_SCHEME = "file"; private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; private static final String SLASH = "/"; @@ -92,7 +91,7 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private final Map fileSystemProvidersMap = Maps.newHashMap(); - private String defaultFileSystemProviderScheme; + private FileSystemProvider defaultFileSystemProvider; HadoopCatalogOperations(EntityStore store) { this.store = store; @@ -133,21 +132,17 @@ public void initialize( (String) propertiesMetadata .catalogPropertiesMetadata() - .getOrDefault( - config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS_CLASSNAMES); - FileSystemUtils.initFileSystemProviders(fileSystemProviders, fileSystemProvidersMap); + .getOrDefault(config, HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS); + this.fileSystemProvidersMap.putAll(FileSystemUtils.getFileSystemProviders(fileSystemProviders)); - String defaultFileSystemProviderClassName = + String defaultFileSystemProviderName = (String) propertiesMetadata .catalogPropertiesMetadata() - .getOrDefault( - config, HadoopCatalogPropertiesMetadata.DEFAULT_FS_PROVIDER_CLASSNAME); - this.defaultFileSystemProviderScheme = - StringUtils.isNotBlank(defaultFileSystemProviderClassName) - ? FileSystemUtils.getSchemeByFileSystemProvider( - defaultFileSystemProviderClassName, fileSystemProvidersMap) - : LOCAL_FILE_SCHEME; + .getOrDefault(config, HadoopCatalogPropertiesMetadata.DEFAULT_FS_PROVIDER); + this.defaultFileSystemProvider = + FileSystemUtils.getFileSystemProviderByName( + fileSystemProvidersMap, defaultFileSystemProviderName); String catalogLocation = (String) @@ -766,14 +761,14 @@ FileSystem getFileSystem(Path path, Map config) throws IOExcepti String scheme = path.toUri().getScheme() != null ? path.toUri().getScheme() - : defaultFileSystemProviderScheme; + : defaultFileSystemProvider.scheme(); FileSystemProvider provider = fileSystemProvidersMap.get(scheme); if (provider == null) { throw new IllegalArgumentException( String.format( - "Unsupported scheme: %s, path: %s, all supported scheme: %s", - scheme, path, fileSystemProvidersMap.keySet())); + "Unsupported scheme: %s, path: %s, all supported scheme: %s and provider: %s", + scheme, path, fileSystemProvidersMap.keySet(), fileSystemProvidersMap.values())); } return provider.getFileSystem(path, config); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 06e4c579789..07f4228962d 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -18,10 +18,11 @@ */ package org.apache.gravitino.catalog.hadoop; +import static org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.KERBEROS_PROPERTY_ENTRIES; + import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.catalog.hadoop.authentication.AuthenticationConfig; -import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata; @@ -37,19 +38,18 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada public static final String LOCATION = "location"; /** - * The class names of {@link FileSystemProvider} to be added to the catalog. Except built-in + * The name of {@link FileSystemProvider} to be added to the catalog. Except built-in * FileSystemProvider like LocalFileSystemProvider and HDFSFileSystemProvider, users can add their - * own FileSystemProvider by specifying the class name here. The value can be - * 'xxxx.yyy.FileSystemProvider1,xxxx.yyy.FileSystemProvider2'. + * own FileSystemProvider by specifying the provider name here. The value can be + * find {@link FileSystemProvider#name()}. */ - public static final String FILESYSTEM_PROVIDERS_CLASSNAMES = "filesystem-providers-classnames"; + public static final String FILESYSTEM_PROVIDERS = "filesystem-providers"; /** * The default file system provider class name, used to create the default file system. If not - * specified, the default file system provider will be {@link LocalFileSystemProvider}. + * specified, the default file system provider will be {@link LocalFileSystemProvider#name()}. */ - public static final String DEFAULT_FS_PROVIDER_CLASSNAME = - "default-filesystem-provider-classname"; + public static final String DEFAULT_FS_PROVIDER = "default-filesystem-provider"; private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() @@ -62,23 +62,23 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada null, false /* hidden */)) .put( - FILESYSTEM_PROVIDERS_CLASSNAMES, + FILESYSTEM_PROVIDERS, PropertyEntry.stringOptionalPropertyEntry( - FILESYSTEM_PROVIDERS_CLASSNAMES, - "The file system provider class name, separated by comma", + FILESYSTEM_PROVIDERS, + "The file system provider names, separated by comma", false /* immutable */, null, false /* hidden */)) .put( - DEFAULT_FS_PROVIDER_CLASSNAME, + DEFAULT_FS_PROVIDER, PropertyEntry.stringOptionalPropertyEntry( - DEFAULT_FS_PROVIDER_CLASSNAME, - "Default file system provider, used to create the default file system", + DEFAULT_FS_PROVIDER, + "Default file system provider name", false /* immutable */, - LocalFileSystemProvider.class.getCanonicalName(), + LocalFileSystemProvider.class.getSimpleName(), false /* hidden */)) // The following two are about authentication. - .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES) + .putAll(KERBEROS_PROPERTY_ENTRIES) .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES) .build(); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java index bbfa60571af..1cd87dd8c49 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java @@ -52,10 +52,18 @@ FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map config throws IOException; /** - * Get the scheme of this FileSystem provider. The value is 'file' for LocalFileSystem, 'hdfs' for - * HDFS, 's3a' for S3AFileSystem, etc. + * Scheme of this FileSystem provider. The value is 'file' for LocalFileSystem, 'hdfs' for HDFS, + * etc. * - * @return The scheme of this FileSystem provider. + * @return The scheme of this FileSystem provider used. */ - String getScheme(); + String scheme(); + + /** + * Name of this FileSystem provider. The value is 'LocalFileSystemProvider' for LocalFileSystem, + * 'HDFSFileSystemProvider' for HDFS, etc. + * + * @return The name of this FileSystem provider. + */ + String name(); } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index cf68e005be9..ff6935e0810 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -18,50 +18,57 @@ */ package org.apache.gravitino.catalog.hadoop.fs; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import java.util.Arrays; import java.util.Map; -import org.apache.commons.lang3.StringUtils; -import org.apache.gravitino.exceptions.GravitinoRuntimeException; +import java.util.ServiceLoader; +import java.util.Set; public class FileSystemUtils { private FileSystemUtils() {} - public static void initFileSystemProviders( - String fileSystemProviders, Map fileProvidersMap) { - FileSystemProvider localFileSystemProvider = new LocalFileSystemProvider(); - FileSystemProvider hdfsFileSystemProvider = new HDFSFileSystemProvider(); - fileProvidersMap.put(localFileSystemProvider.getScheme(), localFileSystemProvider); - fileProvidersMap.put(hdfsFileSystemProvider.getScheme(), hdfsFileSystemProvider); + public static Map getFileSystemProviders(String fileSystemProviders) { + Map resultMap = Maps.newHashMap(); + ServiceLoader allFileSystemProviders = + ServiceLoader.load(FileSystemProvider.class); - if (StringUtils.isBlank(fileSystemProviders)) { - return; - } + Set providersInUses = + fileSystemProviders != null + ? Arrays.stream(fileSystemProviders.split(",")) + .map(String::trim) + .collect(java.util.stream.Collectors.toSet()) + : Sets.newHashSet(); - String[] providers = fileSystemProviders.split(","); - for (String provider : providers) { - try { - FileSystemProvider fileSystemProvider = - (FileSystemProvider) - Class.forName(provider.trim()).getDeclaredConstructor().newInstance(); - fileProvidersMap.put(fileSystemProvider.getScheme(), fileSystemProvider); - } catch (Exception e) { - throw new GravitinoRuntimeException( - e, "Failed to initialize file system provider: %s", provider); - } - } - } + // Always add the built-in LocalFileSystemProvider and HDFSFileSystemProvider to the catalog. + providersInUses.add(LocalFileSystemProvider.class.getName()); + providersInUses.add(HDFSFileSystemProvider.class.getName()); + + allFileSystemProviders.forEach( + fileSystemProvider -> { + if (providersInUses.contains(fileSystemProvider.getClass().getName())) { + if (resultMap.containsKey(fileSystemProvider.scheme())) { + throw new UnsupportedOperationException( + String.format( + "File system provider with scheme '%s' already exists in the use provider list " + + "Please make sure the file system provider scheme is unique.", + fileSystemProvider.name())); + } - public static String getSchemeByFileSystemProvider( - String providerClassName, Map fileProvidersMap) { - for (Map.Entry entry : fileProvidersMap.entrySet()) { - if (entry.getValue().getClass().getName().equals(providerClassName)) { - return entry.getKey(); - } - } + resultMap.put(fileSystemProvider.scheme(), fileSystemProvider); + } + }); + + return resultMap; + } - throw new UnsupportedOperationException( - String.format( - "File system provider class name '%s' not found. Supported file system providers: %s", - providerClassName, fileProvidersMap.values())); + public static FileSystemProvider getFileSystemProviderByName( + Map fileSystemProviders, String defaultFileSystemProvider) { + return fileSystemProviders.entrySet().stream() + .filter(entry -> entry.getValue().name().equals(defaultFileSystemProvider)) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index af679d19a79..3364d42a7e7 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -42,7 +42,12 @@ public FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map } @Override - public String getScheme() { + public String scheme() { return "hdfs"; } + + @Override + public String name() { + return "HDFSFileSystemProvider"; + } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 29ded3782af..6f8ac9c9214 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -41,7 +41,12 @@ public FileSystem getFileSystem(Path path, Map config) throws IO } @Override - public String getScheme() { + public String scheme() { return "file"; } + + @Override + public String name() { + return "LocalFileSystemProvider"; + } } diff --git a/catalogs/catalog-hadoop/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/catalogs/catalog-hadoop/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider new file mode 100644 index 00000000000..93a84744aa5 --- /dev/null +++ b/catalogs/catalog-hadoop/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.gravitino.catalog.hadoop.fs.HDFSFileSystemProvider +org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider \ No newline at end of file diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index d4e3722c755..05e769667da 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -134,7 +134,7 @@ public void initialize(URI name, Configuration configuration) throws IOException // Register the default local and HDFS FileSystemProvider String fileSystemProviders = configuration.get(FS_FILESYSTEM_PROVIDERS); - FileSystemUtils.initFileSystemProviders(fileSystemProviders, fileSystemProvidersMap); + fileSystemProvidersMap.putAll(FileSystemUtils.getFileSystemProviders(fileSystemProviders)); this.workingDirectory = new Path(name); this.uri = URI.create(name.getScheme() + "://" + name.getAuthority()); diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index 7a365a98920..6a50b0e6631 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -42,8 +42,7 @@ class GravitinoVirtualFileSystemConfiguration { * example: * * 

-   * fs.gvfs.filesystem.providers=org.apache.gravitino.catalog.hadoop.fs.XFileSystemProvider,
-   * org.apache.gravitino.catalog.hadoop.fs.YFileSystemProvider
+   * XFileSystemProvider, FileSystemProvider
    *
*/ public static final String FS_FILESYSTEM_PROVIDERS = "fs.gvfs.filesystem.providers"; diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index a195c615fc8..b310ffbf5bc 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -25,19 +25,19 @@ Hadoop 3. If there's any compatibility issue, please create an [issue](https://g Besides the [common catalog properties](./gravitino-server-config.md#gravitino-catalog-properties-configuration), the Hadoop catalog has the following properties: -| Property Name | Description | Default Value | Required | Since Version | -|-----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------|-------------------------------------------------------------|---------------| -| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | -| `filesystem-providers-classnames` | The fully qualified classnames of filesystem providers for the Hadoop catalog. Gravitino already support built-in `LocalFileSystemProvider`(`local file`) and `HDFSFileSystemProvider`(`hdfs`). If you want to support more file system and add it to Gravitino, you can implement `FileSystemProvider` and set this value | (none) | No | 0.7.0 | -| `default-filesystem-provider-classname` | The fully qualified classnames of default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider` | `org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider` | No | 0.7.0 | -| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | -| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | - -For more about `filesystem-providers-classnames`, please refer to `HadoopFileSystemProvider` or `LocalFileSystemProvider` in the source code. Furthermore, you also need to place the jar of the file system provider into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory if it's not in the classpath. +| Property Name | Description | Default Value | Required | Since Version | +|----------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------|-------------------------------------------------------------|---------------| +| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | +| `filesystem-providers` | The names (split by comma) of filesystem providers for the Hadoop catalog. Gravitino already support built-in `LocalFileSystemProvider`(`local file`) and `HDFSFileSystemProvider`(`hdfs`). If users want to support more file system and add it to Gravitino, they custom more file system by implementing `FileSystemProvider`. | (none) | No | 0.7.0 | +| `default-filesystem-provider` | The name default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `LocalFileSystemProvider` | `LocalFileSystemProvider` | No | 0.7.0 | +| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | +| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | + +For more about `filesystem-providers`, please refer to `HadoopFileSystemProvider` or `LocalFileSystemProvider` in the source code. Furthermore, you also need to place the jar of the file system provider into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory if it's not in the classpath. ### Authentication for Hadoop Catalog From 9dc0f5abd948596ffa74eb3c938cda63b0903060 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 11:41:20 +0800 Subject: [PATCH 57/89] Fix --- .../catalog/hadoop/HadoopCatalogPropertiesMetadata.java | 4 ++-- .../apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 07f4228962d..fab79d1c426 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -40,8 +40,8 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada /** * The name of {@link FileSystemProvider} to be added to the catalog. Except built-in * FileSystemProvider like LocalFileSystemProvider and HDFSFileSystemProvider, users can add their - * own FileSystemProvider by specifying the provider name here. The value can be - * find {@link FileSystemProvider#name()}. + * own FileSystemProvider by specifying the provider name here. The value can be find {@link + * FileSystemProvider#name()}. */ public static final String FILESYSTEM_PROVIDERS = "filesystem-providers"; diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index ff6935e0810..89bcff7322f 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -42,12 +42,12 @@ public static Map getFileSystemProviders(String file : Sets.newHashSet(); // Always add the built-in LocalFileSystemProvider and HDFSFileSystemProvider to the catalog. - providersInUses.add(LocalFileSystemProvider.class.getName()); - providersInUses.add(HDFSFileSystemProvider.class.getName()); + providersInUses.add(LocalFileSystemProvider.class.getSimpleName()); + providersInUses.add(HDFSFileSystemProvider.class.getSimpleName()); allFileSystemProviders.forEach( fileSystemProvider -> { - if (providersInUses.contains(fileSystemProvider.getClass().getName())) { + if (providersInUses.contains(fileSystemProvider.getClass().getSimpleName())) { if (resultMap.containsKey(fileSystemProvider.scheme())) { throw new UnsupportedOperationException( String.format( From 1fee1e4006b0975caa096a652b0940046222d675 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 11:51:39 +0800 Subject: [PATCH 58/89] Fix --- .../gcs => gcs/fs}/GCSFileSystemProvider.java | 9 +++++++-- ...itino.catalog.hadoop.fs.FileSystemProvider | 20 +++++++++++++++++++ .../integration/test/HadoopGCPCatalogIT.java | 5 ++--- 3 files changed, 29 insertions(+), 5 deletions(-) rename bundles/gcs-bundle/src/main/java/org/apache/gravitino/{fileset/gcs => gcs/fs}/GCSFileSystemProvider.java (91%) create mode 100644 bundles/gcs-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java similarity index 91% rename from bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java rename to bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java index 5a5b6edd5c0..42f97e7966e 100644 --- a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/fileset/gcs/GCSFileSystemProvider.java +++ b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.fileset.gcs; +package org.apache.gravitino.gcs.fs; import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; import java.io.IOException; @@ -39,7 +39,12 @@ public FileSystem getFileSystem(Path path, Map config) throws IO } @Override - public String getScheme() { + public String scheme() { return "gs"; } + + @Override + public String name() { + return "GCSFileSystemProvider"; + } } diff --git a/bundles/gcs-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/gcs-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider new file mode 100644 index 00000000000..8a65be70fd5 --- /dev/null +++ b/bundles/gcs-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.gravitino.gcs.fs.GCSFileSystemProvider \ No newline at end of file diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java index 5f61d96049e..aa45790c93a 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java @@ -18,7 +18,7 @@ */ package org.apache.gravitino.catalog.hadoop.integration.test; -import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS_CLASSNAMES; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; import com.google.common.collect.Maps; import java.io.IOException; @@ -86,8 +86,7 @@ protected void createCatalog() { Map map = Maps.newHashMap(); map.put("gravitino.bypass.fs.gs.auth.service.account.enable", "true"); map.put("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); - map.put( - FILESYSTEM_PROVIDERS_CLASSNAMES, "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider"); + map.put(FILESYSTEM_PROVIDERS, "GCSFileSystemProvider"); metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map); From 1789bd24c9dac46d9f44d0c3c282f5733bffa2ec Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 11:52:48 +0800 Subject: [PATCH 59/89] fix --- clients/client-python/tests/integration/test_gvfs_with_hdfs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index a91d7f93dcf..2722aeb1da7 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -375,7 +375,6 @@ def test_mv(self): with self.assertRaises(GravitinoRuntimeException): fs.mv(self.fileset_gvfs_location, self.fileset_gvfs_location + "/test_mv") - @unittest.skip("Not implemented") def test_rm(self): rm_dir = self.fileset_gvfs_location + "/test_rm" rm_actual_dir = self.fileset_storage_location + "/test_rm" @@ -640,7 +639,6 @@ def test_get_file(self): with self.assertRaises(GravitinoRuntimeException): fs.get_file(get_file, remote_path) - @unittest.skip("Not implemented") def test_pandas(self): pands_dir = self.fileset_gvfs_location + "/test_pandas" pands_actual_dir = self.fileset_storage_location + "/test_pandas" @@ -687,7 +685,6 @@ def test_pandas(self): ds2 = pandas.read_csv(csv_file, storage_options=storage_options) self.assertTrue(data.equals(ds2)) - @unittest.skip("Not implemented") def test_pyarrow(self): pyarrow_dir = self.fileset_gvfs_location + "/test_pyarrow" pyarrow_actual_dir = self.fileset_storage_location + "/test_pyarrow" From 2ee1709283614f240dcced85726d429b27ef3d22 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 13:03:47 +0800 Subject: [PATCH 60/89] Fix --- .../HadoopCatalogPropertiesMetadata.java | 3 ++- .../catalog/hadoop/fs/FileSystemProvider.java | 4 ++-- .../catalog/hadoop/fs/FileSystemUtils.java | 5 ++--- .../hadoop/fs/HDFSFileSystemProvider.java | 2 +- .../hadoop/fs/LocalFileSystemProvider.java | 2 +- docs/hadoop-catalog.md | 22 +++++++++---------- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index fab79d1c426..42a8bab139e 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -47,7 +47,8 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada /** * The default file system provider class name, used to create the default file system. If not - * specified, the default file system provider will be {@link LocalFileSystemProvider#name()}. + * specified, the default file system provider will be {@link LocalFileSystemProvider#name()}: + * 'builtin-local'. */ public static final String DEFAULT_FS_PROVIDER = "default-filesystem-provider"; diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java index 1cd87dd8c49..5bee821e505 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java @@ -60,8 +60,8 @@ FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map config String scheme(); /** - * Name of this FileSystem provider. The value is 'LocalFileSystemProvider' for LocalFileSystem, - * 'HDFSFileSystemProvider' for HDFS, etc. + * Name of this FileSystem provider. The value is 'builtin-local' for LocalFileSystem, + * 'builtin-hdfs' for HDFS, etc. * * @return The name of this FileSystem provider. */ diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 89bcff7322f..61443a02b23 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -40,7 +40,6 @@ public static Map getFileSystemProviders(String file .map(String::trim) .collect(java.util.stream.Collectors.toSet()) : Sets.newHashSet(); - // Always add the built-in LocalFileSystemProvider and HDFSFileSystemProvider to the catalog. providersInUses.add(LocalFileSystemProvider.class.getSimpleName()); providersInUses.add(HDFSFileSystemProvider.class.getSimpleName()); @@ -64,9 +63,9 @@ public static Map getFileSystemProviders(String file } public static FileSystemProvider getFileSystemProviderByName( - Map fileSystemProviders, String defaultFileSystemProvider) { + Map fileSystemProviders, String fileSystemProviderName) { return fileSystemProviders.entrySet().stream() - .filter(entry -> entry.getValue().name().equals(defaultFileSystemProvider)) + .filter(entry -> entry.getValue().name().equals(fileSystemProviderName)) .map(Map.Entry::getValue) .findFirst() .orElse(null); diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index 3364d42a7e7..f7ec556284c 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -48,6 +48,6 @@ public String scheme() { @Override public String name() { - return "HDFSFileSystemProvider"; + return "builtin-hdfs"; } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 6f8ac9c9214..194b34246b3 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -47,6 +47,6 @@ public String scheme() { @Override public String name() { - return "LocalFileSystemProvider"; + return "builtin-local"; } } diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index b310ffbf5bc..ba5ce9efe89 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -25,17 +25,17 @@ Hadoop 3. If there's any compatibility issue, please create an [issue](https://g Besides the [common catalog properties](./gravitino-server-config.md#gravitino-catalog-properties-configuration), the Hadoop catalog has the following properties: -| Property Name | Description | Default Value | Required | Since Version | -|----------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------|-------------------------------------------------------------|---------------| -| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | -| `filesystem-providers` | The names (split by comma) of filesystem providers for the Hadoop catalog. Gravitino already support built-in `LocalFileSystemProvider`(`local file`) and `HDFSFileSystemProvider`(`hdfs`). If users want to support more file system and add it to Gravitino, they custom more file system by implementing `FileSystemProvider`. | (none) | No | 0.7.0 | -| `default-filesystem-provider` | The name default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `LocalFileSystemProvider` | `LocalFileSystemProvider` | No | 0.7.0 | -| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | -| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | +| Property Name | Description | Default Value | Required | Since Version | +|----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|-------------------------------------------------------------|---------------| +| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | +| `filesystem-providers` | The names (split by comma) of filesystem providers for the Hadoop catalog. Gravitino already support built-in `builtin-local`(`local file`) and `builtin-hdfs`(`hdfs`). If users want to support more file system and add it to Gravitino, they custom more file system by implementing `FileSystemProvider`. | (none) | No | 0.7.0 | +| `default-filesystem-provider` | The name default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `builtin-local` | `builtin-local` | No | 0.7.0 | +| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | +| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | For more about `filesystem-providers`, please refer to `HadoopFileSystemProvider` or `LocalFileSystemProvider` in the source code. Furthermore, you also need to place the jar of the file system provider into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory if it's not in the classpath. From 05e5d20dcfed4a77c2ad0c93c22a1256aec87ef3 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 13:41:52 +0800 Subject: [PATCH 61/89] Fix --- .../HadoopCatalogPropertiesMetadata.java | 2 +- .../catalog/hadoop/fs/FileSystemUtils.java | 30 +++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 42a8bab139e..4f59f1a7dab 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -76,7 +76,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada DEFAULT_FS_PROVIDER, "Default file system provider name", false /* immutable */, - LocalFileSystemProvider.class.getSimpleName(), + "builtin-local", // please see LocalFileSystemProvider#name() false /* hidden */)) // The following two are about authentication. .putAll(KERBEROS_PROPERTY_ENTRIES) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 61443a02b23..261c3dbd6a9 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.ServiceLoader; import java.util.Set; +import java.util.stream.Collectors; public class FileSystemUtils { @@ -40,13 +41,11 @@ public static Map getFileSystemProviders(String file .map(String::trim) .collect(java.util.stream.Collectors.toSet()) : Sets.newHashSet(); - // Always add the built-in LocalFileSystemProvider and HDFSFileSystemProvider to the catalog. - providersInUses.add(LocalFileSystemProvider.class.getSimpleName()); - providersInUses.add(HDFSFileSystemProvider.class.getSimpleName()); + // Only get the file system providers that are in the use list. allFileSystemProviders.forEach( fileSystemProvider -> { - if (providersInUses.contains(fileSystemProvider.getClass().getSimpleName())) { + if (providersInUses.contains(fileSystemProvider.name())) { if (resultMap.containsKey(fileSystemProvider.scheme())) { throw new UnsupportedOperationException( String.format( @@ -54,11 +53,32 @@ public static Map getFileSystemProviders(String file + "Please make sure the file system provider scheme is unique.", fileSystemProvider.name())); } - resultMap.put(fileSystemProvider.scheme(), fileSystemProvider); } }); + // Always add the built-in LocalFileSystemProvider and HDFSFileSystemProvider to the catalog. + FileSystemProvider builtInLocalFileSystemProvider = new LocalFileSystemProvider(); + FileSystemProvider builtInHDFSFileSystemProvider = new HDFSFileSystemProvider(); + resultMap.put(builtInLocalFileSystemProvider.scheme(), builtInLocalFileSystemProvider); + resultMap.put(builtInHDFSFileSystemProvider.scheme(), builtInHDFSFileSystemProvider); + + // If not all providersInUses was found, throw an exception. + Set notFoundProviders = + Sets.difference( + providersInUses, + resultMap.values().stream() + .map(FileSystemProvider::name) + .collect(Collectors.toSet())) + .immutableCopy(); + if (!notFoundProviders.isEmpty()) { + throw new UnsupportedOperationException( + String.format( + "File system providers %s not found in the classpath. Please make sure the file system " + + "provider is in the classpath.", + notFoundProviders)); + } + return resultMap; } From 8f28211ed88318cc5f644f0a29fb8f390421d74e Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 14:24:45 +0800 Subject: [PATCH 62/89] Fix --- catalogs/catalog-hadoop/build.gradle.kts | 2 + .../HadoopCatalogPropertiesMetadata.java | 5 ++- .../catalog/hadoop/fs/FileSystemUtils.java | 43 ++++++++++--------- .../hadoop/fs/HDFSFileSystemProvider.java | 3 +- .../hadoop/fs/LocalFileSystemProvider.java | 3 +- ...avitinoVirtualFileSystemConfiguration.java | 11 ++--- 6 files changed, 36 insertions(+), 31 deletions(-) diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index ba60a161d8f..94028934721 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -36,6 +36,8 @@ dependencies { exclude(group = "*") } + compileOnly(libs.guava) + implementation(libs.hadoop3.common) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 4f59f1a7dab..397e13aa4af 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -52,6 +52,9 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada */ public static final String DEFAULT_FS_PROVIDER = "default-filesystem-provider"; + public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local"; + public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs"; + private static final Map> HADOOP_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() .put( @@ -76,7 +79,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada DEFAULT_FS_PROVIDER, "Default file system provider name", false /* immutable */, - "builtin-local", // please see LocalFileSystemProvider#name() + BUILTIN_LOCAL_FS_PROVIDER, // please see LocalFileSystemProvider#name() false /* hidden */)) // The following two are about authentication. .putAll(KERBEROS_PROPERTY_ENTRIES) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 261c3dbd6a9..7c20b37151b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -18,8 +18,12 @@ */ package org.apache.gravitino.catalog.hadoop.fs; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_HDFS_FS_PROVIDER; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_LOCAL_FS_PROVIDER; + import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.collect.Streams; import java.util.Arrays; import java.util.Map; import java.util.ServiceLoader; @@ -42,28 +46,27 @@ public static Map getFileSystemProviders(String file .collect(java.util.stream.Collectors.toSet()) : Sets.newHashSet(); - // Only get the file system providers that are in the use list. - allFileSystemProviders.forEach( - fileSystemProvider -> { - if (providersInUses.contains(fileSystemProvider.name())) { - if (resultMap.containsKey(fileSystemProvider.scheme())) { - throw new UnsupportedOperationException( - String.format( - "File system provider with scheme '%s' already exists in the use provider list " - + "Please make sure the file system provider scheme is unique.", - fileSystemProvider.name())); - } - resultMap.put(fileSystemProvider.scheme(), fileSystemProvider); - } - }); + // Add built-in file system providers to the use list automatically. + providersInUses.add(BUILTIN_LOCAL_FS_PROVIDER); + providersInUses.add(BUILTIN_HDFS_FS_PROVIDER); - // Always add the built-in LocalFileSystemProvider and HDFSFileSystemProvider to the catalog. - FileSystemProvider builtInLocalFileSystemProvider = new LocalFileSystemProvider(); - FileSystemProvider builtInHDFSFileSystemProvider = new HDFSFileSystemProvider(); - resultMap.put(builtInLocalFileSystemProvider.scheme(), builtInLocalFileSystemProvider); - resultMap.put(builtInHDFSFileSystemProvider.scheme(), builtInHDFSFileSystemProvider); + // Only get the file system providers that are in the user list and check if the scheme is + // unique. + Streams.stream(allFileSystemProviders.iterator()) + .filter(fileSystemProvider -> providersInUses.contains(fileSystemProvider.name())) + .forEach( + fileSystemProvider -> { + if (resultMap.containsKey(fileSystemProvider.scheme())) { + throw new UnsupportedOperationException( + String.format( + "File system provider: '%s' with scheme '%s' already exists in the use provider list " + + "Please make sure the file system provider scheme is unique.", + fileSystemProvider.getClass().getName(), fileSystemProvider.scheme())); + } + resultMap.put(fileSystemProvider.scheme(), fileSystemProvider); + }); - // If not all providersInUses was found, throw an exception. + // If not all file system providers in providersInUses was found, throw an exception. Set notFoundProviders = Sets.difference( providersInUses, diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index f7ec556284c..7c9ceebdd36 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.catalog.hadoop.fs; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_HDFS_FS_PROVIDER; import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; import java.io.IOException; @@ -48,6 +49,6 @@ public String scheme() { @Override public String name() { - return "builtin-hdfs"; + return BUILTIN_HDFS_FS_PROVIDER; } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 194b34246b3..70e44c76f6b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.catalog.hadoop.fs; +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_LOCAL_FS_PROVIDER; import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; import java.io.IOException; @@ -47,6 +48,6 @@ public String scheme() { @Override public String name() { - return "builtin-local"; + return BUILTIN_LOCAL_FS_PROVIDER; } } diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index 6a50b0e6631..cd1ecb92fa8 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -36,14 +36,9 @@ class GravitinoVirtualFileSystemConfiguration { public static final String FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY = "fs.gravitino.client.authType"; /** - * Full class name of file systems that implement {@link FileSystemProvider}` spilt by a comma. - * - *

This configuration is used to register file system providers to the gvfs file system. For - * example: - * - * 

-   * XFileSystemProvider, FileSystemProvider
-   *
+ * File system provider names configuration key. The value is a comma separated list of file + * system provider name which is defined in the service loader. Users can custom their own file + * system by implementing the {@link FileSystemProvider} interface. */ public static final String FS_FILESYSTEM_PROVIDERS = "fs.gvfs.filesystem.providers"; From 35cba1ea1821f029cd0aa8771b30ad7dec2d4940 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 14:27:31 +0800 Subject: [PATCH 63/89] Fix --- docs/hadoop-catalog.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index ba5ce9efe89..d28e6d93b04 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -25,17 +25,17 @@ Hadoop 3. If there's any compatibility issue, please create an [issue](https://g Besides the [common catalog properties](./gravitino-server-config.md#gravitino-catalog-properties-configuration), the Hadoop catalog has the following properties: -| Property Name | Description | Default Value | Required | Since Version | -|----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|-------------------------------------------------------------|---------------| -| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | -| `filesystem-providers` | The names (split by comma) of filesystem providers for the Hadoop catalog. Gravitino already support built-in `builtin-local`(`local file`) and `builtin-hdfs`(`hdfs`). If users want to support more file system and add it to Gravitino, they custom more file system by implementing `FileSystemProvider`. | (none) | No | 0.7.0 | -| `default-filesystem-provider` | The name default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `builtin-local` | `builtin-local` | No | 0.7.0 | -| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | -| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | +| Property Name | Description | Default Value | Required | Since Version | +|----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|-------------------------------------------------------------|------------------| +| `location` | The storage location managed by Hadoop catalog. | (none) | No | 0.5.0 | +| `filesystem-providers` | The names (split by comma) of filesystem providers for the Hadoop catalog. Gravitino already support built-in `builtin-local`(`local file`) and `builtin-hdfs`(`hdfs`). If users want to support more file system and add it to Gravitino, they custom more file system by implementing `FileSystemProvider`. | (none) | No | 0.7.0-incubating | +| `default-filesystem-provider` | The name default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `builtin-local` | `builtin-local` | No | 0.7.0-incubating | +| `authentication.impersonation-enable` | Whether to enable impersonation for the Hadoop catalog. | `false` | No | 0.5.1 | +| `authentication.type` | The type of authentication for Hadoop catalog, currently we only support `kerberos`, `simple`. | `simple` | No | 0.5.1 | +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.5.1 | +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Hadoop catalog. | 60 | No | 0.5.1 | +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.5.1 | For more about `filesystem-providers`, please refer to `HadoopFileSystemProvider` or `LocalFileSystemProvider` in the source code. Furthermore, you also need to place the jar of the file system provider into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory if it's not in the classpath. From bcf2f1201f333b16a273dad33741a87b749252d9 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 15:48:12 +0800 Subject: [PATCH 64/89] Fix --- .../hadoop/HadoopCatalogOperations.java | 2 +- .../catalog/hadoop/fs/FileSystemUtils.java | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index dedc050e5c1..8515ea7d20f 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -767,7 +767,7 @@ FileSystem getFileSystem(Path path, Map config) throws IOExcepti if (provider == null) { throw new IllegalArgumentException( String.format( - "Unsupported scheme: %s, path: %s, all supported scheme: %s and provider: %s", + "Unsupported scheme: %s, path: %s, all supported schemes: %s and providers: %s", scheme, path, fileSystemProvidersMap.keySet(), fileSystemProvidersMap.values())); } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 7c20b37151b..3a959ff3738 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -25,6 +25,7 @@ import com.google.common.collect.Sets; import com.google.common.collect.Streams; import java.util.Arrays; +import java.util.Locale; import java.util.Map; import java.util.ServiceLoader; import java.util.Set; @@ -42,18 +43,20 @@ public static Map getFileSystemProviders(String file Set providersInUses = fileSystemProviders != null ? Arrays.stream(fileSystemProviders.split(",")) - .map(String::trim) + .map(f -> f.trim().toLowerCase(Locale.ROOT)) .collect(java.util.stream.Collectors.toSet()) : Sets.newHashSet(); // Add built-in file system providers to the use list automatically. - providersInUses.add(BUILTIN_LOCAL_FS_PROVIDER); - providersInUses.add(BUILTIN_HDFS_FS_PROVIDER); + providersInUses.add(BUILTIN_LOCAL_FS_PROVIDER.toLowerCase(Locale.ROOT)); + providersInUses.add(BUILTIN_HDFS_FS_PROVIDER.toLowerCase(Locale.ROOT)); // Only get the file system providers that are in the user list and check if the scheme is // unique. Streams.stream(allFileSystemProviders.iterator()) - .filter(fileSystemProvider -> providersInUses.contains(fileSystemProvider.name())) + .filter( + fileSystemProvider -> + providersInUses.contains(fileSystemProvider.name().toLowerCase(Locale.ROOT))) .forEach( fileSystemProvider -> { if (resultMap.containsKey(fileSystemProvider.scheme())) { @@ -71,7 +74,7 @@ public static Map getFileSystemProviders(String file Sets.difference( providersInUses, resultMap.values().stream() - .map(FileSystemProvider::name) + .map(p -> p.name().toLowerCase(Locale.ROOT)) .collect(Collectors.toSet())) .immutableCopy(); if (!notFoundProviders.isEmpty()) { @@ -91,6 +94,11 @@ public static FileSystemProvider getFileSystemProviderByName( .filter(entry -> entry.getValue().name().equals(fileSystemProviderName)) .map(Map.Entry::getValue) .findFirst() - .orElse(null); + .orElseThrow( + () -> + new UnsupportedOperationException( + String.format( + "File system provider with name '%s' not found in the file system provider list.", + fileSystemProviderName))); } } From f25a37d0ccad534ebf8b9fcd607154659fb35543 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 17:49:07 +0800 Subject: [PATCH 65/89] Fix a problem --- .../tests/integration/integration_test_env.py | 6 ++++++ .../client-python/tests/integration/test_catalog.py | 2 +- .../tests/integration/test_gvfs_with_gcs.py | 2 ++ .../tests/integration/test_gvfs_with_hdfs.py | 10 ++++++++-- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/clients/client-python/tests/integration/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py index cfe6c0eda09..50a4fd5bdd5 100644 --- a/clients/client-python/tests/integration/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -141,6 +141,12 @@ def restart_server(cls): "project root directory." ) + # remove data dir under gravitino_home + data_dir = os.path.join(gravitino_home, "data") + if os.path.exists(data_dir): + logger.info("Remove Gravitino data directory: %s", data_dir) + subprocess.run(["rm", "-rf", data_dir], check=False) + # Restart Gravitino Server env_vars = os.environ.copy() env_vars["HADOOP_USER_NAME"] = "anonymous" diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 71caafbc206..ec0380b457a 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -39,7 +39,7 @@ class TestCatalog(IntegrationTestEnv): metalake_name: str = "TestSchema_metalake" + str(randint(1, 10000)) - catalog_name: str = "testCatalog" + catalog_name: str = "testCatalog" + str(randint(1, 10000)) catalog_comment: str = "catalogComment" catalog_location_prop: str = "location" # Fileset Catalog must set `location` catalog_provider: str = "hadoop" diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index d694941bd63..4311c11c6a4 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -16,6 +16,7 @@ # under the License. import os +from random import randint from fsspec.implementations.arrow import ArrowFSWrapper from pyarrow.fs import GcsFileSystem @@ -31,6 +32,7 @@ class TestGvfsWithGCS(TestGvfsWithHDFS): key_file = "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" bucket_name = "example_qazwsx" + metalake_name: str = "TestGvfsWithGCS_metalake" + str(randint(1, 10000)) def setUp(self): self.options = {"gravitino.bypass.gcs.service-account-key-path": self.key_file} diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 2722aeb1da7..c4157437a6f 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -390,7 +390,7 @@ def test_rm(self): rm_file = self.fileset_gvfs_location + "/test_rm/test.file" rm_actual_file = self.fileset_storage_location + "/test_rm/test.file" - self.fs.touch(rm_file) + fs.touch(rm_file) self.assertTrue(self.fs.exists(rm_actual_file)) self.assertTrue(fs.exists(rm_file)) @@ -639,6 +639,9 @@ def test_get_file(self): with self.assertRaises(GravitinoRuntimeException): fs.get_file(get_file, remote_path) + @unittest.skip( + "This test will fail for https://github.com/apache/arrow/issues/44438" + ) def test_pandas(self): pands_dir = self.fileset_gvfs_location + "/test_pandas" pands_actual_dir = self.fileset_storage_location + "/test_pandas" @@ -668,7 +671,7 @@ def test_pandas(self): storage_options = { "server_uri": "http://localhost:8090", "metalake_name": self.metalake_name, - "options": self.options + "options": self.options, } # to csv csv_file = self.fileset_gvfs_location + "/test_pandas/test.csv" @@ -685,6 +688,9 @@ def test_pandas(self): ds2 = pandas.read_csv(csv_file, storage_options=storage_options) self.assertTrue(data.equals(ds2)) + @unittest.skip( + "This test will fail for https://github.com/apache/arrow/issues/44438" + ) def test_pyarrow(self): pyarrow_dir = self.fileset_gvfs_location + "/test_pyarrow" pyarrow_actual_dir = self.fileset_storage_location + "/test_pyarrow" From 27a911aba3264c9b78c4362a72ceaf9d50340ea1 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 18:49:27 +0800 Subject: [PATCH 66/89] fix --- .../java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java | 2 +- .../catalog/hadoop/integration/test/HadoopGCPCatalogIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java index 42f97e7966e..919baa03b19 100644 --- a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java +++ b/bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java @@ -45,6 +45,6 @@ public String scheme() { @Override public String name() { - return "GCSFileSystemProvider"; + return "gcs"; } } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java index aa45790c93a..8179f8e3a6e 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java @@ -86,7 +86,7 @@ protected void createCatalog() { Map map = Maps.newHashMap(); map.put("gravitino.bypass.fs.gs.auth.service.account.enable", "true"); map.put("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); - map.put(FILESYSTEM_PROVIDERS, "GCSFileSystemProvider"); + map.put(FILESYSTEM_PROVIDERS, "gcs"); metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map); From 6bae7e58f4034d0b826839e7fca62351d0060186 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 19:22:29 +0800 Subject: [PATCH 67/89] Fix a problem --- .../tests/integration/test_gvfs_with_gcs.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 4311c11c6a4..a632a2a7837 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -103,3 +103,24 @@ def _init_test_entities(cls): os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.key_file arrow_gcs_fs = GcsFileSystem() cls.fs = ArrowFSWrapper(arrow_gcs_fs) + + + def test_modified(self): + modified_dir = self.fileset_gvfs_location + "/test_modified" + modified_actual_dir = self.fileset_storage_location + "/test_modified" + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:8090", + metalake_name=self.metalake_name, + options=self.options, + **self.conf, + ) + self.fs.mkdir(modified_actual_dir) + self.assertTrue(self.fs.exists(modified_actual_dir)) + self.assertTrue(fs.exists(modified_dir)) + + # Disable the following test case as it is not working for GCS + # >>> gcs.mkdir('example_qazwsx/catalog/schema/fileset3') + # >>> r = gcs.modified('example_qazwsx/catalog/schema/fileset3') + # >>> print(r) + # None + # self.assertIsNotNone(fs.modified(modified_dir)) From fe13f5ea656dd221db6081603ce27fc92cc0d014 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 19:24:13 +0800 Subject: [PATCH 68/89] Fix a problem --- clients/client-python/tests/integration/test_gvfs_with_gcs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index a632a2a7837..5c108c7c693 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -104,7 +104,6 @@ def _init_test_entities(cls): arrow_gcs_fs = GcsFileSystem() cls.fs = ArrowFSWrapper(arrow_gcs_fs) - def test_modified(self): modified_dir = self.fileset_gvfs_location + "/test_modified" modified_actual_dir = self.fileset_storage_location + "/test_modified" From 0181632df45418df5a90bbda7fcca8a63164dc09 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 19:26:21 +0800 Subject: [PATCH 69/89] Fix a problem --- clients/client-python/tests/integration/test_gvfs_with_gcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 5c108c7c693..f9427385f45 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -23,6 +23,7 @@ from tests.integration.test_gvfs_with_hdfs import TestGvfsWithHDFS from gravitino import ( + gvfs, GravitinoClient, Catalog, Fileset, From 3ff9eef03233cd7517c4a4a293c50e9f8d40879d Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 19:42:02 +0800 Subject: [PATCH 70/89] Fix --- clients/client-python/tests/integration/test_gvfs_with_hdfs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index c4157437a6f..ac8f9e20d92 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -210,6 +210,7 @@ def _clean_test_data(cls): logger.warning("Failed to drop metalake %s", cls.metalake_name) def test_simple_auth(self): + options = {"auth_type": "simple"} current_user = ( None if os.environ.get("user.name") is None else os.environ["user.name"] ) @@ -218,7 +219,7 @@ def test_simple_auth(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, - options=self.options, + options=options, ) token = fs._client._rest_client.auth_data_provider.get_token_data() token_string = base64.b64decode( From 2ce660c81669a029d347e784f4eb5333b83b655b Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 20:26:16 +0800 Subject: [PATCH 71/89] Fix --- .../tests/integration/integration_test_env.py | 6 ++++++ .../tests/integration/test_catalog.py | 19 +++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/clients/client-python/tests/integration/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py index 50a4fd5bdd5..f54da0915d5 100644 --- a/clients/client-python/tests/integration/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -80,6 +80,12 @@ def setUpClass(cls): ) sys.exit(0) + # remove data dir under gravitino_home + data_dir = os.path.join(gravitino_home, "data") + if os.path.exists(data_dir): + logger.info("Remove Gravitino data directory: %s", data_dir) + subprocess.run(["rm", "-rf", data_dir], check=False) + logger.info("Starting integration test environment...") # Start Gravitino Server diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index ec0380b457a..6f4640936c4 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -40,6 +40,7 @@ class TestCatalog(IntegrationTestEnv): metalake_name: str = "TestSchema_metalake" + str(randint(1, 10000)) catalog_name: str = "testCatalog" + str(randint(1, 10000)) + catalog_name_bak = catalog_name catalog_comment: str = "catalogComment" catalog_location_prop: str = "location" # Fileset Catalog must set `location` catalog_provider: str = "hadoop" @@ -80,21 +81,25 @@ def clean_test_data(self): ) try: logger.info( - "Drop catalog %s[%s]", + "TestCatalog: drop catalog %s[%s]", self.catalog_ident, self.gravitino_client.drop_catalog(name=self.catalog_name), ) except GravitinoRuntimeException: - logger.warning("Failed to drop catalog %s", self.catalog_name) + logger.warning("TestCatalog: failed to drop catalog %s", self.catalog_name) try: logger.info( - "Drop metalake %s[%s]", + "TestCatalog: drop metalake %s[%s]", self.metalake_name, self.gravitino_admin_client.drop_metalake(self.metalake_name), ) except GravitinoRuntimeException: - logger.warning("Failed to drop metalake %s", self.metalake_name) + logger.warning( + "TestCatalog: failed to drop metalake %s", self.metalake_name + ) + + self.catalog_name = self.catalog_name_bak def test_list_catalogs(self): self.create_catalog(self.catalog_name) @@ -102,6 +107,12 @@ def test_list_catalogs(self): self.assertTrue(self.catalog_name in catalog_names) def test_create_catalog(self): + try: + self.gravitino_client.load_catalog(self.catalog_name) + except NoSuchCatalogException: + logger.info("TestCatalog: Catalog %s does not exist", self.catalog_name) + + self.gravitino_client.load_catalog(self.catalog_name) catalog = self.create_catalog(self.catalog_name) self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual( From f0fa87bedcc93b5d9eb345bcc995098213126e63 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 20:28:59 +0800 Subject: [PATCH 72/89] Fix --- clients/client-python/tests/integration/integration_test_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/client-python/tests/integration/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py index f54da0915d5..db53c16fa1d 100644 --- a/clients/client-python/tests/integration/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -81,7 +81,7 @@ def setUpClass(cls): sys.exit(0) # remove data dir under gravitino_home - data_dir = os.path.join(gravitino_home, "data") + data_dir = os.path.join(cls.gravitino_home, "data") if os.path.exists(data_dir): logger.info("Remove Gravitino data directory: %s", data_dir) subprocess.run(["rm", "-rf", data_dir], check=False) From d2921a827c8510be5b2620229793497ef6ec144b Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 20:43:23 +0800 Subject: [PATCH 73/89] Fix --- .../client-python/gravitino/client/gravitino_client.py | 10 ++++++++++ .../client-python/tests/integration/test_catalog.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/clients/client-python/gravitino/client/gravitino_client.py b/clients/client-python/gravitino/client/gravitino_client.py index 4e2b064caca..c6da24e76e5 100644 --- a/clients/client-python/gravitino/client/gravitino_client.py +++ b/clients/client-python/gravitino/client/gravitino_client.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import logging from typing import List, Dict from gravitino.api.catalog import Catalog @@ -23,6 +24,7 @@ from gravitino.client.gravitino_client_base import GravitinoClientBase from gravitino.client.gravitino_metalake import GravitinoMetalake +logger = logging.getLogger(__name__) class GravitinoClient(GravitinoClientBase): """Gravitino Client for a user to interact with the Gravitino API, allowing the client to list, @@ -82,6 +84,14 @@ def create_catalog( comment: str, properties: Dict[str, str], ) -> Catalog: + logger.info( + "Creating catalog %s with type %s, provider %s, comment %s, properties %s", + name, + catalog_type, + provider, + comment, + properties, + ) return self.get_metalake().create_catalog( name, catalog_type, provider, comment, properties ) diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 6f4640936c4..acecfefa45c 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -112,8 +112,8 @@ def test_create_catalog(self): except NoSuchCatalogException: logger.info("TestCatalog: Catalog %s does not exist", self.catalog_name) - self.gravitino_client.load_catalog(self.catalog_name) catalog = self.create_catalog(self.catalog_name) + logger.info("TestCatalog: Catalog %s created, properties: %s", catalog.name(), catalog.properties) self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual( catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} From dc68dd19b1525cedc7189702d6ea38e59566f346 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 21:05:44 +0800 Subject: [PATCH 74/89] Fix --- .../tests/integration/test_gvfs_with_gcs.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index f9427385f45..39309628397 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -15,12 +15,14 @@ # specific language governing permissions and limitations # under the License. +import logging import os from random import randint from fsspec.implementations.arrow import ArrowFSWrapper from pyarrow.fs import GcsFileSystem +from gravitino.exceptions.base import GravitinoRuntimeException from tests.integration.test_gvfs_with_hdfs import TestGvfsWithHDFS from gravitino import ( gvfs, @@ -29,6 +31,7 @@ Fileset, ) +logger = logging.getLogger(__name__) class TestGvfsWithGCS(TestGvfsWithHDFS): key_file = "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" @@ -46,7 +49,6 @@ def setUpClass(cls): cls._get_gravitino_home() cls.hadoop_conf_path = f"{cls.gravitino_home}/catalogs/hadoop/conf/hadoop.conf" - # append the hadoop conf to server # restart the server cls.restart_server() @@ -61,6 +63,28 @@ def tearDownClass(cls): # restart server cls.restart_server() + + # clear all config in the conf_path + @classmethod + def _reset_conf(cls, config, conf_path): + logger.info("Reset %s.", conf_path) + if not os.path.exists(conf_path): + raise GravitinoRuntimeException( + f"Conf file is not found at `{conf_path}`.") + filtered_lines = [] + with open(conf_path, mode="r", encoding="utf-8") as file: + origin_lines = file.readlines() + + for line in origin_lines: + line = line.strip() + if line.startswith("#"): + # append annotations directly + filtered_lines.append(line + "\n") + + with open(conf_path, mode="w", encoding="utf-8") as file: + for line in filtered_lines: + file.write(line) + @classmethod def _init_test_entities(cls): cls.gravitino_admin_client.create_metalake( From 6431e2fbfea6ed67705600d9e11f8c1f28fbfb6e Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 21:21:13 +0800 Subject: [PATCH 75/89] Fix --- .../tests/integration/test_gvfs_with_gcs.py | 15 ++++++++++++++- .../tests/integration/test_gvfs_with_hdfs.py | 6 ------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 39309628397..9ed2ddffc35 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -22,7 +22,6 @@ from fsspec.implementations.arrow import ArrowFSWrapper from pyarrow.fs import GcsFileSystem -from gravitino.exceptions.base import GravitinoRuntimeException from tests.integration.test_gvfs_with_hdfs import TestGvfsWithHDFS from gravitino import ( gvfs, @@ -30,6 +29,8 @@ Catalog, Fileset, ) +from gravitino.exceptions.base import GravitinoRuntimeException + logger = logging.getLogger(__name__) @@ -148,3 +149,15 @@ def test_modified(self): # >>> print(r) # None # self.assertIsNotNone(fs.modified(modified_dir)) + + @unittest.skip( + "This test will fail for https://github.com/apache/arrow/issues/44438" + ) + def test_pandas(self): + pass + + @unittest.skip( + "This test will fail for https://github.com/apache/arrow/issues/44438" + ) + def test_pyarrow(self): + pass diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index ac8f9e20d92..6b011031583 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -640,9 +640,6 @@ def test_get_file(self): with self.assertRaises(GravitinoRuntimeException): fs.get_file(get_file, remote_path) - @unittest.skip( - "This test will fail for https://github.com/apache/arrow/issues/44438" - ) def test_pandas(self): pands_dir = self.fileset_gvfs_location + "/test_pandas" pands_actual_dir = self.fileset_storage_location + "/test_pandas" @@ -689,9 +686,6 @@ def test_pandas(self): ds2 = pandas.read_csv(csv_file, storage_options=storage_options) self.assertTrue(data.equals(ds2)) - @unittest.skip( - "This test will fail for https://github.com/apache/arrow/issues/44438" - ) def test_pyarrow(self): pyarrow_dir = self.fileset_gvfs_location + "/test_pyarrow" pyarrow_actual_dir = self.fileset_storage_location + "/test_pyarrow" From 242888f10800cf936c6062c6246ebb913108ab3f Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 21:25:55 +0800 Subject: [PATCH 76/89] Fix --- clients/client-python/tests/integration/test_gvfs_with_gcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 9ed2ddffc35..8adf751e601 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -18,6 +18,7 @@ import logging import os from random import randint +import unittest from fsspec.implementations.arrow import ArrowFSWrapper from pyarrow.fs import GcsFileSystem From f754997039b801dcbf4c0090281e93c1b8fe447a Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 21:39:48 +0800 Subject: [PATCH 77/89] Fix --- clients/client-python/tests/integration/test_catalog.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index acecfefa45c..c89e179f3ab 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -107,13 +107,7 @@ def test_list_catalogs(self): self.assertTrue(self.catalog_name in catalog_names) def test_create_catalog(self): - try: - self.gravitino_client.load_catalog(self.catalog_name) - except NoSuchCatalogException: - logger.info("TestCatalog: Catalog %s does not exist", self.catalog_name) - catalog = self.create_catalog(self.catalog_name) - logger.info("TestCatalog: Catalog %s created, properties: %s", catalog.name(), catalog.properties) self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual( catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} From 2cdfb35350be65ed37725b14dec9206f5e48a77e Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 16 Oct 2024 21:54:04 +0800 Subject: [PATCH 78/89] Fix --- .../client-python/gravitino/client/gravitino_client.py | 10 ---------- .../tests/integration/test_gvfs_with_gcs.py | 10 +++++----- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/clients/client-python/gravitino/client/gravitino_client.py b/clients/client-python/gravitino/client/gravitino_client.py index c6da24e76e5..4e2b064caca 100644 --- a/clients/client-python/gravitino/client/gravitino_client.py +++ b/clients/client-python/gravitino/client/gravitino_client.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. -import logging from typing import List, Dict from gravitino.api.catalog import Catalog @@ -24,7 +23,6 @@ from gravitino.client.gravitino_client_base import GravitinoClientBase from gravitino.client.gravitino_metalake import GravitinoMetalake -logger = logging.getLogger(__name__) class GravitinoClient(GravitinoClientBase): """Gravitino Client for a user to interact with the Gravitino API, allowing the client to list, @@ -84,14 +82,6 @@ def create_catalog( comment: str, properties: Dict[str, str], ) -> Catalog: - logger.info( - "Creating catalog %s with type %s, provider %s, comment %s, properties %s", - name, - catalog_type, - provider, - comment, - properties, - ) return self.get_metalake().create_catalog( name, catalog_type, provider, comment, properties ) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 8adf751e601..ee67eab2f39 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -35,9 +35,11 @@ logger = logging.getLogger(__name__) + +@unittest.skip("This test require GCS service account key file") class TestGvfsWithGCS(TestGvfsWithHDFS): - key_file = "/home/ec2-user/silken-physics-431108-g3-30ab3d97bb60.json" - bucket_name = "example_qazwsx" + key_file = "your_key_file.json" + bucket_name = "your_bucket_name" metalake_name: str = "TestGvfsWithGCS_metalake" + str(randint(1, 10000)) def setUp(self): @@ -65,14 +67,12 @@ def tearDownClass(cls): # restart server cls.restart_server() - # clear all config in the conf_path @classmethod def _reset_conf(cls, config, conf_path): logger.info("Reset %s.", conf_path) if not os.path.exists(conf_path): - raise GravitinoRuntimeException( - f"Conf file is not found at `{conf_path}`.") + raise GravitinoRuntimeException(f"Conf file is not found at `{conf_path}`.") filtered_lines = [] with open(conf_path, mode="r", encoding="utf-8") as file: origin_lines = file.readlines() From 67dbc3a730891605d1d00fb6103d6be4d180c76a Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 17 Oct 2024 11:36:12 +0800 Subject: [PATCH 79/89] Resolve comments. --- build.gradle.kts | 4 +- .../build.gradle.kts | 0 .../gcs/fs/GCSFileSystemProvider.java | 0 ...itino.catalog.hadoop.fs.FileSystemProvider | 0 catalogs/catalog-hadoop/build.gradle.kts | 2 +- ...CatalogIT.java => HadoopGCSCatalogIT.java} | 6 +- clients/filesystem-hadoop3/build.gradle.kts | 1 + ...avitinoVirtualFileSystemConfiguration.java | 2 +- .../test/GravitinoVirtualFileSystemGCSIT.java | 166 ++++++++++++++++++ .../test/GravitinoVirtualFileSystemIT.java | 86 +++++---- .../integration/test/util/ITUtils.java | 1 + settings.gradle.kts | 2 +- 12 files changed, 230 insertions(+), 40 deletions(-) rename bundles/{gcs-bundle => gcp-bundle}/build.gradle.kts (100%) rename bundles/{gcs-bundle => gcp-bundle}/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java (100%) rename bundles/{gcs-bundle => gcp-bundle}/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider (100%) rename catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/{HadoopGCPCatalogIT.java => HadoopGCSCatalogIT.java} (93%) create mode 100644 clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java diff --git a/build.gradle.kts b/build.gradle.kts index 24e270d662f..9733a17912f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -745,7 +745,7 @@ tasks { if (!it.name.startsWith("catalog") && !it.name.startsWith("authorization") && !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && - it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && it.name != "gcs-bundle" + it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && it.name != "gcp-bundle" ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -764,7 +764,7 @@ tasks { !it.name.startsWith("integration-test") && !it.name.startsWith("flink") && !it.name.startsWith("trino-connector") && - it.name != "hive-metastore-common" && it.name != "gcs-bundle" + it.name != "hive-metastore-common" && it.name != "gcp-bundle" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/bundles/gcs-bundle/build.gradle.kts b/bundles/gcp-bundle/build.gradle.kts similarity index 100% rename from bundles/gcs-bundle/build.gradle.kts rename to bundles/gcp-bundle/build.gradle.kts diff --git a/bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java b/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java similarity index 100% rename from bundles/gcs-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java rename to bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java diff --git a/bundles/gcs-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/gcs-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index b076733660c..9ff3cc0e31c 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -73,7 +73,7 @@ dependencies { testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(project(":server")) testImplementation(project(":server-common")) - testImplementation(project(":bundles:gcs-bundle")) + testImplementation(project(":bundles:gcp-bundle")) testImplementation(libs.minikdc) testImplementation(libs.hadoop3.minicluster) diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java similarity index 93% rename from catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java rename to catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java index 8179f8e3a6e..74ae2a77cdb 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCPCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java @@ -32,14 +32,12 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.TestInstance; @Tag("gravitino-docker-test") -@TestInstance(TestInstance.Lifecycle.PER_CLASS) @Disabled( - "Disabled due to as we don't have a real GCP account to test. If you have a GCP account," + "Disabled due to we don't have a real GCP account to test. If you have a GCP account," + "please change the configuration(YOUR_KEY_FILE, YOUR_BUCKET) and enable this test.") -public class HadoopGCPCatalogIT extends HadoopCatalogIT { +public class HadoopGCSCatalogIT extends HadoopCatalogIT { public static final String BUCKET_NAME = "YOUR_BUCKET"; public static final String SERVICE_ACCOUNT_FILE = "YOUR_KEY_FILE"; diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts index aefac5f28b9..cae1888185a 100644 --- a/clients/filesystem-hadoop3/build.gradle.kts +++ b/clients/filesystem-hadoop3/build.gradle.kts @@ -39,6 +39,7 @@ dependencies { testImplementation(project(":server-common")) testImplementation(project(":clients:client-java")) testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":bundles:gcp-bundle")) testImplementation(libs.awaitility) testImplementation(libs.bundles.jetty) testImplementation(libs.bundles.jersey) diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index cd1ecb92fa8..e00e2e06125 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -21,7 +21,7 @@ import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; /** Configuration class for Gravitino Virtual File System. */ -class GravitinoVirtualFileSystemConfiguration { +public class GravitinoVirtualFileSystemConfiguration { public static final String GVFS_FILESET_PREFIX = "gvfs://fileset"; public static final String GVFS_SCHEME = "gvfs"; public static final String GVFS_CONFIG_PREFIX = "fs.gvfs."; diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java new file mode 100644 index 00000000000..988d58d7d7d --- /dev/null +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.filesystem.hadoop.integration.test; + +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; +import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS; + +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.integration.test.util.DownloaderUtils; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.integration.test.util.ITUtils; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Disabled( + "Disabled due to we don't have a real GCP account to test. If you have a GCP account," + + "please change the configuration(YOUR_KEY_FILE, YOUR_BUCKET) and enable this test.") +public class GravitinoVirtualFileSystemGCSIT extends GravitinoVirtualFileSystemIT { + private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemGCSIT.class); + + public static final String BUCKET_NAME = "YOUR_BUCKET"; + public static final String SERVICE_ACCOUNT_FILE = "YOUR_KEY_FILE"; + + @BeforeAll + public void startIntegrationTest() { + // Do nothing + } + + @BeforeAll + public void startUp() throws Exception { + copyGCPJars(); + // Need to download jars to gravitino server + super.startIntegrationTest(); + + // This value can be by tune by the user, please change it accordingly. + defaultBockSize = 64 * 1024 * 1024; + + metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); + catalogName = GravitinoITUtils.genRandomName("catalog"); + schemaName = GravitinoITUtils.genRandomName("schema"); + + Assertions.assertFalse(client.metalakeExists(metalakeName)); + metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); + Assertions.assertTrue(client.metalakeExists(metalakeName)); + + Map properties = Maps.newHashMap(); + properties.put(FILESYSTEM_PROVIDERS, "gcs"); + properties.put( + "gravitino.bypass.fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); + + Catalog catalog = + metalake.createCatalog( + catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", properties); + Assertions.assertTrue(metalake.catalogExists(catalogName)); + + catalog.asSchemas().createSchema(schemaName, "schema comment", properties); + Assertions.assertTrue(catalog.asSchemas().schemaExists(schemaName)); + + conf.set("fs.gvfs.impl", "org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem"); + conf.set("fs.AbstractFileSystem.gvfs.impl", "org.apache.gravitino.filesystem.hadoop.Gvfs"); + conf.set("fs.gvfs.impl.disable.cache", "true"); + conf.set("fs.gravitino.server.uri", serverUri); + conf.set("fs.gravitino.client.metalake", metalakeName); + + // Pass this configuration to the real file system + conf.set("gravitino.bypass.fs.gs.auth.service.account.enable", "true"); + conf.set("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE); + conf.set(FS_FILESYSTEM_PROVIDERS, "gcs"); + } + + @AfterAll + public void tearDown() throws IOException { + Catalog catalog = metalake.loadCatalog(catalogName); + catalog.asSchemas().dropSchema(schemaName, true); + metalake.dropCatalog(catalogName); + client.dropMetalake(metalakeName); + + if (client != null) { + client.close(); + client = null; + } + + try { + closer.close(); + } catch (Exception e) { + LOG.error("Exception in closing CloseableGroup", e); + } + } + + /** + * Remove the `gravitino.bypass` prefix from the configuration and pass it to the real file system + * This method corresponds to the method org.apache.gravitino.filesystem.hadoop + * .GravitinoVirtualFileSystem#getConfigMap(Configuration) in the original code. + */ + protected Configuration convertGvfsConfigToRealFileSystemConfig(Configuration gvfsConf) { + Configuration gcsConf = new Configuration(); + gvfsConf.forEach( + entry -> { + gcsConf.set(entry.getKey().replace("gravitino.bypass.", ""), entry.getValue()); + }); + + return gcsConf; + } + + protected String genStorageLocation(String fileset) { + return String.format("gs://%s/%s", BUCKET_NAME, fileset); + } + + private static boolean isDeploy() { + String mode = + System.getProperty(ITUtils.TEST_MODE) == null + ? ITUtils.EMBEDDED_TEST_MODE + : System.getProperty(ITUtils.TEST_MODE); + + return Objects.equals(mode, ITUtils.DEPLOY_TEST_MODE); + } + + private void copyGCPJars() { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String jarName = String.format("gravitino-gcp-bundle-%s.jar", System.getenv("PROJECT_VERSION")); + String gcsJars = + ITUtils.joinPath( + gravitinoHome, "..", "..", "bundles", "gcp-bundle", "build", "libs", jarName); + gcsJars = "file://" + gcsJars; + try { + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs"); + DownloaderUtils.downloadFile(gcsJars, hadoopLibDirs); + } + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to copy the gcs dependency jars: %s", gcsJars), e); + } + } + + @Disabled( + "GCS does not support append, java.io.IOException: The append operation is not supported") + public void testAppend() throws IOException {} +} diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java index 9b6334e092b..ced9a0b8b89 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java @@ -57,14 +57,15 @@ public class GravitinoVirtualFileSystemIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemIT.class); private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static final String metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); - private static final String catalogName = GravitinoITUtils.genRandomName("catalog"); - private static final String schemaName = GravitinoITUtils.genRandomName("schema"); - private static GravitinoMetalake metalake; - private static Configuration conf = new Configuration(); + protected String metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); + protected String catalogName = GravitinoITUtils.genRandomName("catalog"); + protected String schemaName = GravitinoITUtils.genRandomName("schema"); + protected GravitinoMetalake metalake; + protected Configuration conf = new Configuration(); + protected int defaultBockSize = 128 * 1024 * 1024; @BeforeAll - public void startUp() { + public void startUp() throws Exception { containerSuite.startHiveContainer(); Assertions.assertFalse(client.metalakeExists(metalakeName)); metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); @@ -112,10 +113,14 @@ public void tearDown() throws IOException { } } + protected Configuration convertGvfsConfigToRealFileSystemConfig(Configuration gvfsConf) { + return gvfsConf; + } + @Test public void testCreate() throws IOException { // create fileset - String filesetName = "test_fileset_create"; + String filesetName = GravitinoITUtils.genRandomName("test_fileset_create"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -131,25 +136,28 @@ public void testCreate() throws IOException { // test gvfs create Path hdfsPath = new Path(storageLocation); - try (FileSystem fs = hdfsPath.getFileSystem(conf)) { + try (FileSystem fs = hdfsPath.getFileSystem(convertGvfsConfigToRealFileSystemConfig(conf))) { Assertions.assertTrue(fs.exists(hdfsPath)); Path gvfsPath = genGvfsPath(filesetName); try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { Assertions.assertTrue(gvfs.exists(gvfsPath)); String fileName = "test.txt"; Path createPath = new Path(gvfsPath + "/" + fileName); - gvfs.create(createPath); + // GCS need to close the stream to create the file manually. + gvfs.create(createPath).close(); Assertions.assertTrue(gvfs.exists(createPath)); Assertions.assertTrue(gvfs.getFileStatus(createPath).isFile()); Assertions.assertTrue(fs.exists(new Path(storageLocation + "/" + fileName))); } } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } @Test public void testAppend() throws IOException { // create fileset - String filesetName = "test_fileset_append"; + String filesetName = GravitinoITUtils.genRandomName("test_fileset_append"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -165,7 +173,7 @@ public void testAppend() throws IOException { // test gvfs append Path hdfsPath = new Path(storageLocation); - try (FileSystem fs = hdfsPath.getFileSystem(conf)) { + try (FileSystem fs = hdfsPath.getFileSystem(convertGvfsConfigToRealFileSystemConfig(conf))) { Assertions.assertTrue(fs.exists(hdfsPath)); Path gvfsPath = genGvfsPath(filesetName); String fileName = "test.txt"; @@ -173,7 +181,7 @@ public void testAppend() throws IOException { try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { Assertions.assertTrue(gvfs.exists(gvfsPath)); - gvfs.create(appendPath); + gvfs.create(appendPath).close(); Assertions.assertTrue(gvfs.exists(appendPath)); Assertions.assertTrue(gvfs.getFileStatus(appendPath).isFile()); Assertions.assertTrue(fs.exists(new Path(storageLocation + "/" + fileName))); @@ -203,12 +211,14 @@ public void testAppend() throws IOException { } } } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } @Test public void testDelete() throws IOException { // create fileset - String filesetName = "test_fileset_delete"; + String filesetName = GravitinoITUtils.genRandomName("test_fileset_delete"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -224,14 +234,14 @@ public void testDelete() throws IOException { // test gvfs delete Path hdfsPath = new Path(storageLocation); - try (FileSystem fs = hdfsPath.getFileSystem(conf)) { + try (FileSystem fs = hdfsPath.getFileSystem(convertGvfsConfigToRealFileSystemConfig(conf))) { Assertions.assertTrue(fs.exists(hdfsPath)); Path gvfsPath = genGvfsPath(filesetName); String fileName = "test.txt"; Path deletePath = new Path(gvfsPath + "/" + fileName); try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { Assertions.assertTrue(gvfs.exists(gvfsPath)); - gvfs.create(deletePath); + gvfs.create(deletePath).close(); Assertions.assertTrue(gvfs.exists(deletePath)); Assertions.assertTrue(gvfs.getFileStatus(deletePath).isFile()); Assertions.assertTrue(fs.exists(new Path(storageLocation + "/" + fileName))); @@ -242,12 +252,14 @@ public void testDelete() throws IOException { Assertions.assertFalse(fs.exists(new Path(storageLocation + "/" + fileName))); } } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } @Test public void testGetStatus() throws IOException { // create fileset - String filesetName = "test_fileset_get_status"; + String filesetName = GravitinoITUtils.genRandomName("test_fileset_get_status"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -263,14 +275,14 @@ public void testGetStatus() throws IOException { // test gvfs get status Path hdfsPath = new Path(storageLocation); - try (FileSystem fs = hdfsPath.getFileSystem(conf)) { + try (FileSystem fs = hdfsPath.getFileSystem(convertGvfsConfigToRealFileSystemConfig(conf))) { Assertions.assertTrue(fs.exists(hdfsPath)); Path gvfsPath = genGvfsPath(filesetName); String fileName = "test.txt"; Path statusPath = new Path(gvfsPath + "/" + fileName); try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { Assertions.assertTrue(gvfs.exists(gvfsPath)); - gvfs.create(statusPath); + gvfs.create(statusPath).close(); Assertions.assertTrue(gvfs.exists(statusPath)); Assertions.assertTrue(gvfs.getFileStatus(statusPath).isFile()); Assertions.assertTrue(fs.exists(new Path(storageLocation + "/" + fileName))); @@ -284,12 +296,14 @@ public void testGetStatus() throws IOException { .replaceFirst(genGvfsPath(filesetName).toString(), storageLocation)); } } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } @Test public void testListStatus() throws IOException { // create fileset - String filesetName = "test_fileset_list_status"; + String filesetName = GravitinoITUtils.genRandomName("test_fileset_list_status"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -305,7 +319,7 @@ public void testListStatus() throws IOException { // test gvfs list status Path hdfsPath = new Path(storageLocation); - try (FileSystem fs = hdfsPath.getFileSystem(conf)) { + try (FileSystem fs = hdfsPath.getFileSystem(convertGvfsConfigToRealFileSystemConfig(conf))) { Assertions.assertTrue(fs.exists(hdfsPath)); Path gvfsPath = genGvfsPath(filesetName); for (int i = 0; i < 10; i++) { @@ -313,7 +327,7 @@ public void testListStatus() throws IOException { Path statusPath = new Path(gvfsPath + "/" + fileName); try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { Assertions.assertTrue(gvfs.exists(gvfsPath)); - gvfs.create(statusPath); + gvfs.create(statusPath).close(); Assertions.assertTrue(gvfs.exists(statusPath)); Assertions.assertTrue(gvfs.getFileStatus(statusPath).isFile()); Assertions.assertTrue(fs.exists(new Path(storageLocation + "/" + fileName))); @@ -340,12 +354,14 @@ public void testListStatus() throws IOException { } } } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } @Test public void testMkdirs() throws IOException { // create fileset - String filesetName = "test_fileset_mkdirs"; + String filesetName = GravitinoITUtils.genRandomName("test_fileset_mkdirs"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -361,7 +377,7 @@ public void testMkdirs() throws IOException { // test gvfs mkdirs Path hdfsPath = new Path(storageLocation); - try (FileSystem fs = hdfsPath.getFileSystem(conf)) { + try (FileSystem fs = hdfsPath.getFileSystem(convertGvfsConfigToRealFileSystemConfig(conf))) { Assertions.assertTrue(fs.exists(hdfsPath)); Path gvfsPath = genGvfsPath(filesetName); try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { @@ -374,12 +390,14 @@ public void testMkdirs() throws IOException { Assertions.assertTrue(fs.exists(new Path(storageLocation + "/" + dirName))); } } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } @Test public void testRename() throws IOException { // create fileset - String filesetName = "test_fileset_rename"; + String filesetName = GravitinoITUtils.genRandomName("test_fileset_rename"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -395,7 +413,7 @@ public void testRename() throws IOException { // test gvfs rename Path hdfsPath = new Path(storageLocation); - try (FileSystem fs = hdfsPath.getFileSystem(conf)) { + try (FileSystem fs = hdfsPath.getFileSystem(convertGvfsConfigToRealFileSystemConfig(conf))) { Assertions.assertTrue(fs.exists(hdfsPath)); Path gvfsPath = genGvfsPath(filesetName); String srcName = "test_src"; @@ -420,11 +438,13 @@ public void testRename() throws IOException { Assertions.assertFalse(fs.exists(new Path(storageLocation + "/" + srcName))); } } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } @Test public void testGetDefaultReplications() throws IOException { - String filesetName = "test_get_default_replications"; + String filesetName = GravitinoITUtils.genRandomName("test_get_default_replications"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -441,11 +461,13 @@ public void testGetDefaultReplications() throws IOException { try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { assertEquals(3, gvfs.getDefaultReplication(gvfsPath)); } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } @Test public void testGetDefaultBlockSizes() throws IOException { - String filesetName = "test_get_default_block_sizes"; + String filesetName = GravitinoITUtils.genRandomName("test_get_default_block_sizes"); NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Catalog catalog = metalake.loadCatalog(catalogName); String storageLocation = genStorageLocation(filesetName); @@ -460,15 +482,17 @@ public void testGetDefaultBlockSizes() throws IOException { Assertions.assertTrue(catalog.asFilesetCatalog().filesetExists(filesetIdent)); Path gvfsPath = genGvfsPath(filesetName); try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { - assertEquals(128 * 1024 * 1024, gvfs.getDefaultBlockSize(gvfsPath)); + assertEquals(defaultBockSize, gvfs.getDefaultBlockSize(gvfsPath)); } + + catalog.asFilesetCatalog().dropFileset(filesetIdent); } - private String genStorageLocation(String fileset) { + protected String genStorageLocation(String fileset) { return String.format("%s/%s", baseHdfsPath(), fileset); } - private static String baseHdfsPath() { + private String baseHdfsPath() { return String.format( "hdfs://%s:%d/%s/%s", containerSuite.getHiveContainer().getContainerIpAddress(), @@ -477,7 +501,7 @@ private static String baseHdfsPath() { schemaName); } - private Path genGvfsPath(String fileset) { + protected Path genGvfsPath(String fileset) { return new Path(String.format("gvfs://fileset/%s/%s/%s", catalogName, schemaName, fileset)); } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java index e5454199f8a..9a6d7b13010 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java @@ -50,6 +50,7 @@ public class ITUtils { public static final String TEST_MODE = "testMode"; public static final String EMBEDDED_TEST_MODE = "embedded"; + public static final String DEPLOY_TEST_MODE = "deploy"; public static String joinPath(String... dirs) { return String.join(File.separator, dirs); diff --git a/settings.gradle.kts b/settings.gradle.kts index dcaa8fbe6f4..36d66504f47 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -70,4 +70,4 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") -include(":bundles:gcs-bundle") +include(":bundles:gcp-bundle") From 70a545e76e21ed195f5c71fd045a486d4334b88d Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 17 Oct 2024 11:47:16 +0800 Subject: [PATCH 80/89] Fix the java doc problem. --- ...avitinoVirtualFileSystemConfiguration.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index e00e2e06125..95ce4df2a8f 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -22,8 +22,17 @@ /** Configuration class for Gravitino Virtual File System. */ public class GravitinoVirtualFileSystemConfiguration { + + /** + * The prefix of the Gravitino fileset URI. The URI of the Gravitino fileset should start with + * this prefix. + */ public static final String GVFS_FILESET_PREFIX = "gvfs://fileset"; + + /** The scheme of the Gravitino Virtual File System. */ public static final String GVFS_SCHEME = "gvfs"; + + /** The prefix of the Gravitino Virtual File System. */ public static final String GVFS_CONFIG_PREFIX = "fs.gvfs."; /** The configuration key for the Gravitino server URI. */ @@ -42,8 +51,12 @@ public class GravitinoVirtualFileSystemConfiguration { */ public static final String FS_FILESYSTEM_PROVIDERS = "fs.gvfs.filesystem.providers"; + /** The authentication type for simple authentication. */ public static final String SIMPLE_AUTH_TYPE = "simple"; + /** The authentication type for oauth2 authentication. */ public static final String OAUTH2_AUTH_TYPE = "oauth2"; + + /** The authentication type for kerberos authentication. */ public static final String KERBEROS_AUTH_TYPE = "kerberos"; // oauth2 /** The configuration key for the URI of the default OAuth server. */ @@ -74,6 +87,10 @@ public class GravitinoVirtualFileSystemConfiguration { public static final String FS_GRAVITINO_FILESET_CACHE_MAX_CAPACITY_KEY = "fs.gravitino.fileset.cache.maxCapacity"; + /** + * The default value for the maximum capacity of the Gravitino fileset cache. The default value is + * 20. + */ public static final int FS_GRAVITINO_FILESET_CACHE_MAX_CAPACITY_DEFAULT = 20; /** @@ -83,6 +100,10 @@ public class GravitinoVirtualFileSystemConfiguration { public static final String FS_GRAVITINO_FILESET_CACHE_EVICTION_MILLS_AFTER_ACCESS_KEY = "fs.gravitino.fileset.cache.evictionMillsAfterAccess"; + /** + * The default value for the eviction time of the Gravitino fileset cache, measured in mills after + * access. + */ public static final long FS_GRAVITINO_FILESET_CACHE_EVICTION_MILLS_AFTER_ACCESS_DEFAULT = 1000L * 60 * 60; From 15bbf99d36243b4d6abee3f13010e07e03e7d955 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 17 Oct 2024 15:04:16 +0800 Subject: [PATCH 81/89] rebase issue_5074 --- clients/client-python/tests/integration/test_gvfs_with_gcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index ee67eab2f39..cb88194951a 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -104,7 +104,7 @@ def _init_test_entities(cls): provider=cls.catalog_provider, comment="", properties={ - "filesystem-providers-classnames": "org.apache.gravitino.fileset.gcs.GCSFileSystemProvider", + "filesystem-providers": "gcs", "gravitino.bypass.fs.gs.auth.service.account.enable": "true", "gravitino.bypass.fs.gs.auth.service.account.json.keyfile": cls.key_file, }, From cfcc5440997b9e20ce3765321bdcd925a8fa5bd8 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 17 Oct 2024 18:04:28 +0800 Subject: [PATCH 82/89] Optimize code. --- .../gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java | 3 +-- .../gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java | 3 +-- .../integration/test/GravitinoVirtualFileSystemGCSIT.java | 4 ++++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java index 7c9ceebdd36..c7c2fd393f6 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java @@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; public class HDFSFileSystemProvider implements FileSystemProvider { @@ -39,7 +38,7 @@ public FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map (k, v) -> { configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v); }); - return DistributedFileSystem.newInstance(path.toUri(), configuration); + return FileSystem.newInstance(path.toUri(), configuration); } @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java index 70e44c76f6b..e940e2bb6ba 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java @@ -25,7 +25,6 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; public class LocalFileSystemProvider implements FileSystemProvider { @@ -38,7 +37,7 @@ public FileSystem getFileSystem(Path path, Map config) throws IO configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v); }); - return LocalFileSystem.newInstance(path.toUri(), configuration); + return FileSystem.newInstance(path.toUri(), configuration); } @Override diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java index 988d58d7d7d..a42d1c4b7b3 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java @@ -143,6 +143,10 @@ private static boolean isDeploy() { } private void copyGCPJars() { + if (!isDeploy()) { + return; + } + String gravitinoHome = System.getenv("GRAVITINO_HOME"); String jarName = String.format("gravitino-gcp-bundle-%s.jar", System.getenv("PROJECT_VERSION")); String gcsJars = From 4f00a2f11bbe7a1fa498b2dfc8a2ec6b415e1b1e Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 17 Oct 2024 20:29:48 +0800 Subject: [PATCH 83/89] Remove s3 related code. --- .../gravitino/filesystem/gvfs.py | 66 ++----------------- .../gravitino/filesystem/gvfs_config.py | 1 - 2 files changed, 6 insertions(+), 61 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index 0c7f4dfedba..cd9052a7911 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -28,7 +28,6 @@ from fsspec.utils import infer_storage_options from pyarrow.fs import HadoopFileSystem from pyarrow.fs import GcsFileSystem -from pyarrow.fs import S3FileSystem from readerwriterlock import rwlock from gravitino.audit.caller_context import CallerContext, CallerContextHolder @@ -51,7 +50,6 @@ class StorageType(Enum): HDFS = "hdfs" LOCAL = "file" GCS = "gs" - S3 = "s3" class FilesetContextPair: @@ -317,7 +315,7 @@ def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): # convert the following to in - if storage_type in [StorageType.HDFS, StorageType.GCS, StorageType.S3]: + if storage_type in [StorageType.HDFS, StorageType.GCS]: src_context_pair.filesystem().mv( self._strip_storage_protocol(storage_type, src_actual_path), self._strip_storage_protocol(storage_type, dst_actual_path), @@ -548,13 +546,11 @@ def _convert_actual_path( :param virtual_location: Virtual location :return A virtual path """ - # If the storage path start with hdfs, gcs, s3, s3a or s3n, we should use the path as the prefix. - if ( - storage_location.startswith(f"{StorageType.HDFS.value}://") - or storage_location.startswith(f"{StorageType.GCS.value}://") - or storage_location.startswith(f"{StorageType.S3.value}://") - ): + # If the storage path starts with hdfs, gcs, we should use the path as the prefix. + if storage_location.startswith( + f"{StorageType.HDFS.value}://" + ) or storage_location.startswith(f"{StorageType.GCS.value}://"): actual_prefix = infer_storage_options(storage_location)["path"] elif storage_location.startswith(f"{StorageType.LOCAL.value}:/"): actual_prefix = storage_location[len(f"{StorageType.LOCAL.value}:") :] @@ -697,8 +693,6 @@ def _recognize_storage_type(path: str): return StorageType.LOCAL if path.startswith(f"{StorageType.GCS.value}://"): return StorageType.GCS - if path.startswith(f"{StorageType.S3.value}://"): - return StorageType.S3 raise GravitinoRuntimeException( f"Storage type doesn't support now. Path:{path}" ) @@ -723,7 +717,7 @@ def _strip_storage_protocol(storage_type: StorageType, path: str): :param path: The path :return: The stripped path """ - if storage_type in (StorageType.HDFS, StorageType.S3, StorageType.GCS): + if storage_type in (StorageType.HDFS, StorageType.GCS): return path if storage_type == StorageType.LOCAL: return path[len(f"{StorageType.LOCAL.value}:") :] @@ -798,8 +792,6 @@ def _get_filesystem(self, actual_file_location: str): fs = LocalFileSystem() elif storage_type == StorageType.GCS: fs = ArrowFSWrapper(self._get_gcs_filesystem()) - elif storage_type == StorageType.S3: - fs = ArrowFSWrapper(self._get_s3_filesystem()) else: raise GravitinoRuntimeException( f"Storage type: `{storage_type}` doesn't support now." @@ -823,54 +815,8 @@ def _get_gcs_filesystem(self): raise GravitinoRuntimeException( "Service account key is not found in the options." ) - - # scopes = ["https://www.googleapis.com/auth/cloud-platform"] - # credentials = service_account.Credentials.from_service_account_file( - # service_account_key_path, scopes=scopes) - # credentials.refresh(Request()) - - # access_token = credentials.token - # expiration = credentials.expiry - - # return GcsFileSystem(access_token=access_token, - # credential_token_expiration=expiration) os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_path return GcsFileSystem() - def _get_s3_filesystem(self): - # get All keys from the options that start with 'gravitino.bypass.s3.' and remove the prefix - s3_options = { - key[len(GVFSConfig.GVFS_FILESYSTEM_BY_PASS_S3) :]: value - for key, value in self._options.items() - if key.startswith(GVFSConfig.GVFS_FILESYSTEM_BY_PASS_S3) - } - - # get 'aws_access_key_id' from s3_options, if the key is not found, throw an exception - aws_access_key_id = s3_options.get("aws_access_key_id") - if aws_access_key_id is None: - raise GravitinoRuntimeException( - "AWS access key id is not found in the options." - ) - - # get 'aws_secret_access_key' from s3_options, if the key is not found, throw an exception - aws_secret_access_key = s3_options.get("aws_secret_access_key") - if aws_secret_access_key is None: - raise GravitinoRuntimeException( - "AWS secret access key is not found in the options." - ) - - # get 'aws_endpoint_url' from s3_options, if the key is not found, throw an exception - aws_endpoint_url = s3_options.get("aws_endpoint_url") - if aws_endpoint_url is None: - raise GravitinoRuntimeException( - "AWS endpoint url is not found in the options." - ) - - return S3FileSystem( - key=aws_access_key_id, - secret=aws_secret_access_key, - endpoint_override=aws_endpoint_url, - ) - fsspec.register_implementation(PROTOCOL_NAME, GravitinoVirtualFileSystem) diff --git a/clients/client-python/gravitino/filesystem/gvfs_config.py b/clients/client-python/gravitino/filesystem/gvfs_config.py index 743216d7ed3..6a4d865fc3e 100644 --- a/clients/client-python/gravitino/filesystem/gvfs_config.py +++ b/clients/client-python/gravitino/filesystem/gvfs_config.py @@ -35,4 +35,3 @@ class GVFSConfig: GVFS_FILESYSTEM_BY_PASS = "gravitino.bypass" GVFS_FILESYSTEM_BY_PASS_GCS = "gravitino.bypass.gcs." - GVFS_FILESYSTEM_BY_PASS_S3 = "gravitino.bypass.s3." From b9ef8f0afb3730a458aaab848b93b6c070773db4 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 17 Oct 2024 21:42:23 +0800 Subject: [PATCH 84/89] fix --- .../integration/test/HadoopGCSCatalogIT.java | 13 +++++++ .../tests/integration/test_gvfs_with_gcs.py | 2 + .../test/GravitinoVirtualFileSystemGCSIT.java | 37 +------------------ .../integration/test/util/BaseIT.java | 33 +++++++++++++++++ 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java index 74ae2a77cdb..cca13b77047 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java @@ -42,8 +42,21 @@ public class HadoopGCSCatalogIT extends HadoopCatalogIT { public static final String BUCKET_NAME = "YOUR_BUCKET"; public static final String SERVICE_ACCOUNT_FILE = "YOUR_KEY_FILE"; + @Override + public void startIntegrationTest() throws Exception { + // Just overwrite super, do nothing. + } + @BeforeAll public void setup() throws IOException { + copyBundleJarsToHadoop("gcp-bundle"); + + try { + super.startIntegrationTest(); + } catch (Exception e) { + throw new RuntimeException(e); + } + metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); schemaName = GravitinoITUtils.genRandomName("CatalogFilesetIT_schema"); diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index cb88194951a..13c316c8496 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -38,6 +38,8 @@ @unittest.skip("This test require GCS service account key file") class TestGvfsWithGCS(TestGvfsWithHDFS): + # Before running this test, please set the make sure gcp-bundle-x.jar has been + # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory key_file = "your_key_file.json" bucket_name = "your_bucket_name" metalake_name: str = "TestGvfsWithGCS_metalake" + str(randint(1, 10000)) diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java index a42d1c4b7b3..312236fe5da 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java @@ -26,11 +26,8 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; -import java.util.Objects; import org.apache.gravitino.Catalog; -import org.apache.gravitino.integration.test.util.DownloaderUtils; import org.apache.gravitino.integration.test.util.GravitinoITUtils; -import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -55,7 +52,8 @@ public void startIntegrationTest() { @BeforeAll public void startUp() throws Exception { - copyGCPJars(); + // Copy the GCP jars to the gravitino server if in deploy mode. + copyBundleJarsToHadoop("gcp-bundle"); // Need to download jars to gravitino server super.startIntegrationTest(); @@ -133,37 +131,6 @@ protected String genStorageLocation(String fileset) { return String.format("gs://%s/%s", BUCKET_NAME, fileset); } - private static boolean isDeploy() { - String mode = - System.getProperty(ITUtils.TEST_MODE) == null - ? ITUtils.EMBEDDED_TEST_MODE - : System.getProperty(ITUtils.TEST_MODE); - - return Objects.equals(mode, ITUtils.DEPLOY_TEST_MODE); - } - - private void copyGCPJars() { - if (!isDeploy()) { - return; - } - - String gravitinoHome = System.getenv("GRAVITINO_HOME"); - String jarName = String.format("gravitino-gcp-bundle-%s.jar", System.getenv("PROJECT_VERSION")); - String gcsJars = - ITUtils.joinPath( - gravitinoHome, "..", "..", "bundles", "gcp-bundle", "build", "libs", jarName); - gcsJars = "file://" + gcsJars; - try { - if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { - String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs"); - DownloaderUtils.downloadFile(gcsJars, hadoopLibDirs); - } - } catch (Exception e) { - throw new RuntimeException( - String.format("Failed to copy the gcs dependency jars: %s", gcsJars), e); - } - } - @Disabled( "GCS does not support append, java.io.IOException: The append operation is not supported") public void testAppend() throws IOException {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java index e8f688f96ea..8bbb5a3b23f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java @@ -38,6 +38,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; @@ -386,4 +387,36 @@ protected String readGitCommitIdFromGitFile() { return ""; } } + + private static boolean isDeploy() { + String mode = + System.getProperty(ITUtils.TEST_MODE) == null + ? ITUtils.EMBEDDED_TEST_MODE + : System.getProperty(ITUtils.TEST_MODE); + + return Objects.equals(mode, ITUtils.DEPLOY_TEST_MODE); + } + + protected void copyBundleJarsToHadoop(String bundleName) { + if (!isDeploy()) { + return; + } + + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String jarName = + String.format("gravitino-%s-%s.jar", bundleName, System.getenv("PROJECT_VERSION")); + String gcsJars = + ITUtils.joinPath( + gravitinoHome, "..", "..", "bundles", bundleName, "build", "libs", jarName); + gcsJars = "file://" + gcsJars; + try { + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs"); + DownloaderUtils.downloadFile(gcsJars, hadoopLibDirs); + } + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to copy the %s dependency jars: %s", bundleName, gcsJars), e); + } + } } From 9f65fb5413ebf6a6ec8928abcad8e90e7cc0dfa3 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 18 Oct 2024 16:49:24 +0800 Subject: [PATCH 85/89] try to import lazily. --- clients/client-python/gravitino/filesystem/gvfs.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index cd9052a7911..127fb7b846e 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -21,13 +21,17 @@ import re import fsspec + from cachetools import TTLCache, LRUCache +import importlib from fsspec import AbstractFileSystem from fsspec.implementations.local import LocalFileSystem from fsspec.implementations.arrow import ArrowFSWrapper from fsspec.utils import infer_storage_options -from pyarrow.fs import HadoopFileSystem -from pyarrow.fs import GcsFileSystem + + +# from pyarrow.fs import HadoopFileSystem +# from pyarrow.fs import GcsFileSystem from readerwriterlock import rwlock from gravitino.audit.caller_context import CallerContext, CallerContextHolder @@ -787,7 +791,8 @@ def _get_filesystem(self, actual_file_location: str): if cache_value is not None: return cache_value if storage_type == StorageType.HDFS: - fs = ArrowFSWrapper(HadoopFileSystem.from_uri(actual_file_location)) + fs_class = importlib.import_module("pyarrow.fs").HadoopFileSystem + fs = ArrowFSWrapper(fs_class.from_uri(actual_file_location)) elif storage_type == StorageType.LOCAL: fs = LocalFileSystem() elif storage_type == StorageType.GCS: @@ -816,7 +821,8 @@ def _get_gcs_filesystem(self): "Service account key is not found in the options." ) os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_path - return GcsFileSystem() + + return importlib.import_module("pyarrow.fs").GcsFileSystem() fsspec.register_implementation(PROTOCOL_NAME, GravitinoVirtualFileSystem) From 4defcc64e2fc1cb5d39b01c132bfa131484aae50 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 18 Oct 2024 17:17:16 +0800 Subject: [PATCH 86/89] format code. --- clients/client-python/gravitino/filesystem/gvfs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index 127fb7b846e..eebf57dbb70 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -19,11 +19,10 @@ from pathlib import PurePosixPath from typing import Dict, Tuple import re +import importlib import fsspec - from cachetools import TTLCache, LRUCache -import importlib from fsspec import AbstractFileSystem from fsspec.implementations.local import LocalFileSystem from fsspec.implementations.arrow import ArrowFSWrapper From 3a907f4395abf56c316c44ba6ae536c624614e94 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 18 Oct 2024 17:23:42 +0800 Subject: [PATCH 87/89] fix --- clients/client-python/gravitino/filesystem/gvfs.py | 2 +- clients/client-python/gravitino/filesystem/gvfs_config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index eebf57dbb70..f8a79e83813 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -814,7 +814,7 @@ def _get_gcs_filesystem(self): } # get 'service-account-key' from gcs_options, if the key is not found, throw an exception - service_account_key_path = gcs_options.get("service-account-key-path") + service_account_key_path = gcs_options.get(GVFSConfig.GVFS_FILESYSTEM_KEY_FILE) if service_account_key_path is None: raise GravitinoRuntimeException( "Service account key is not found in the options." diff --git a/clients/client-python/gravitino/filesystem/gvfs_config.py b/clients/client-python/gravitino/filesystem/gvfs_config.py index 6a4d865fc3e..5bd3c642556 100644 --- a/clients/client-python/gravitino/filesystem/gvfs_config.py +++ b/clients/client-python/gravitino/filesystem/gvfs_config.py @@ -35,3 +35,4 @@ class GVFSConfig: GVFS_FILESYSTEM_BY_PASS = "gravitino.bypass" GVFS_FILESYSTEM_BY_PASS_GCS = "gravitino.bypass.gcs." + GVFS_FILESYSTEM_KEY_FILE = "service-account-key-path" From 76912b7f55443110538ea8962866c89fa218619c Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 18 Oct 2024 17:26:55 +0800 Subject: [PATCH 88/89] fix --- clients/client-python/gravitino/filesystem/gvfs.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index f8a79e83813..8f1b2008ab9 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -28,10 +28,6 @@ from fsspec.implementations.arrow import ArrowFSWrapper from fsspec.utils import infer_storage_options - -# from pyarrow.fs import HadoopFileSystem -# from pyarrow.fs import GcsFileSystem - from readerwriterlock import rwlock from gravitino.audit.caller_context import CallerContext, CallerContextHolder from gravitino.audit.fileset_audit_constants import FilesetAuditConstants From 44786731213512b51db7f3a207f91bedea9976d0 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 18 Oct 2024 17:29:24 +0800 Subject: [PATCH 89/89] fix --- clients/client-python/gravitino/filesystem/gvfs_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs_config.py b/clients/client-python/gravitino/filesystem/gvfs_config.py index 5bd3c642556..618565c70eb 100644 --- a/clients/client-python/gravitino/filesystem/gvfs_config.py +++ b/clients/client-python/gravitino/filesystem/gvfs_config.py @@ -31,7 +31,6 @@ class GVFSConfig: OAUTH2_CREDENTIAL = "oauth2_credential" OAUTH2_PATH = "oauth2_path" OAUTH2_SCOPE = "oauth2_scope" - DEFAULT_AUTH_TYPE = "simple" GVFS_FILESYSTEM_BY_PASS = "gravitino.bypass" GVFS_FILESYSTEM_BY_PASS_GCS = "gravitino.bypass.gcs."