diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesOperatorClusterDescriptorAdapter.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesOperatorClusterDescriptorAdapter.java index c77fc655ca..2f0318254e 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesOperatorClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesOperatorClusterDescriptorAdapter.java @@ -38,8 +38,12 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; import io.fabric8.kubernetes.client.dsl.Resource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class KubernetesOperatorClusterDescriptorAdapter extends ClusterDescriptorAdapter { + private static final Logger logger = + LoggerFactory.getLogger(KubernetesOperatorClusterDescriptorAdapter.class); protected SparkConfig sparkConfig; protected KubernetesClient client; @@ -57,6 +61,10 @@ public KubernetesOperatorClusterDescriptorAdapter(ExecutionContext executionCont public void deployCluster(String mainClass, String args, Map confMap) { + logger.info( + "The spark k8s operator task start,k8sNamespace: {},appName: {}", + this.sparkConfig.getK8sNamespace(), + this.sparkConfig.getAppName()); CustomResourceDefinitionList crds = client.apiextensions().v1().customResourceDefinitions().list(); @@ -101,8 +109,10 @@ public void deployCluster(String mainClass, String args, Map con .executor(executor) .build(); + logger.info("Spark k8s operator task parameters: {}", sparkApplicationSpec); sparkApplication.setSpec(sparkApplicationSpec); SparkApplication created = sparkApplicationClient.createOrReplace(sparkApplication); + logger.info("Preparing to submit the Spark k8s operator Task: {}", created); // Wait three seconds to get the status try { diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationSpec.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationSpec.java index b170233b58..80c03f2df1 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationSpec.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationSpec.java @@ -133,6 +133,41 @@ public void setExecutor(SparkPodSpec executor) { this.executor = executor; } + @Override + public String toString() { + return "SparkApplicationSpec{" + + "type='" + + type + + '\'' + + ", mode='" + + mode + + '\'' + + ", image='" + + image + + '\'' + + ", imagePullPolicy='" + + imagePullPolicy + + '\'' + + ", mainClass='" + + mainClass + + '\'' + + ", mainApplicationFile='" + + mainApplicationFile + + '\'' + + ", sparkVersion='" + + sparkVersion + + '\'' + + ", restartPolicy=" + + restartPolicy + + ", volumes=" + + volumes + + ", driver=" + + driver + + ", executor=" + + executor + + '}'; + } + public static SparkApplicationSpecBuilder Builder() { return new SparkApplicationSpecBuilder(); } diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/KubernetesHelper.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/KubernetesHelper.java index 04b82a843f..709308d084 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/KubernetesHelper.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/KubernetesHelper.java @@ -23,8 +23,11 @@ import java.io.IOException; import io.fabric8.kubernetes.client.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class KubernetesHelper { + private static final Logger logger = LoggerFactory.getLogger(KubernetesHelper.class); public static KubernetesClient getKubernetesClientByUrl( String k8sMasterUrl, String k8sUsername, String k8sPassword) { @@ -34,16 +37,33 @@ public static KubernetesClient getKubernetesClientByUrl( .withUsername(k8sUsername) .withPassword(k8sPassword) .build(); - return new DefaultKubernetesClient(config); + DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient(config); + logger.info( + "KubernetesClient Create success,kubernetesClient masterUrl: {}", + kubernetesClient.getMasterUrl().toString()); + return kubernetesClient; } public static KubernetesClient getKubernetesClientByUrl(String k8sMasterUrl) { Config config = new ConfigBuilder().withMasterUrl(k8sMasterUrl).build(); - return new DefaultKubernetesClient(config); + DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient(config); + logger.info( + "KubernetesClient Create success,kubernetesClient masterUrl: {}", + kubernetesClient.getMasterUrl().toString()); + return kubernetesClient; } public static KubernetesClient getKubernetesClient( String kubeConfigFile, String k8sMasterUrl, String k8sUsername, String k8sPassword) { + logger.info( + "Start create KubernetesClient,kubeConfigFile: {},k8sMasterUrl: {}", + kubeConfigFile, + k8sMasterUrl); + + if (StringUtils.isBlank(kubeConfigFile) && StringUtils.isBlank(kubeConfigFile)) { + throw new KubernetesClientException( + "Both kubeConfigFile and k8sMasterUrl are empty. Initializing KubernetesClient failed."); + } // The ConfigFile mode is preferred if (StringUtils.isNotBlank(kubeConfigFile)) { return getKubernetesClientByKubeConfigFile(kubeConfigFile); @@ -59,8 +79,7 @@ public static KubernetesClient getKubernetesClient( return getKubernetesClientByUrl(k8sMasterUrl); } - throw new KubernetesClientException( - "Both kubeConfigFile and k8sMasterUrl are empty. Initializing KubernetesClient failed."); + throw new KubernetesClientException("Initializing KubernetesClient failed."); } public static KubernetesClient getKubernetesClientByKubeConfigFile(String kubeConfigFile) { @@ -77,6 +96,10 @@ public static KubernetesClient getKubernetesClientByKubeConfigFile(String kubeCo config = Config.autoConfigure(null); } - return new DefaultKubernetesClient(config); + DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient(config); + logger.info( + "KubernetesClient Create success,kubernetesClient masterUrl: {}", + kubernetesClient.getMasterUrl().toString()); + return kubernetesClient; } }