diff --git a/core/src/main/resources/operatorsScore-databricks-aws-a10G.csv b/core/src/main/resources/operatorsScore-databricks-aws-a10G.csv index 81d5edc82..bc2790483 100644 --- a/core/src/main/resources/operatorsScore-databricks-aws-a10G.csv +++ b/core/src/main/resources/operatorsScore-databricks-aws-a10G.csv @@ -190,7 +190,8 @@ PosExplode,2.45 Pow,2.45 PreciseTimestampConversion,2.45 PromotePrecision,2.45 -PythonUDF,2.45 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,2.45 RLike,2.45 RaiseError,2.45 diff --git a/core/src/main/resources/operatorsScore-databricks-aws-t4.csv b/core/src/main/resources/operatorsScore-databricks-aws-t4.csv index 81d5edc82..bc2790483 100644 --- a/core/src/main/resources/operatorsScore-databricks-aws-t4.csv +++ b/core/src/main/resources/operatorsScore-databricks-aws-t4.csv @@ -190,7 +190,8 @@ PosExplode,2.45 Pow,2.45 PreciseTimestampConversion,2.45 PromotePrecision,2.45 -PythonUDF,2.45 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,2.45 RLike,2.45 RaiseError,2.45 diff --git a/core/src/main/resources/operatorsScore-databricks-azure-t4.csv b/core/src/main/resources/operatorsScore-databricks-azure-t4.csv index cb84688e5..7913171c3 100644 --- a/core/src/main/resources/operatorsScore-databricks-azure-t4.csv +++ b/core/src/main/resources/operatorsScore-databricks-azure-t4.csv @@ -190,7 +190,8 @@ PosExplode,2.73 Pow,2.73 PreciseTimestampConversion,2.73 PromotePrecision,2.73 -PythonUDF,2.73 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,2.73 RLike,2.73 RaiseError,2.73 diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv index 566d26b0c..5849e5929 100644 --- a/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv @@ -183,7 +183,8 @@ PosExplode,3.74 Pow,3.74 PreciseTimestampConversion,3.74 PromotePrecision,3.74 -PythonUDF,3.74 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,3.74 RLike,3.74 RaiseError,3.74 diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv index d1b3376c9..acf903f4b 100644 --- a/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv @@ -183,7 +183,8 @@ PosExplode,3.65 Pow,3.65 PreciseTimestampConversion,3.65 PromotePrecision,3.65 -PythonUDF,3.65 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,3.65 RLike,3.65 RaiseError,3.65 diff --git a/core/src/main/resources/operatorsScore-dataproc-l4.csv b/core/src/main/resources/operatorsScore-dataproc-l4.csv index 4cedbd62c..c4158191e 100644 --- a/core/src/main/resources/operatorsScore-dataproc-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-l4.csv @@ -190,7 +190,8 @@ PosExplode,4.16 Pow,4.16 PreciseTimestampConversion,4.16 PromotePrecision,4.16 -PythonUDF,4.16 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,4.16 RLike,4.16 RaiseError,4.16 diff --git a/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv b/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv index 5d724c037..2ffad4ac0 100644 --- a/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv @@ -183,7 +183,8 @@ PosExplode,4.25 Pow,4.25 PreciseTimestampConversion,4.25 PromotePrecision,4.25 -PythonUDF,4.25 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,4.25 RLike,4.25 RaiseError,4.25 diff --git a/core/src/main/resources/operatorsScore-dataproc-t4.csv b/core/src/main/resources/operatorsScore-dataproc-t4.csv index 42310164a..90d6f91e7 100644 --- a/core/src/main/resources/operatorsScore-dataproc-t4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-t4.csv @@ -190,7 +190,8 @@ PosExplode,4.88 Pow,4.88 PreciseTimestampConversion,4.88 PromotePrecision,4.88 -PythonUDF,4.88 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,4.88 RLike,4.88 RaiseError,4.88 diff --git a/core/src/main/resources/operatorsScore-emr-a10.csv b/core/src/main/resources/operatorsScore-emr-a10.csv index 2cd9d57a5..013235be6 100644 --- a/core/src/main/resources/operatorsScore-emr-a10.csv +++ b/core/src/main/resources/operatorsScore-emr-a10.csv @@ -190,7 +190,8 @@ PosExplode,2.59 Pow,2.59 PreciseTimestampConversion,2.59 PromotePrecision,2.59 -PythonUDF,2.59 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,2.59 RLike,2.59 RaiseError,2.59 diff --git a/core/src/main/resources/operatorsScore-emr-a10G.csv b/core/src/main/resources/operatorsScore-emr-a10G.csv index 2cd9d57a5..013235be6 100644 --- a/core/src/main/resources/operatorsScore-emr-a10G.csv +++ b/core/src/main/resources/operatorsScore-emr-a10G.csv @@ -190,7 +190,8 @@ PosExplode,2.59 Pow,2.59 PreciseTimestampConversion,2.59 PromotePrecision,2.59 -PythonUDF,2.59 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,2.59 RLike,2.59 RaiseError,2.59 diff --git a/core/src/main/resources/operatorsScore-emr-t4.csv b/core/src/main/resources/operatorsScore-emr-t4.csv index e738376f1..550e35309 100644 --- a/core/src/main/resources/operatorsScore-emr-t4.csv +++ b/core/src/main/resources/operatorsScore-emr-t4.csv @@ -190,7 +190,8 @@ PosExplode,2.07 Pow,2.07 PreciseTimestampConversion,2.07 PromotePrecision,2.07 -PythonUDF,2.07 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,2.07 RLike,2.07 RaiseError,2.07 diff --git a/core/src/main/resources/operatorsScore-onprem-a100.csv b/core/src/main/resources/operatorsScore-onprem-a100.csv index 3ac26e5ac..732921c16 100644 --- a/core/src/main/resources/operatorsScore-onprem-a100.csv +++ b/core/src/main/resources/operatorsScore-onprem-a100.csv @@ -197,7 +197,8 @@ PosExplode,4 Pow,4 PreciseTimestampConversion,4 PromotePrecision,4 -PythonUDF,4 +PythonUDF,1.2 +PythonUDAF,1.2 Quarter,4 RLike,4 RaiseError,4 diff --git a/core/src/main/resources/supportedExprs.csv b/core/src/main/resources/supportedExprs.csv index c4fb75764..70ac9ae4d 100644 --- a/core/src/main/resources/supportedExprs.csv +++ b/core/src/main/resources/supportedExprs.csv @@ -766,14 +766,14 @@ Empty2Null,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA Empty2Null,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA KnownNullable,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S KnownNullable,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S -PythonUDAF,TNEW, ,None,aggregation,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS -PythonUDAF,TNEW, ,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA -PythonUDAF,TNEW, ,None,reduction,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS -PythonUDAF,TNEW, ,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA -PythonUDAF,TNEW, ,None,window,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS -PythonUDAF,TNEW, ,None,window,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA -PythonUDAF,TNEW, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS -PythonUDAF,TNEW, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA +PythonUDAF,S, ,None,aggregation,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +PythonUDAF,S, ,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA +PythonUDAF,S, ,None,reduction,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +PythonUDAF,S, ,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA +PythonUDAF,S, ,None,window,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +PythonUDAF,S, ,None,window,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA +PythonUDAF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +PythonUDAF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA EphemeralSubstring,S,`substr`; `substring`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NS,NS EphemeralSubstring,S,`substr`; `substring`,None,project,pos,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,NS EphemeralSubstring,S,`substr`; `substring`,None,project,len,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,NS diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/BatchEvalPythonExecParser.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/BatchEvalPythonExecParser.scala new file mode 100644 index 000000000..ab5220066 --- /dev/null +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/BatchEvalPythonExecParser.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.tool.planparser + +import com.nvidia.spark.rapids.tool.qualification.PluginTypeChecker + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.execution.ui.SparkPlanGraphNode + +case class BatchEvalPythonExecParser( + node: SparkPlanGraphNode, + checker: PluginTypeChecker, + sqlID: Long) extends ExecParser with Logging { + + val fullExecName = node.name + "Exec" + + override def parse: ExecInfo = { + val duration = None + // TODO - add in parsing expressions - average speedup across? + ExecInfo(node, sqlID, node.name, "", 1.0, duration, node.id, true, None) + } +} diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/ProjectExecParser.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/ProjectExecParser.scala index 838c1c6d2..5a13f9059 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/ProjectExecParser.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/ProjectExecParser.scala @@ -19,6 +19,7 @@ package com.nvidia.spark.rapids.tool.planparser import com.nvidia.spark.rapids.tool.qualification.PluginTypeChecker import org.apache.spark.sql.execution.ui.SparkPlanGraphNode +import org.apache.spark.sql.rapids.tool.ExecHelper case class ProjectExecParser( node: SparkPlanGraphNode, @@ -33,8 +34,9 @@ case class ProjectExecParser( val exprString = node.desc.replaceFirst("Project ", "") val expressions = SQLPlanParser.parseProjectExpressions(exprString) val notSupportedExprs = checker.getNotSupportedExprs(expressions) + val containsPythonUDF = ExecHelper.isPythonUDF(node) val (speedupFactor, isSupported) = if (checker.isExecSupported(fullExecName) && - notSupportedExprs.isEmpty) { + (notSupportedExprs.isEmpty || containsPythonUDF)) { (checker.getSpeedupFactor(fullExecName), true) } else { (1.0, false) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/SQLPlanParser.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/SQLPlanParser.scala index a82337dd5..98396aab2 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/SQLPlanParser.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/SQLPlanParser.scala @@ -471,6 +471,8 @@ object SQLPlanParser extends Logging { AggregateInPandasExecParser(node, checker, sqlID).parse case "ArrowEvalPython" => ArrowEvalPythonExecParser(node, checker, sqlID).parse + case "BatchEvalPython" => + BatchEvalPythonExecParser(node, checker, sqlID).parse case "BatchScan" => BatchScanExecParser(node, checker, sqlID, app).parse case "BroadcastExchange" => diff --git a/core/src/main/scala/org/apache/spark/sql/rapids/tool/AppBase.scala b/core/src/main/scala/org/apache/spark/sql/rapids/tool/AppBase.scala index af97aa7e6..6a24a14d6 100644 --- a/core/src/main/scala/org/apache/spark/sql/rapids/tool/AppBase.scala +++ b/core/src/main/scala/org/apache/spark/sql/rapids/tool/AppBase.scala @@ -331,7 +331,7 @@ abstract class AppBase( } } - private val UDFRegex = ".*UDF.*" + private val UDFRegex = ".*(? "UDF", diff --git a/core/src/main/scala/org/apache/spark/sql/rapids/tool/ToolUtils.scala b/core/src/main/scala/org/apache/spark/sql/rapids/tool/ToolUtils.scala index b736460ac..9955cbda9 100644 --- a/core/src/main/scala/org/apache/spark/sql/rapids/tool/ToolUtils.scala +++ b/core/src/main/scala/org/apache/spark/sql/rapids/tool/ToolUtils.scala @@ -357,7 +357,11 @@ object RDDCheckHelper { object ExecHelper { private val UDFRegExLookup = Set( - ".*UDF.*".r + ".*(? node.desc.matches(regEx.regex)) + } + def shouldBeRemoved(nodeName: String): Boolean = { execsToBeRemoved.contains(nodeName) } diff --git a/core/src/test/resources/spark-events-qualification/python_udf_eventlog b/core/src/test/resources/spark-events-qualification/python_udf_eventlog new file mode 100644 index 000000000..5245bcd71 --- /dev/null +++ b/core/src/test/resources/spark-events-qualification/python_udf_eventlog @@ -0,0 +1,153 @@ +{"Event":"SparkListenerLogStart","Spark Version":"3.4.0"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1713205378123,"Executor ID":"driver","Executor Info":{"Host":"10.110.46.117","Total Cores":64,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"10.110.46.117","Port":35485},"Maximum Memory":384093388,"Timestamp":1713205378136,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_402 (Private Build)","Scala Version":"version 2.12.17"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"10.110.46.117","spark.serializer.objectStreamReset":"100","spark.eventLog.enabled":"true","spark.driver.port":"43465","spark.rdd.compress":"True","spark.app.name":"PySparkShell","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.submitTime":"1713205377213","spark.app.startTime":"1713205377492","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"../../pyspark_eventlogs","spark.sql.catalogImplementation":"hive","spark.app.id":"local-1713205378082"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"96","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","fs.viewfs.overload.scheme.target.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"0","yarn.dispatcher.print-events-info.threshold":"5000","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","fs.s3a.metadatastore.fail.on.write.error":"true","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"s3.amazonaws.com","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.s3guard.consistency.retry.interval":"2s","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"64","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.s3guard.consistency.retry.limit":"7","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","fs.s3a.s3guard.ddb.table.sse.enabled":"false","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","fs.s3a.metadatastore.metadata.ttl":"15m","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${hadoop.tmp.dir}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","fs.s3a.connection.request.timeout":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"\n org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\n org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\n com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\n org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\n ","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.maintenance.version":"5","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Private Build","java.vm.specification.version":"1.8","user.home":"/home/cindyj","file.encoding.pkg":"sun.io","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.402-b06","jetty.git.hash":"da9a0b30691a45daf90a9f17b5defa2f1434f882","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_402-8u402-ga-2ubuntu1~20.04-b06","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"5.15.0-67-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Private Build","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"cindyj","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=../../pyspark_eventlogs --name PySparkShell pyspark-shell","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_402","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","*.sink.servlet.path":"/metrics/json","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json"},"Classpath Entries":{"/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/parquet-encoding-1.12.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/avro-1.11.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jersey-client-2.36.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/parquet-common-1.12.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/libthrift-0.12.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-common-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-exec-2.3.9-core.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/HikariCP-2.5.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/curator-client-2.13.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/orc-core-1.8.3-shaded-protobuf.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-extensions-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/RoaringBitmap-0.9.38.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/oro-2.0.8.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/stream-2.9.6.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-certificates-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/rocksdbjni-7.9.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-yarn_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/log4j-1.2-api-2.19.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/stax-api-1.0.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/snakeyaml-1.33.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/super-csv-2.2.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/zookeeper-jute-3.6.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-kubernetes_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-mllib_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-text-1.10.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/curator-framework-2.13.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-pool-1.5.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-graphx_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/gson-2.2.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-sketch_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jackson-module-scala_2.12-2.14.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-codec-http2-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/JTransforms-3.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/metrics-json-4.2.15.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/paranamer-2.8.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/metrics-jvm-4.2.15.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/janino-3.1.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/javolution-5.5.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-launcher_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-client-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jersey-container-servlet-core-2.36.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/ST4-4.0.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/arrow-format-11.0.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-transport-classes-epoll-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-policy-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jersey-hk2-2.36.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/tink-1.7.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/annotations-17.0.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/py4j-0.10.9.7.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-core_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hadoop-yarn-server-web-proxy-3.3.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-admissionregistration-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hk2-api-2.6.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/lz4-java-1.8.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-mesos_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-lang-2.6.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-common-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/joda-time-2.12.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-io-2.11.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-sql_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/scala-library-2.12.17.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-codec-socks-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-streaming_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/arpack-3.0.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jsr305-3.0.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/scala-parser-combinators_2.12-2.1.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-handler-proxy-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-gatewayapi-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hadoop-client-runtime-3.3.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/scala-reflect-2.12.17.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-repl_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/metrics-graphite-4.2.15.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jackson-dataformat-yaml-2.14.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-math3-3.6.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-mllib-local_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-kvstore_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jackson-databind-2.14.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.87.Final-osx-x86_64.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/log4j-core-2.19.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/log4j-slf4j2-impl-2.19.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/zookeeper-3.6.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/slf4j-api-2.0.6.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-apps-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/libfb303-0.9.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/objenesis-3.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jersey-common-2.36.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-flowcontrol-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/threeten-extra-1.7.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-cli-1.5.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-collections4-4.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/algebra_2.12-2.0.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-cli-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-scheduling-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/pickle-1.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/aircompressor-0.21.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-shims-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/json-1.8.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/minlog-1.3.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/arrow-memory-netty-11.0.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-transport-native-epoll-4.1.87.Final-linux-aarch_64.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/zstd-jni-1.5.2-5.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-tags_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/chill_2.12-0.10.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-transport-classes-kqueue-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/avro-mapred-1.11.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/activation-1.1.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/JLargeArrays-1.5.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-jdbc-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-shims-common-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/guava-14.0.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-handler-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/log4j-api-2.19.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/lapack-3.0.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hadoop-client-api-3.3.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-storage-api-2.8.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jersey-server-2.36.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/parquet-format-structures-1.12.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/parquet-jackson-1.12.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/derby-10.14.2.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/scala-compiler-2.12.17.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-hive_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/orc-shims-1.8.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/okhttp-3.12.12.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-transport-native-unix-common-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jdo-api-3.0.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/parquet-hadoop-1.12.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-node-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-logging-1.1.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/xz-1.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/transaction-api-1.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-dbcp-1.4.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spire_2.12-0.17.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-metrics-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-hive-thriftserver_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/xbean-asm9-shaded-4.22.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-coordination-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-transport-native-epoll-4.1.87.Final-linux-x86_64.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-common-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-storageclass-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jline-2.14.6.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/orc-mapreduce-1.8.3-shaded-protobuf.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jpam-1.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-discovery-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/conf/":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-httpclient-okhttp-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-all-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jackson-datatype-jsr310-2.14.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/metrics-core-4.2.15.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/metrics-jmx-4.2.15.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jackson-core-2.14.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/avro-ipc-1.11.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-catalyst_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/ivy-2.5.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-compiler-3.1.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/httpcore-4.4.16.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jcl-over-slf4j-2.0.6.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.87.Final-osx-aarch_64.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-collections-3.2.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/arrow-memory-core-11.0.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/okio-1.15.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-llap-common-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-compress-1.22.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/opencsv-2.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-resolver-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-batch-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jackson-annotations-2.14.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-buffer-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jersey-container-servlet-2.36.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/blas-3.0.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-serde-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/parquet-column-1.12.3.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/httpclient-4.5.14.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-codec-http-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-apiextensions-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-beeline-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/breeze_2.12-2.1.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/compress-lzf-1.1.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-core-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-unsafe_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-rbac-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-transport-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-autoscaling-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-codec-1.15.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-network-shuffle_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hive-metastore-2.3.9.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/spark-network-common_2.12-3.4.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jul-to-slf4j-2.0.6.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-events-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/netty-codec-4.1.87.Final.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jta-1.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/shims-0.9.38.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/snappy-java-1.1.9.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/arrow-vector-11.0.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/commons-lang3-3.12.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/jodd-core-3.5.2.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/chill-java-0.10.0.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-model-networking-6.4.1.jar":"System Classpath","/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/jars/kubernetes-client-api-6.4.1.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"PySparkShell","App ID":"local-1713205378082","Timestamp":1713205377492,"User":"cindyj"} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","physicalPlanDescription":"== Physical Plan ==\nCollectLimit (6)\n+- * Project (5)\n +- BatchEvalPython (4)\n +- * LocalLimit (3)\n +- * Project (2)\n +- * Scan ExistingRDD (1)\n\n\n(1) Scan ExistingRDD [codegen id : 1]\nOutput [3]: [id#0L, name#1, age#2L]\nArguments: [id#0L, name#1, age#2L], MapPartitionsRDD[4] at applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0, ExistingRDD, UnknownPartitioning(0)\n\n(2) Project [codegen id : 1]\nOutput [2]: [name#1, age#2L]\nInput [3]: [id#0L, name#1, age#2L]\n\n(3) LocalLimit [codegen id : 1]\nInput [2]: [name#1, age#2L]\nArguments: 21\n\n(4) BatchEvalPython\nInput [2]: [name#1, age#2L]\nArguments: [(name#1)#6, to_upper(name#1)#8, add_one(age#2L)#9], [pythonUDF0#24, pythonUDF1#25, pythonUDF2#26]\n\n(5) Project [codegen id : 2]\nOutput [3]: [cast(pythonUDF0#24 as string) AS slen(name)#18, pythonUDF1#25 AS to_upper(name)#19, cast(pythonUDF2#26 as string) AS add_one(age)#20]\nInput [5]: [name#1, age#2L, pythonUDF0#24, pythonUDF1#25, pythonUDF2#26]\n\n(6) CollectLimit\nInput [3]: [slen(name)#18, to_upper(name)#19, add_one(age)#20]\nArguments: 21\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Project","simpleString":"Project [cast(pythonUDF0#24 as string) AS slen(name)#18, pythonUDF1#25 AS to_upper(name)#19, cast(pythonUDF2#26 as string) AS add_one(age)#20]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BatchEvalPython","simpleString":"BatchEvalPython [(name#1)#6, to_upper(name#1)#8, add_one(age#2L)#9], [pythonUDF0#24, pythonUDF1#25, pythonUDF2#26]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [name#1, age#2L]","children":[{"nodeName":"Scan ExistingRDD","simpleString":"Scan ExistingRDD[id#0L,name#1,age#2L]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":35,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":34,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"data sent to Python workers","accumulatorId":10,"metricType":"size"},{"name":"data returned from Python workers","accumulatorId":11,"metricType":"size"},{"name":"number of output rows","accumulatorId":12,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":33,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":31,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":25,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":32,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":26,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":23,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":20,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":29,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":22,"metricType":"sum"},{"name":"records read","accumulatorId":19,"metricType":"sum"},{"name":"local bytes read","accumulatorId":17,"metricType":"size"},{"name":"fetch wait time","accumulatorId":18,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":15,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":21,"metricType":"sum"},{"name":"local blocks read","accumulatorId":14,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":24,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":13,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":27,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":28,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":16,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":30,"metricType":"size"}]},"time":1713205512992,"modifiedConfigs":{}} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1713205513323,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0],"Properties":{"spark.sql.warehouse.dir":"file:/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python/spark-warehouse","spark.executor.id":"driver","spark.driver.host":"10.110.46.117","spark.sql.catalogImplementation":"hive","spark.sql.execution.root.id":"0","spark.submit.deployMode":"client","spark.app.submitTime":"1713205377213","spark.eventLog.enabled":"true","spark.sql.execution.id":"0","spark.ui.showConsoleProgress":"true","spark.master":"local[*]","spark.serializer.objectStreamReset":"100","spark.submit.pyFiles":"","spark.app.startTime":"1713205377492","spark.rdd.compress":"True","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.id":"local-1713205378082","spark.app.name":"PySparkShell","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.port":"43465","spark.eventLog.dir":"../../pyspark_eventlogs"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Submission Time":1713205513335,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python/spark-warehouse","spark.executor.id":"driver","spark.driver.host":"10.110.46.117","spark.sql.catalogImplementation":"hive","spark.sql.execution.root.id":"0","spark.submit.deployMode":"client","spark.app.submitTime":"1713205377213","spark.eventLog.enabled":"true","spark.sql.execution.id":"0","spark.ui.showConsoleProgress":"true","spark.master":"local[*]","spark.serializer.objectStreamReset":"100","spark.submit.pyFiles":"","spark.app.startTime":"1713205377492","spark.rdd.compress":"True","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.id":"local-1713205378082","spark.app.name":"PySparkShell","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.port":"43465","spark.eventLog.dir":"../../pyspark_eventlogs"}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1713205513434,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1713205513434,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514172,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"27","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"213","Value":"213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"internal.metrics.executorDeserializeTime","Update":145,"Value":145,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.executorDeserializeCpuTime","Update":131090625,"Value":131090625,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.executorRunTime","Update":553,"Value":553,"Internal":true,"Count Failed Values":true},{"ID":39,"Name":"internal.metrics.executorCpuTime","Update":90683689,"Value":90683689,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.resultSize","Update":2180,"Value":2180,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.resultSerializationTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":145,"Executor Deserialize CPU Time":131090625,"Executor Run Time":553,"Executor CPU Time":90683689,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":5,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Submission Time":1713205513335,"Completion Time":1713205514183,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Value":"213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"internal.metrics.executorDeserializeTime","Value":145,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.executorDeserializeCpuTime","Value":131090625,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.executorRunTime","Value":553,"Internal":true,"Count Failed Values":true},{"ID":39,"Name":"internal.metrics.executorCpuTime","Value":90683689,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.resultSize","Value":2180,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.resultSerializationTime","Value":5,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1713205514187,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1713205514210,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":4,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[1],"Properties":{"spark.sql.warehouse.dir":"file:/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python/spark-warehouse","spark.executor.id":"driver","spark.driver.host":"10.110.46.117","spark.sql.catalogImplementation":"hive","spark.sql.execution.root.id":"0","spark.submit.deployMode":"client","spark.app.submitTime":"1713205377213","spark.eventLog.enabled":"true","spark.sql.execution.id":"0","spark.ui.showConsoleProgress":"true","spark.master":"local[*]","spark.serializer.objectStreamReset":"100","spark.submit.pyFiles":"","spark.app.startTime":"1713205377492","spark.rdd.compress":"True","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.id":"local-1713205378082","spark.app.name":"PySparkShell","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.port":"43465","spark.eventLog.dir":"../../pyspark_eventlogs"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":4,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Submission Time":1713205514212,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python/spark-warehouse","spark.executor.id":"driver","spark.driver.host":"10.110.46.117","spark.sql.catalogImplementation":"hive","spark.sql.execution.root.id":"0","spark.submit.deployMode":"client","spark.app.submitTime":"1713205377213","spark.eventLog.enabled":"true","spark.sql.execution.id":"0","spark.ui.showConsoleProgress":"true","spark.master":"local[*]","spark.serializer.objectStreamReset":"100","spark.submit.pyFiles":"","spark.app.startTime":"1713205377492","spark.rdd.compress":"True","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.id":"local-1713205378082","spark.app.name":"PySparkShell","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.port":"43465","spark.eventLog.dir":"../../pyspark_eventlogs"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":1,"Launch Time":1713205514217,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":1,"Attempt":0,"Partition ID":2,"Launch Time":1713205514218,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":2,"Attempt":0,"Partition ID":3,"Launch Time":1713205514218,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":3,"Attempt":0,"Partition ID":4,"Launch Time":1713205514219,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":1,"Attempt":0,"Partition ID":2,"Launch Time":1713205514218,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514300,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"40","Value":"67","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"42","Value":"255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":71,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6458793,"Value":6458793,"Internal":true,"Count Failed Values":true},{"ID":73,"Name":"internal.metrics.executorRunTime","Update":67,"Value":67,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.executorCpuTime","Update":10660767,"Value":10660767,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.resultSize","Update":2180,"Value":2180,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":6458793,"Executor Run Time":67,"Executor CPU Time":10660767,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":2,"Attempt":0,"Partition ID":3,"Launch Time":1713205514218,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514313,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"39","Value":"106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"43","Value":"298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":71,"Name":"internal.metrics.executorDeserializeTime","Update":31,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9650328,"Value":16109121,"Internal":true,"Count Failed Values":true},{"ID":73,"Name":"internal.metrics.executorRunTime","Update":57,"Value":124,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.executorCpuTime","Update":10573951,"Value":21234718,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.resultSize","Update":2180,"Value":4360,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":31,"Executor Deserialize CPU Time":9650328,"Executor Run Time":57,"Executor CPU Time":10573951,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":1,"Launch Time":1713205514217,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514317,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"37","Value":"143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"43","Value":"341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":71,"Name":"internal.metrics.executorDeserializeTime","Update":32,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10457855,"Value":26566976,"Internal":true,"Count Failed Values":true},{"ID":73,"Name":"internal.metrics.executorRunTime","Update":61,"Value":185,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.executorCpuTime","Update":11583721,"Value":32818439,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.resultSize","Update":2180,"Value":6540,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":32,"Executor Deserialize CPU Time":10457855,"Executor Run Time":61,"Executor CPU Time":11583721,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":3,"Attempt":0,"Partition ID":4,"Launch Time":1713205514219,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514318,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"40","Value":"183","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"41","Value":"382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":71,"Name":"internal.metrics.executorDeserializeTime","Update":31,"Value":103,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9645539,"Value":36212515,"Internal":true,"Count Failed Values":true},{"ID":73,"Name":"internal.metrics.executorRunTime","Update":62,"Value":247,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.executorCpuTime","Update":10688177,"Value":43506616,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.resultSize","Update":2180,"Value":8720,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":31,"Executor Deserialize CPU Time":9645539,"Executor Run Time":62,"Executor CPU Time":10688177,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":4,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Submission Time":1713205514212,"Completion Time":1713205514319,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Value":"183","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Value":"382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":71,"Name":"internal.metrics.executorDeserializeTime","Value":103,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.executorDeserializeCpuTime","Value":36212515,"Internal":true,"Count Failed Values":true},{"ID":73,"Name":"internal.metrics.executorRunTime","Value":247,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.executorCpuTime","Value":43506616,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.resultSize","Value":8720,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.jvmGCTime","Value":36,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1713205514320,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1713205514327,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[2],"Properties":{"spark.sql.warehouse.dir":"file:/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python/spark-warehouse","spark.executor.id":"driver","spark.driver.host":"10.110.46.117","spark.sql.catalogImplementation":"hive","spark.sql.execution.root.id":"0","spark.submit.deployMode":"client","spark.app.submitTime":"1713205377213","spark.eventLog.enabled":"true","spark.sql.execution.id":"0","spark.ui.showConsoleProgress":"true","spark.master":"local[*]","spark.serializer.objectStreamReset":"100","spark.submit.pyFiles":"","spark.app.startTime":"1713205377492","spark.rdd.compress":"True","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.id":"local-1713205378082","spark.app.name":"PySparkShell","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.port":"43465","spark.eventLog.dir":"../../pyspark_eventlogs"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Submission Time":1713205514332,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python/spark-warehouse","spark.executor.id":"driver","spark.driver.host":"10.110.46.117","spark.sql.catalogImplementation":"hive","spark.sql.execution.root.id":"0","spark.submit.deployMode":"client","spark.app.submitTime":"1713205377213","spark.eventLog.enabled":"true","spark.sql.execution.id":"0","spark.ui.showConsoleProgress":"true","spark.master":"local[*]","spark.serializer.objectStreamReset":"100","spark.submit.pyFiles":"","spark.app.startTime":"1713205377492","spark.rdd.compress":"True","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.id":"local-1713205378082","spark.app.name":"PySparkShell","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.port":"43465","spark.eventLog.dir":"../../pyspark_eventlogs"}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":5,"Launch Time":1713205514338,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":1,"Attempt":0,"Partition ID":6,"Launch Time":1713205514338,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":2,"Attempt":0,"Partition ID":7,"Launch Time":1713205514339,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":3,"Attempt":0,"Partition ID":8,"Launch Time":1713205514339,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":4,"Attempt":0,"Partition ID":9,"Launch Time":1713205514339,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":5,"Attempt":0,"Partition ID":10,"Launch Time":1713205514339,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":6,"Attempt":0,"Partition ID":11,"Launch Time":1713205514340,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":7,"Attempt":0,"Partition ID":12,"Launch Time":1713205514340,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":8,"Attempt":0,"Partition ID":13,"Launch Time":1713205514340,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":9,"Attempt":0,"Partition ID":14,"Launch Time":1713205514341,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":10,"Attempt":0,"Partition ID":15,"Launch Time":1713205514341,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":11,"Attempt":0,"Partition ID":16,"Launch Time":1713205514342,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":12,"Attempt":0,"Partition ID":17,"Launch Time":1713205514342,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":13,"Attempt":0,"Partition ID":18,"Launch Time":1713205514343,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":14,"Attempt":0,"Partition ID":19,"Launch Time":1713205514343,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":15,"Attempt":0,"Partition ID":20,"Launch Time":1713205514343,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":16,"Attempt":0,"Partition ID":21,"Launch Time":1713205514343,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":17,"Attempt":0,"Partition ID":22,"Launch Time":1713205514344,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":18,"Attempt":0,"Partition ID":23,"Launch Time":1713205514344,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":19,"Attempt":0,"Partition ID":24,"Launch Time":1713205514344,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":5,"Launch Time":1713205514338,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514493,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"102","Value":"285","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"50","Value":"432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":14233174,"Value":14233174,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":122,"Value":122,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":10366266,"Value":10366266,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2180,"Value":2180,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":14233174,"Executor Run Time":122,"Executor CPU Time":10366266,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":12,"Attempt":0,"Partition ID":17,"Launch Time":1713205514342,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514494,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"49","Value":"334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"77","Value":"509","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18372456,"Value":32605630,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":113,"Value":235,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":14439251,"Value":24805517,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":4317,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":18372456,"Executor Run Time":113,"Executor CPU Time":14439251,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":8,"Attempt":0,"Partition ID":13,"Launch Time":1713205514340,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514496,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"58","Value":"392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"64","Value":"573","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":68,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19021220,"Value":51626850,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":114,"Value":349,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":11793380,"Value":36598897,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":6454,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":19021220,"Executor Run Time":114,"Executor CPU Time":11793380,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":3,"Attempt":0,"Partition ID":8,"Launch Time":1713205514339,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514497,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"103","Value":"495","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"53","Value":"626","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":86,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13484238,"Value":65111088,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":122,"Value":471,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":10322908,"Value":46921805,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2180,"Value":8634,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":13484238,"Executor Run Time":122,"Executor CPU Time":10322908,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":4,"Attempt":0,"Partition ID":9,"Launch Time":1713205514339,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514497,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"103","Value":"598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"71","Value":"697","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":103,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13058916,"Value":78170004,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":125,"Value":596,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":14206077,"Value":61127882,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2180,"Value":10814,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":13058916,"Executor Run Time":125,"Executor CPU Time":14206077,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":5,"Attempt":0,"Partition ID":10,"Launch Time":1713205514339,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514497,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"102","Value":"700","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"50","Value":"747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":120,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13604744,"Value":91774748,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":125,"Value":721,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":11536134,"Value":72664016,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":12951,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":13604744,"Executor Run Time":125,"Executor CPU Time":11536134,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":19,"Attempt":0,"Partition ID":24,"Launch Time":1713205514344,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514497,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"49","Value":"749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"72","Value":"819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":22,"Value":142,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":16399892,"Value":108174640,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":114,"Value":835,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":14299206,"Value":86963222,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":15088,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":22,"Executor Deserialize CPU Time":16399892,"Executor Run Time":114,"Executor CPU Time":14299206,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":16,"Attempt":0,"Partition ID":21,"Launch Time":1713205514343,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514497,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"52","Value":"801","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"69","Value":"888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":165,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":15880193,"Value":124054833,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":114,"Value":949,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":17500470,"Value":104463692,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2180,"Value":17268,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":15880193,"Executor Run Time":114,"Executor CPU Time":17500470,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":18,"Attempt":0,"Partition ID":23,"Launch Time":1713205514344,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514498,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"56","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"41","Value":"842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"74","Value":"962","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":189,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":16018424,"Value":140073257,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":114,"Value":1063,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":17761589,"Value":122225281,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2180,"Value":19448,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":16018424,"Executor Run Time":114,"Executor CPU Time":17761589,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":11,"Attempt":0,"Partition ID":16,"Launch Time":1713205514342,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514501,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"46","Value":"888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"83","Value":"1045","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":26,"Value":215,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18082740,"Value":158155997,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":120,"Value":1183,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":11781764,"Value":134007045,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":21585,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":26,"Executor Deserialize CPU Time":18082740,"Executor Run Time":120,"Executor CPU Time":11781764,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":2,"Attempt":0,"Partition ID":7,"Launch Time":1713205514339,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514505,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"46","Value":"934","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"83","Value":"1128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":32,"Value":247,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21715590,"Value":179871587,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":121,"Value":1304,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":9671381,"Value":143678426,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":23722,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":32,"Executor Deserialize CPU Time":21715590,"Executor Run Time":121,"Executor CPU Time":9671381,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":15,"Attempt":0,"Partition ID":20,"Launch Time":1713205514343,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514507,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"68","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"45","Value":"979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"85","Value":"1213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":272,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":16975587,"Value":196847174,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":125,"Value":1429,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":11301087,"Value":154979513,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":25859,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":16975587,"Executor Run Time":125,"Executor CPU Time":11301087,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":17,"Attempt":0,"Partition ID":22,"Launch Time":1713205514344,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514510,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"72","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"42","Value":"1021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"86","Value":"1299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":297,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":17991020,"Value":214838194,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":126,"Value":1555,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":13177275,"Value":168156788,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":27996,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":17991020,"Executor Run Time":126,"Executor CPU Time":13177275,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":13,"Attempt":0,"Partition ID":18,"Launch Time":1713205514343,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514514,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"76","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"42","Value":"1063","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"87","Value":"1386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":27,"Value":324,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18120712,"Value":232958906,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":134,"Value":1689,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":14453453,"Value":182610241,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":30133,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":27,"Executor Deserialize CPU Time":18120712,"Executor Run Time":134,"Executor CPU Time":14453453,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":9,"Attempt":0,"Partition ID":14,"Launch Time":1713205514341,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514518,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"41","Value":"1104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"90","Value":"1476","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":27,"Value":351,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18687120,"Value":251646026,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":138,"Value":1827,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":12473691,"Value":195083932,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":32270,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":27,"Executor Deserialize CPU Time":18687120,"Executor Run Time":138,"Executor CPU Time":12473691,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":14,"Attempt":0,"Partition ID":19,"Launch Time":1713205514343,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514522,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"84","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"42","Value":"1146","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"92","Value":"1568","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":376,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18652924,"Value":270298950,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":143,"Value":1970,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":11306449,"Value":206390381,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":34407,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":18652924,"Executor Run Time":143,"Executor CPU Time":11306449,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":6,"Attempt":0,"Partition ID":11,"Launch Time":1713205514340,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514523,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"33","Value":"1179","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"95","Value":"1663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":401,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":17707192,"Value":288006142,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":148,"Value":2118,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":13451174,"Value":219841555,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":36544,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":17707192,"Executor Run Time":148,"Executor CPU Time":13451174,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":1,"Attempt":0,"Partition ID":6,"Launch Time":1713205514338,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514528,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"92","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"36","Value":"1215","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"105","Value":"1768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":21,"Value":422,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":15098624,"Value":303104766,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":159,"Value":2277,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":12254308,"Value":232095863,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":38681,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":21,"Executor Deserialize CPU Time":15098624,"Executor Run Time":159,"Executor CPU Time":12254308,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":7,"Attempt":0,"Partition ID":12,"Launch Time":1713205514340,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514531,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"96","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"36","Value":"1251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"66","Value":"1834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":447,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":17357441,"Value":320462207,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":156,"Value":2433,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":9402825,"Value":241498688,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":40818,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":17357441,"Executor Run Time":156,"Executor CPU Time":9402825,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":10,"Attempt":0,"Partition ID":15,"Launch Time":1713205514341,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514531,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"37","Value":"1288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"68","Value":"1902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Update":26,"Value":473,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Update":17624170,"Value":338086377,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Update":153,"Value":2586,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Update":12595334,"Value":254094022,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Update":2137,"Value":42955,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":26,"Executor Deserialize CPU Time":17624170,"Executor Run Time":153,"Executor CPU Time":12595334,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Submission Time":1713205514332,"Completion Time":1713205514533,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Value":"100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Value":"1288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Value":"1902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorDeserializeTime","Value":473,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.executorDeserializeCpuTime","Value":338086377,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.executorRunTime","Value":2586,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.executorCpuTime","Value":254094022,"Internal":true,"Count Failed Values":true},{"ID":110,"Name":"internal.metrics.resultSize","Value":42955,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.resultSerializationTime","Value":5,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1713205514533,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1713205514540,"Stage Infos":[{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":39,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[3],"Properties":{"spark.sql.warehouse.dir":"file:/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python/spark-warehouse","spark.executor.id":"driver","spark.driver.host":"10.110.46.117","spark.sql.catalogImplementation":"hive","spark.sql.execution.root.id":"0","spark.submit.deployMode":"client","spark.app.submitTime":"1713205377213","spark.eventLog.enabled":"true","spark.sql.execution.id":"0","spark.ui.showConsoleProgress":"true","spark.master":"local[*]","spark.serializer.objectStreamReset":"100","spark.submit.pyFiles":"","spark.app.startTime":"1713205377492","spark.rdd.compress":"True","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.id":"local-1713205378082","spark.app.name":"PySparkShell","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.port":"43465","spark.eventLog.dir":"../../pyspark_eventlogs"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":39,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Submission Time":1713205514543,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/cindyj/Desktop/spark-3.4.0-bin-hadoop3/python/spark-warehouse","spark.executor.id":"driver","spark.driver.host":"10.110.46.117","spark.sql.catalogImplementation":"hive","spark.sql.execution.root.id":"0","spark.submit.deployMode":"client","spark.app.submitTime":"1713205377213","spark.eventLog.enabled":"true","spark.sql.execution.id":"0","spark.ui.showConsoleProgress":"true","spark.master":"local[*]","spark.serializer.objectStreamReset":"100","spark.submit.pyFiles":"","spark.app.startTime":"1713205377492","spark.rdd.compress":"True","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.id":"local-1713205378082","spark.app.name":"PySparkShell","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.port":"43465","spark.eventLog.dir":"../../pyspark_eventlogs"}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":0,"Attempt":0,"Partition ID":25,"Launch Time":1713205514549,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":1,"Attempt":0,"Partition ID":26,"Launch Time":1713205514550,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":2,"Attempt":0,"Partition ID":27,"Launch Time":1713205514550,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":3,"Attempt":0,"Partition ID":28,"Launch Time":1713205514551,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":4,"Attempt":0,"Partition ID":29,"Launch Time":1713205514551,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":5,"Attempt":0,"Partition ID":30,"Launch Time":1713205514551,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":6,"Attempt":0,"Partition ID":31,"Launch Time":1713205514551,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":7,"Attempt":0,"Partition ID":32,"Launch Time":1713205514552,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":8,"Attempt":0,"Partition ID":33,"Launch Time":1713205514552,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":9,"Attempt":0,"Partition ID":34,"Launch Time":1713205514552,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":10,"Attempt":0,"Partition ID":35,"Launch Time":1713205514553,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":11,"Attempt":0,"Partition ID":36,"Launch Time":1713205514553,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":12,"Attempt":0,"Partition ID":37,"Launch Time":1713205514553,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":13,"Attempt":0,"Partition ID":38,"Launch Time":1713205514553,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":14,"Attempt":0,"Partition ID":39,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":40,"Index":15,"Attempt":0,"Partition ID":40,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":41,"Index":16,"Attempt":0,"Partition ID":41,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":42,"Index":17,"Attempt":0,"Partition ID":42,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":43,"Index":18,"Attempt":0,"Partition ID":43,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":44,"Index":19,"Attempt":0,"Partition ID":44,"Launch Time":1713205514555,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":45,"Index":20,"Attempt":0,"Partition ID":45,"Launch Time":1713205514555,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":46,"Index":21,"Attempt":0,"Partition ID":46,"Launch Time":1713205514555,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":47,"Index":22,"Attempt":0,"Partition ID":47,"Launch Time":1713205514555,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":48,"Index":23,"Attempt":0,"Partition ID":48,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":49,"Index":24,"Attempt":0,"Partition ID":49,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":50,"Index":25,"Attempt":0,"Partition ID":50,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":51,"Index":26,"Attempt":0,"Partition ID":51,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":52,"Index":27,"Attempt":0,"Partition ID":52,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":53,"Index":28,"Attempt":0,"Partition ID":53,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":54,"Index":29,"Attempt":0,"Partition ID":54,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":55,"Index":30,"Attempt":0,"Partition ID":55,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":56,"Index":31,"Attempt":0,"Partition ID":56,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":57,"Index":32,"Attempt":0,"Partition ID":57,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":58,"Index":33,"Attempt":0,"Partition ID":58,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":59,"Index":34,"Attempt":0,"Partition ID":59,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":60,"Index":35,"Attempt":0,"Partition ID":60,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":61,"Index":36,"Attempt":0,"Partition ID":61,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":62,"Index":37,"Attempt":0,"Partition ID":62,"Launch Time":1713205514558,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":63,"Index":38,"Attempt":0,"Partition ID":63,"Launch Time":1713205514558,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":10,"Attempt":0,"Partition ID":35,"Launch Time":1713205514553,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514749,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"107","Value":"1395","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"123","Value":"2025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5891358,"Value":5891358,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":148,"Value":148,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":9808449,"Value":9808449,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2137,"Value":2137,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":5891358,"Executor Run Time":148,"Executor CPU Time":9808449,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":57,"Index":32,"Attempt":0,"Partition ID":57,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514749,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"108","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"112","Value":"1507","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"139","Value":"2164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4429918,"Value":10321276,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":163,"Value":311,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":8457562,"Value":18266011,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2137,"Value":4274,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":4429918,"Executor Run Time":163,"Executor CPU Time":8457562,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":56,"Index":31,"Attempt":0,"Partition ID":56,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514788,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"107","Value":"1614","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"123","Value":"2287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4628340,"Value":14949616,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":167,"Value":478,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":10669183,"Value":28935194,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":6454,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4628340,"Executor Run Time":167,"Executor CPU Time":10669183,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":50,"Index":25,"Attempt":0,"Partition ID":50,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514798,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"112","Value":"1726","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"127","Value":"2414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5822700,"Value":20772316,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":167,"Value":645,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":10481405,"Value":39416599,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":8634,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":5822700,"Executor Run Time":167,"Executor CPU Time":10481405,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":55,"Index":30,"Attempt":0,"Partition ID":55,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514798,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"109","Value":"1835","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"127","Value":"2541","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5731119,"Value":26503435,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":162,"Value":807,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":9950122,"Value":49366721,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":10814,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":5731119,"Executor Run Time":162,"Executor CPU Time":9950122,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":58,"Index":33,"Attempt":0,"Partition ID":58,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514799,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"100","Value":"1935","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"118","Value":"2659","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":63,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6002668,"Value":32506103,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":143,"Value":950,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":8183238,"Value":57549959,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":12994,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":6002668,"Executor Run Time":143,"Executor CPU Time":8183238,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":47,"Index":22,"Attempt":0,"Partition ID":47,"Launch Time":1713205514555,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514799,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"107","Value":"2042","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"130","Value":"2789","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":81,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7279801,"Value":39785904,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":147,"Value":1097,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":11625650,"Value":69175609,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2137,"Value":15131,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":7279801,"Executor Run Time":147,"Executor CPU Time":11625650,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":9,"Attempt":0,"Partition ID":34,"Launch Time":1713205514552,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514799,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"110","Value":"2152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"139","Value":"2928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":22,"Value":103,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6239429,"Value":46025333,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":162,"Value":1259,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":11661520,"Value":80837129,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":17311,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":22,"Executor Deserialize CPU Time":6239429,"Executor Run Time":162,"Executor CPU Time":11661520,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":46,"Index":21,"Attempt":0,"Partition ID":46,"Launch Time":1713205514555,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514799,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"112","Value":"2264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"129","Value":"3057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":117,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5200744,"Value":51226077,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":164,"Value":1423,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":12907851,"Value":93744980,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2137,"Value":19448,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":5200744,"Executor Run Time":164,"Executor CPU Time":12907851,"Peak Execution Memory":0,"Result Size":2137,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":49,"Index":24,"Attempt":0,"Partition ID":49,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514800,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"116","Value":"2380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"131","Value":"3188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":134,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7568216,"Value":58794293,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":159,"Value":1582,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":9985720,"Value":103730700,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":21628,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":7568216,"Executor Run Time":159,"Executor CPU Time":9985720,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":11,"Attempt":0,"Partition ID":36,"Launch Time":1713205514553,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514800,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"106","Value":"2486","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"119","Value":"3307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":19,"Value":153,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6562137,"Value":65356430,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":161,"Value":1743,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":10218284,"Value":113948984,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":23808,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":19,"Executor Deserialize CPU Time":6562137,"Executor Run Time":161,"Executor CPU Time":10218284,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":13,"Attempt":0,"Partition ID":38,"Launch Time":1713205514553,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514801,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"148","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"107","Value":"2593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"126","Value":"3433","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":171,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7505407,"Value":72861837,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":166,"Value":1909,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":14692987,"Value":128641971,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":25988,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":7505407,"Executor Run Time":166,"Executor CPU Time":14692987,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":3,"Attempt":0,"Partition ID":28,"Launch Time":1713205514551,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514801,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"101","Value":"2694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"44","Value":"3477","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":182,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5187570,"Value":78049407,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":152,"Value":2061,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":8173847,"Value":136815818,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":28168,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":5187570,"Executor Run Time":152,"Executor CPU Time":8173847,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":60,"Index":35,"Attempt":0,"Partition ID":60,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514801,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"156","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"122","Value":"2816","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"119","Value":"3596","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":190,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5850984,"Value":83900391,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":208,"Value":2269,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":12587215,"Value":149403033,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":30391,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":5850984,"Executor Run Time":208,"Executor CPU Time":12587215,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":53,"Index":28,"Attempt":0,"Partition ID":53,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514801,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"86","Value":"2902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"55","Value":"3651","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":203,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7578794,"Value":91479185,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":209,"Value":2478,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":9300085,"Value":158703118,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":32614,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":7578794,"Executor Run Time":209,"Executor CPU Time":9300085,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":61,"Index":36,"Attempt":0,"Partition ID":61,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514802,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"120","Value":"3022","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"127","Value":"3778","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":207,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4734601,"Value":96213786,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":164,"Value":2642,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":12166617,"Value":170869735,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":34794,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4734601,"Executor Run Time":164,"Executor CPU Time":12166617,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":54,"Index":29,"Attempt":0,"Partition ID":54,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514802,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"123","Value":"3145","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"119","Value":"3897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":219,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5924291,"Value":102138077,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":218,"Value":2860,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":13270315,"Value":184140050,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":37017,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":81,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":5924291,"Executor Run Time":218,"Executor CPU Time":13270315,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":2,"Attempt":0,"Partition ID":27,"Launch Time":1713205514550,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514802,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"100","Value":"3245","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"142","Value":"4039","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":237,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6319260,"Value":108457337,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":170,"Value":3030,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":10799691,"Value":194939741,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":39240,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":108,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":45,"Value":73,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":6319260,"Executor Run Time":170,"Executor CPU Time":10799691,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":45,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":42,"Index":17,"Attempt":0,"Partition ID":42,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514803,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"82","Value":"3327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"62","Value":"4101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":22,"Value":259,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5223046,"Value":113680383,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":216,"Value":3246,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":12281690,"Value":207221431,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":41463,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":135,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":74,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":22,"Executor Deserialize CPU Time":5223046,"Executor Run Time":216,"Executor CPU Time":12281690,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":41,"Index":16,"Attempt":0,"Partition ID":41,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514803,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"80","Value":"3407","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"79","Value":"4180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":282,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4628320,"Value":118308703,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":217,"Value":3463,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":13755277,"Value":220976708,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":43643,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":162,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":4628320,"Executor Run Time":217,"Executor CPU Time":13755277,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":27,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":52,"Index":27,"Attempt":0,"Partition ID":52,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514803,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"65","Value":"3472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"93","Value":"4273","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":302,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5057213,"Value":123365916,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":215,"Value":3678,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":10135521,"Value":231112229,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":45866,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":189,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":75,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":5057213,"Executor Run Time":215,"Executor CPU Time":10135521,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":0,"Attempt":0,"Partition ID":25,"Launch Time":1713205514549,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514806,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"4","Value":"3476","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"125","Value":"4398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":15,"Value":317,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6391547,"Value":129757463,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":224,"Value":3902,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":13860596,"Value":244972825,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":48089,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":216,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":76,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":15,"Executor Deserialize CPU Time":6391547,"Executor Run Time":224,"Executor CPU Time":13860596,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":51,"Index":26,"Attempt":0,"Partition ID":51,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514807,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"192","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"83","Value":"3559","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"132","Value":"4530","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":331,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4340828,"Value":134098291,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":221,"Value":4123,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":13941304,"Value":258914129,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":50312,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":243,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":4340828,"Executor Run Time":221,"Executor CPU Time":13941304,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":7,"Attempt":0,"Partition ID":32,"Launch Time":1713205514552,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514813,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"93","Value":"3652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"59","Value":"4589","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":354,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5337550,"Value":139435841,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":218,"Value":4341,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":10782698,"Value":269696827,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":52535,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":270,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":78,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":5337550,"Executor Run Time":218,"Executor CPU Time":10782698,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":1,"Attempt":0,"Partition ID":26,"Launch Time":1713205514550,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514814,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"127","Value":"3779","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"119","Value":"4708","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":372,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7007880,"Value":146443721,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":221,"Value":4562,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":9266709,"Value":278963536,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":54758,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":297,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":81,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":7007880,"Executor Run Time":221,"Executor CPU Time":9266709,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":12,"Attempt":0,"Partition ID":37,"Launch Time":1713205514553,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514815,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"86","Value":"3865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"62","Value":"4770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":395,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5382924,"Value":151826645,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":215,"Value":4777,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":11595245,"Value":290558781,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":56981,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":324,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":5382924,"Executor Run Time":215,"Executor CPU Time":11595245,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":4,"Attempt":0,"Partition ID":29,"Launch Time":1713205514551,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514816,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"47","Value":"3912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"89","Value":"4859","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":420,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4755369,"Value":156582014,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":213,"Value":4990,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":17804770,"Value":308363551,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":59204,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":351,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":87,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":4755369,"Executor Run Time":213,"Executor CPU Time":17804770,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":8,"Attempt":0,"Partition ID":33,"Launch Time":1713205514552,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514817,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"125","Value":"4037","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"118","Value":"4977","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":440,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6160143,"Value":162742157,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":219,"Value":5209,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":9142144,"Value":317505695,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":61427,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":378,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":90,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":6160143,"Executor Run Time":219,"Executor CPU Time":9142144,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":5,"Attempt":0,"Partition ID":30,"Launch Time":1713205514551,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514820,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"67","Value":"4104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"94","Value":"5071","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":463,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4288594,"Value":167030751,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":220,"Value":5429,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":14433542,"Value":331939237,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":63650,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":405,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":91,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":4288594,"Executor Run Time":220,"Executor CPU Time":14433542,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":14,"Attempt":0,"Partition ID":39,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514820,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"220","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"126","Value":"4230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"144","Value":"5215","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":483,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6130556,"Value":173161307,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":220,"Value":5649,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":14239424,"Value":346178661,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":65873,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":432,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":94,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":6130556,"Executor Run Time":220,"Executor CPU Time":14239424,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":48,"Index":23,"Attempt":0,"Partition ID":48,"Launch Time":1713205514556,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514820,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"118","Value":"4348","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"85","Value":"5300","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":503,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4403871,"Value":177565178,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":233,"Value":5882,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":17106775,"Value":363285436,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":68096,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":459,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":95,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":4403871,"Executor Run Time":233,"Executor CPU Time":17106775,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":43,"Index":18,"Attempt":0,"Partition ID":43,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514820,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"228","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"77","Value":"4425","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"87","Value":"5387","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":526,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6006844,"Value":183572022,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":233,"Value":6115,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":12279758,"Value":375565194,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":70276,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":486,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":6006844,"Executor Run Time":233,"Executor CPU Time":12279758,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":27,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":40,"Index":15,"Attempt":0,"Partition ID":40,"Launch Time":1713205514554,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514821,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"0","Value":"4425","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"175","Value":"5562","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":26,"Value":552,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3513962,"Value":187085984,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":230,"Value":6345,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":11202499,"Value":386767693,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":72456,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":513,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":26,"Executor Deserialize CPU Time":3513962,"Executor Run Time":230,"Executor CPU Time":11202499,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":27,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":45,"Index":20,"Attempt":0,"Partition ID":45,"Launch Time":1713205514555,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514821,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"2","Value":"4427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"115","Value":"5677","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":564,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3625670,"Value":190711654,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":243,"Value":6588,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":16007486,"Value":402775179,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":74679,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":540,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":96,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":3625670,"Executor Run Time":243,"Executor CPU Time":16007486,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":6,"Attempt":0,"Partition ID":31,"Launch Time":1713205514551,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514822,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"1","Value":"4428","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"143","Value":"5820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":580,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5924863,"Value":196636517,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":240,"Value":6828,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":11035392,"Value":413810571,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":76902,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":567,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":97,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":5924863,"Executor Run Time":240,"Executor CPU Time":11035392,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":62,"Index":37,"Attempt":0,"Partition ID":62,"Launch Time":1713205514558,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514822,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"244","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"11","Value":"4439","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"143","Value":"5963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":584,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4736893,"Value":201373410,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":235,"Value":7063,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":9353753,"Value":423164324,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2223,"Value":79125,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":594,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":98,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4736893,"Executor Run Time":235,"Executor CPU Time":9353753,"Peak Execution Memory":0,"Result Size":2223,"JVM GC Time":27,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":63,"Index":38,"Attempt":0,"Partition ID":63,"Launch Time":1713205514558,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514827,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"28","Value":"272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":11,"Name":"data returned from Python workers","Update":"32","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":12,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"78","Value":"4517","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"404","Value":"6367","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":592,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5208429,"Value":206581839,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":241,"Value":7304,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":14806081,"Value":437970405,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2231,"Value":81356,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":621,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":5208429,"Executor Run Time":241,"Executor CPU Time":14806081,"Peak Execution Memory":0,"Result Size":2231,"JVM GC Time":27,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":59,"Index":34,"Attempt":0,"Partition ID":59,"Launch Time":1713205514557,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514842,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"276","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"40","Value":"4557","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"101","Value":"6468","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":601,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4055405,"Value":210637244,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":260,"Value":7564,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":11673297,"Value":449643702,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":83536,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":648,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":4055405,"Executor Run Time":260,"Executor CPU Time":11673297,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":27,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":44,"Index":19,"Attempt":0,"Partition ID":44,"Launch Time":1713205514555,"Executor ID":"driver","Host":"10.110.46.117","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1713205514847,"Failed":false,"Killed":false,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Update":"4","Value":"280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Update":"44","Value":"4601","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Update":"183","Value":"6651","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":625,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4976534,"Value":215613778,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Update":261,"Value":7825,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Update":13923573,"Value":463567275,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Update":2180,"Value":85716,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Update":27,"Value":675,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":4976534,"Executor Run Time":261,"Executor CPU Time":13923573,"Peak Execution Memory":0,"Result Size":2180,"JVM GC Time":27,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":39,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"mapPartitions at SerDeUtil.scala:117","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"mapPartitions\"}","Callsite":"applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"map\"}","Callsite":"map at SerDeUtil.scala:69","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"PythonRDD","Callsite":"RDD at PythonRDD.scala:53","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"BatchEvalPython\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"readRDDFromFile at PythonRDD.scala:287","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":64,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:323)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.lang.Thread.run(Thread.java:750)","Submission Time":1713205514543,"Completion Time":1713205514847,"Accumulables":[{"ID":10,"Name":"data sent to Python workers","Value":"280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":11,"Name":"data returned from Python workers","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":12,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"duration","Value":"4601","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"duration","Value":"6651","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"internal.metrics.executorDeserializeTime","Value":625,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.executorDeserializeCpuTime","Value":215613778,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.executorRunTime","Value":7825,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.executorCpuTime","Value":463567275,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.resultSize","Value":85716,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.jvmGCTime","Value":675,"Internal":true,"Count Failed Values":true},{"ID":147,"Name":"internal.metrics.resultSerializationTime","Value":98,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1713205514848,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1713205514871,"errorMessage":""} +{"Event":"SparkListenerApplicationEnd","Timestamp":1713205516452} diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala index 71a46f72c..b9207dbd3 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala @@ -858,6 +858,19 @@ class SQLPlanParserSuite extends BaseTestSuite { } } + test("Python UDFs should be supported") { + val eventLog = s"$qualLogDir/python_udf_eventlog" + val pluginTypeChecker = new PluginTypeChecker() + val app = createAppFromEventlog(eventLog) + assert(app.sqlPlans.size == 1) + val parsedPlans = app.sqlPlans.map { case (sqlID, plan) => + SQLPlanParser.parseSQLPlan(app.appId, plan, sqlID, "", pluginTypeChecker, app) + } + val execInfo = getAllExecsFromPlan(parsedPlans.toSeq) + val projectExec = execInfo.filter(_.exec.contains("Project")) + assertSizeAndSupported(2, projectExec) + } + test("Expression not supported in FilterExec") { TrampolineUtil.withTempDir { eventLogDir => val (eventLog, _) = ToolTestUtils.generateEventLog(eventLogDir, diff --git a/scripts/sync_plugin_files/override_supported_configs.json b/scripts/sync_plugin_files/override_supported_configs.json index 685356784..e1b88a2f0 100644 --- a/scripts/sync_plugin_files/override_supported_configs.json +++ b/scripts/sync_plugin_files/override_supported_configs.json @@ -77,94 +77,6 @@ } ] }, - { - "Expression": "PythonUDAF", - "Context": "aggregation", - "Params": "param", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "PythonUDAF", - "Context": "aggregation", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "PythonUDAF", - "Context": "reduction", - "Params": "param", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "PythonUDAF", - "Context": "reduction", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "PythonUDAF", - "Context": "window", - "Params": "param", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "PythonUDAF", - "Context": "window", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "PythonUDAF", - "Context": "project", - "Params": "param", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "PythonUDAF", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, { "Expression": "RoundCeil", "Context": "project",