SP1 - Details for Job 138

Status: SUCCEEDED
Submitted: 2026/04/20 16:16:18
Duration: 0.4 s
Associated SQL Query: 149
Job Group: 9ac214b2-87e3-41d6-a889-e95ec7e48cf0
Completed Stages: 1
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_221 { label="Stage 221 (skipped)"; subgraph cluster2053 { label="Scan csv "; 982 [labelType="html" label="FileScanRDD [982] toJavaRDD at AbsExportExecutor.java:55"]; 983 [labelType="html" label="MapPartitionsRDD [983] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2049 { label="WholeStageCodegen (1)"; 984 [labelType="html" label="MapPartitionsRDD [984] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2059 { label="Scan parquet "; 985 [labelType="html" label="FileScanRDD [985] toJavaRDD at AbsExportExecutor.java:55"]; 986 [labelType="html" label="MapPartitionsRDD [986] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2054 { label="WholeStageCodegen (2)"; 987 [labelType="html" label="MapPartitionsRDD [987] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2048 { label="Union"; 988 [labelType="html" label="UnionRDD [988] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2047 { label="Exchange"; 989 [labelType="html" label="MapPartitionsRDD [989] toJavaRDD at AbsExportExecutor.java:55"]; } } 982->983; 983->984; 985->986; 986->987; 984->988; 987->988; 988->989; }

digraph G { subgraph clusterstage_222 { label="Stage 222 (skipped)"; subgraph cluster2066 { label="Scan csv "; 990 [labelType="html" label="FileScanRDD [990] toJavaRDD at AbsExportExecutor.java:55"]; 991 [labelType="html" label="MapPartitionsRDD [991] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2062 { label="WholeStageCodegen (3)"; 992 [labelType="html" label="MapPartitionsRDD [992] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2071 { label="Scan csv "; 993 [labelType="html" label="FileScanRDD [993] toJavaRDD at AbsExportExecutor.java:55"]; 994 [labelType="html" label="MapPartitionsRDD [994] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2067 { label="WholeStageCodegen (4)"; 995 [labelType="html" label="MapPartitionsRDD [995] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2061 { label="Union"; 996 [labelType="html" label="UnionRDD [996] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2060 { label="Exchange"; 997 [labelType="html" label="MapPartitionsRDD [997] toJavaRDD at AbsExportExecutor.java:55"]; } } 990->991; 991->992; 993->994; 994->995; 992->996; 995->996; 996->997; }

digraph G { subgraph clusterstage_223 { label="Stage 223 (skipped)"; subgraph cluster2084 { label="AQEShuffleRead"; 1000 [labelType="html" label="ShuffledRowRDD [1000] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2081 { label="WholeStageCodegen (6)"; 1001 [labelType="html" label="MapPartitionsRDD [1001] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2089 { label="AQEShuffleRead"; 1002 [labelType="html" label="ShuffledRowRDD [1002] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2086 { label="WholeStageCodegen (7)"; 1003 [labelType="html" label="MapPartitionsRDD [1003] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2077 { label="WholeStageCodegen (8)"; 1004 [labelType="html" label="ZippedPartitionsRDD2 [1004] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; 1005 [labelType="html" label="MapPartitionsRDD [1005] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2076 { label="Exchange"; 1006 [labelType="html" label="MapPartitionsRDD [1006] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } } 1000->1001; 1002->1003; 1001->1004; 1003->1004; 1004->1005; 1005->1006; }

989,1000

997,1002

digraph G { subgraph clusterstage_224 { label="Stage 224 (skipped)"; 981 [labelType="html" label="JDBCRDD [981] toJavaRDD at AbsExportExecutor.java:55"]; subgraph cluster2073 { label="WholeStageCodegen (5)"; 998 [labelType="html" label="MapPartitionsRDD [998] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2072 { label="Exchange"; 999 [labelType="html" label="MapPartitionsRDD [999] toJavaRDD at AbsExportExecutor.java:55"]; } } 981->998; 998->999; }

digraph G { subgraph clusterstage_225 { label="Stage 225"; 1014 [labelType="html" label="SQLExecutionRDD [1014] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; subgraph cluster2100 { label="AQEShuffleRead"; 1007 [labelType="html" label="ShuffledRowRDD [1007] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2097 { label="WholeStageCodegen (9)"; 1008 [labelType="html" label="MapPartitionsRDD [1008] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2105 { label="AQEShuffleRead"; 1009 [labelType="html" label="ShuffledRowRDD [1009] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2102 { label="WholeStageCodegen (10)"; 1010 [labelType="html" label="MapPartitionsRDD [1010] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2091 { label="WholeStageCodegen (11)"; 1011 [labelType="html" label="ZippedPartitionsRDD2 [1011] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; 1012 [labelType="html" label="MapPartitionsRDD [1012] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2090 { label="DeserializeToObject"; 1013 [labelType="html" label="MapPartitionsRDD [1013] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2106 { label="mapPartitions"; 1015 [labelType="html" label="MapPartitionsRDD [1015] [Unordered] toJavaRDD at AbsExportExecutor.java:55"]; } subgraph cluster2107 { label="map"; 1016 [labelType="html" label="MapPartitionsRDD [1016] [Unordered] createDataFrame at AbsExportExecutor.java:55"]; } subgraph cluster2114 { label="WholeStageCodegen (1)"; 1021 [labelType="html" label="MapPartitionsRDD [1021] [Unordered] parquet at DataFrameParquetWriter.java:12"]; } subgraph cluster2113 { label="Coalesce"; 1022 [labelType="html" label="CoalescedRDD [1022] [Unordered] parquet at DataFrameParquetWriter.java:12"]; } subgraph cluster2112 { label="WriteFiles"; 1023 [labelType="html" label="MapPartitionsRDD [1023] [Unordered] parquet at DataFrameParquetWriter.java:12"]; } } 1007->1008; 1009->1010; 1008->1011; 1010->1011; 1011->1012; 1012->1013; 1013->1014; 1014->1015; 1015->1016; 1016->1021; 1021->1022; 1022->1023; }

1006,1007

999,1009

Completed Stages (1)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
225	default	DataFrameExportJob_exec_file parquet at DataFrameParquetWriter.java:12 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.exportwriter.DataFrameParquetWriter.write(DataFrameParquetWriter.java:12) com.adb.hdh.spark.job.impl.exportwriter.AbsExportToFileWriter.writeFiles(AbsExportToFileWriter.java:79) com.adb.hdh.spark.job.impl.exportwriter.AbsExportToFileWriter.write(AbsExportToFileWriter.java:58) com.adb.hdh.spark.job.impl.exportexecutor.ExportToFileExecutorImpl.performExport(ExportToFileExecutorImpl.java:69) com.adb.hdh.spark.job.impl.exportexecutor.AbsExportExecutor.writeDataFrame(AbsExportExecutor.java:58) com.adb.hdh.spark.job.impl.exportexecutor.ExportToFileExecutorImpl.execute(ExportToFileExecutorImpl.java:38) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:160) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	2026/04/20 16:16:18	0.3 s	1/1		259.9 KiB	2.2 MiB

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

225

default

DataFrameExportJob_exec_file parquet at DataFrameParquetWriter.java:12 +details

org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)
com.adb.hdh.spark.job.impl.exportwriter.DataFrameParquetWriter.write(DataFrameParquetWriter.java:12)
com.adb.hdh.spark.job.impl.exportwriter.AbsExportToFileWriter.writeFiles(AbsExportToFileWriter.java:79)
com.adb.hdh.spark.job.impl.exportwriter.AbsExportToFileWriter.write(AbsExportToFileWriter.java:58)
com.adb.hdh.spark.job.impl.exportexecutor.ExportToFileExecutorImpl.performExport(ExportToFileExecutorImpl.java:69)
com.adb.hdh.spark.job.impl.exportexecutor.AbsExportExecutor.writeDataFrame(AbsExportExecutor.java:58)
com.adb.hdh.spark.job.impl.exportexecutor.ExportToFileExecutorImpl.execute(ExportToFileExecutorImpl.java:38)
com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:160)
com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47)
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
java.base/java.util.concurrent.FutureTask.run(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.base/java.lang.Thread.run(Unknown Source)

2026/04/20 16:16:18

0.3 s

1/1

259.9 KiB

2.2 MiB

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total
224	default	toJavaRDD at AbsExportExecutor.java:55 +details org.apache.spark.sql.Dataset.toJavaRDD(Dataset.scala:3861) com.adb.hdh.spark.job.impl.exportexecutor.AbsExportExecutor.writeDataFrame(AbsExportExecutor.java:55) com.adb.hdh.spark.job.impl.exportexecutor.ExportToFileExecutorImpl.execute(ExportToFileExecutorImpl.java:38) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:160) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1
223	default	toJavaRDD at AbsExportExecutor.java:55 +details org.apache.spark.sql.Dataset.toJavaRDD(Dataset.scala:3861) com.adb.hdh.spark.job.impl.exportexecutor.AbsExportExecutor.writeDataFrame(AbsExportExecutor.java:55) com.adb.hdh.spark.job.impl.exportexecutor.ExportToFileExecutorImpl.execute(ExportToFileExecutorImpl.java:38) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:160) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1
222	default	toJavaRDD at AbsExportExecutor.java:55 +details org.apache.spark.sql.Dataset.toJavaRDD(Dataset.scala:3861) com.adb.hdh.spark.job.impl.exportexecutor.AbsExportExecutor.writeDataFrame(AbsExportExecutor.java:55) com.adb.hdh.spark.job.impl.exportexecutor.ExportToFileExecutorImpl.execute(ExportToFileExecutorImpl.java:38) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:160) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/2
221	default	toJavaRDD at AbsExportExecutor.java:55 +details org.apache.spark.sql.Dataset.toJavaRDD(Dataset.scala:3861) com.adb.hdh.spark.job.impl.exportexecutor.AbsExportExecutor.writeDataFrame(AbsExportExecutor.java:55) com.adb.hdh.spark.job.impl.exportexecutor.ExportToFileExecutorImpl.execute(ExportToFileExecutorImpl.java:38) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:160) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/2

Page: