hdh-spark-job - Details for Job 1879

Status: SUCCEEDED
Submitted: 2026/08/31 16:00:23
Duration: 6 s
Associated SQL Query: 1443
Job Group: 8568145d-a538-4d18-a2bf-46139a7e41ae
Completed Stages: 1
Skipped Stages: 5

Enable zooming

digraph G { subgraph clusterstage_2841 { label="Stage 2841 (skipped)"; subgraph cluster19182 { label="Scan csv "; 8137 [labelType="html" label="FileScanRDD [8137] count at CreateSubscriptionService.java:123"]; 8138 [labelType="html" label="MapPartitionsRDD [8138] count at CreateSubscriptionService.java:123"]; } subgraph cluster19177 { label="WholeStageCodegen (3)"; 8139 [labelType="html" label="MapPartitionsRDD [8139] count at CreateSubscriptionService.java:123"]; } subgraph cluster19188 { label="Scan csv "; 8140 [labelType="html" label="FileScanRDD [8140] count at CreateSubscriptionService.java:123"]; 8141 [labelType="html" label="MapPartitionsRDD [8141] count at CreateSubscriptionService.java:123"]; } subgraph cluster19183 { label="WholeStageCodegen (4)"; 8142 [labelType="html" label="MapPartitionsRDD [8142] count at CreateSubscriptionService.java:123"]; } subgraph cluster19176 { label="Union"; 8143 [labelType="html" label="UnionRDD [8143] count at CreateSubscriptionService.java:123"]; } subgraph cluster19173 { label="WholeStageCodegen (5)"; 8144 [labelType="html" label="MapPartitionsRDD [8144] count at CreateSubscriptionService.java:123"]; } subgraph cluster19172 { label="Exchange"; 8145 [labelType="html" label="MapPartitionsRDD [8145] count at CreateSubscriptionService.java:123"]; } } 8137->8138; 8138->8139; 8140->8141; 8141->8142; 8139->8143; 8142->8143; 8143->8144; 8144->8145; }

digraph G { subgraph clusterstage_2842 { label="Stage 2842 (skipped)"; subgraph cluster19200 { label="AQEShuffleRead"; 8150 [labelType="html" label="ShuffledRowRDD [8150] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19197 { label="WholeStageCodegen (7)"; 8151 [labelType="html" label="MapPartitionsRDD [8151] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19196 { label="Exchange"; 8152 [labelType="html" label="MapPartitionsRDD [8152] [Unordered] count at CreateSubscriptionService.java:123"]; } } 8150->8151; 8151->8152; }

8145,8150

digraph G { subgraph clusterstage_2843 { label="Stage 2843 (skipped)"; subgraph cluster19195 { label="Scan parquet "; 8146 [labelType="html" label="FileScanRDD [8146] count at CreateSubscriptionService.java:123"]; 8147 [labelType="html" label="MapPartitionsRDD [8147] count at CreateSubscriptionService.java:123"]; } subgraph cluster19190 { label="WholeStageCodegen (6)"; 8148 [labelType="html" label="MapPartitionsRDD [8148] count at CreateSubscriptionService.java:123"]; } subgraph cluster19189 { label="Exchange"; 8149 [labelType="html" label="MapPartitionsRDD [8149] count at CreateSubscriptionService.java:123"]; } } 8146->8147; 8147->8148; 8148->8149; }

digraph G { subgraph clusterstage_2844 { label="Stage 2844 (skipped)"; 8130 [labelType="html" label="JDBCRDD [8130] cache at CreateSubscriptionService.java:122"]; subgraph cluster19167 { label="Scan csv "; 8131 [labelType="html" label="FileScanRDD [8131] count at CreateSubscriptionService.java:123"]; 8132 [labelType="html" label="MapPartitionsRDD [8132] count at CreateSubscriptionService.java:123"]; } subgraph cluster19162 { label="WholeStageCodegen (1)"; 8133 [labelType="html" label="MapPartitionsRDD [8133] count at CreateSubscriptionService.java:123"]; } subgraph cluster19168 { label="WholeStageCodegen (2)"; 8134 [labelType="html" label="MapPartitionsRDD [8134] count at CreateSubscriptionService.java:123"]; } subgraph cluster19161 { label="Union"; 8135 [labelType="html" label="UnionRDD [8135] count at CreateSubscriptionService.java:123"]; } subgraph cluster19160 { label="Exchange"; 8136 [labelType="html" label="MapPartitionsRDD [8136] count at CreateSubscriptionService.java:123"]; } } 8131->8132; 8132->8133; 8130->8134; 8133->8135; 8134->8135; 8135->8136; }

digraph G { subgraph clusterstage_2845 { label="Stage 2845 (skipped)"; subgraph cluster19224 { label="AQEShuffleRead"; 8153 [labelType="html" label="ShuffledRowRDD [8153] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19221 { label="WholeStageCodegen (8)"; 8154 [labelType="html" label="MapPartitionsRDD [8154] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19229 { label="AQEShuffleRead"; 8155 [labelType="html" label="ShuffledRowRDD [8155] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19226 { label="WholeStageCodegen (9)"; 8156 [labelType="html" label="MapPartitionsRDD [8156] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19217 { label="WholeStageCodegen (10)"; 8157 [labelType="html" label="ZippedPartitionsRDD2 [8157] [Unordered] count at CreateSubscriptionService.java:123"]; 8158 [labelType="html" label="MapPartitionsRDD [8158] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19234 { label="AQEShuffleRead"; 8159 [labelType="html" label="ShuffledRowRDD [8159] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19231 { label="WholeStageCodegen (11)"; 8160 [labelType="html" label="MapPartitionsRDD [8160] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19211 { label="WholeStageCodegen (12)"; 8161 [labelType="html" label="ZippedPartitionsRDD2 [8161] [Unordered] count at CreateSubscriptionService.java:123"]; 8162 [labelType="html" label="MapPartitionsRDD [8162] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19210 { label="Window"; 8163 [labelType="html" label="MapPartitionsRDD [8163] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19206 { label="WholeStageCodegen (13)"; 8164 [labelType="html" label="MapPartitionsRDD [8164] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19205 { label="Window"; 8165 [labelType="html" label="MapPartitionsRDD [8165] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19202 { label="WholeStageCodegen (14)"; 8166 [labelType="html" label="MapPartitionsRDD [8166] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19201 { label="Exchange"; 8167 [labelType="html" label="MapPartitionsRDD [8167] [Unordered] count at CreateSubscriptionService.java:123"]; } } 8153->8154; 8155->8156; 8154->8157; 8156->8157; 8157->8158; 8159->8160; 8158->8161; 8160->8161; 8161->8162; 8162->8163; 8163->8164; 8164->8165; 8165->8166; 8166->8167; }

8136,8153

8152,8155

8149,8159

digraph G { subgraph clusterstage_2846 { label="Stage 2846"; subgraph cluster19243 { label="Exchange"; 8168 [labelType="html" label="ShuffledRowRDD [8168] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19239 { label="WholeStageCodegen (15)"; 8169 [labelType="html" label="MapPartitionsRDD [8169] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19238 { label="Window"; 8170 [labelType="html" label="MapPartitionsRDD [8170] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19235 { label="WholeStageCodegen (16)"; 8171 [labelType="html" label="MapPartitionsRDD [8171] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19244 { label="mapPartitionsInternal"; 8172 [labelType="html" label="MapPartitionsRDD [8172] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19245 { label="mapPartitionsInternal"; 8173 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [COL_04AF725C_66A0_4299_A535_7D820A79F925#317835 AS 04AF725C_66A0_4299_A535_7D820A79F925#320998, COL_81492BFD_C38C_4C09_8A5D_1EE53F619CB4#317836 AS 81492BFD_C38C_4C09_8A5D_1EE53F619CB4#320999, COL_8CAF0B1F_914D_45B3_919A_E87F9882A9B9#317837 AS 8CAF0B1F_914D_45B3_919A_E87F9882A9B9#321000, COL_5DBB07B2_DD4E_4DF8_A772_64564BC20B53#317839 AS 5DBB07B2_DD4E_4DF8_A772_64564BC20B53#321001, COL_3CD2052B_33C5_4457_9C30_D79AE298A405#317840 AS 3CD2052B_33C5_4457_9C30_D79AE298A405#321002, COL_49DA49E0_7469_4C3F_A05A_3FA643E894EA#317841 AS 49DA49E0_7469_4C3F_A05A_3FA643E894EA#321003, COL_D10143B3_53A3_4856_987D_3E4376BC7D0F#317842 AS D10143B3_53A3_4856_987D_3E4376BC7D0F#321004, COL_4FBE155F_08B2_4439_869C_ED16CA25E269#317843 AS 4FBE155F_08B2_4439_869C_ED16CA25E269#321005, COL_FE7AEAA4_23AE_49C1_82AC_4A9ADE3181CE#317844 AS FE7AEAA4_23AE_49C1_82AC_4A9ADE3181CE#321006, COL_8953E1CC_CB11_4CCC_87D4_F7518DBE39D9#317845 AS 8953E1CC_CB11_4CCC_87D4_F7518DBE39D9#321007, COL_C17C60DB_... [8173] [Cached] [Unordered] count at CreateSubscriptionService.java:123"]; } subgraph cluster19313 { label="InMemoryTableScan"; 8182 [labelType="html" label="MapPartitionsRDD [8182] [Unordered] parquet at CreateSubscriptionService.java:124"]; 8183 [labelType="html" label="MapPartitionsRDD [8183] [Unordered] parquet at CreateSubscriptionService.java:124"]; 8184 [labelType="html" label="MapPartitionsRDD [8184] [Unordered] parquet at CreateSubscriptionService.java:124"]; } subgraph cluster19311 { label="WriteFiles"; 8185 [labelType="html" label="MapPartitionsRDD [8185] [Unordered] parquet at CreateSubscriptionService.java:124"]; } } 8168->8169; 8169->8170; 8170->8171; 8171->8172; 8172->8173; 8173->8182; 8182->8183; 8183->8184; 8184->8185; }

8167,8168

8173

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
2846	GenerateSubscriptionSnapshot parquet at CreateSubscriptionService.java:124 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.executeTyped(CreateSubscriptionService.java:124) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.execute(CreateSubscriptionService.java:80) com.adb.hdh.spark.job.server.JobDispatcher.dispatch(JobDispatcher.java:78) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.runJob(JobSubmitHandler.java:76) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.lambda$handle$0(JobSubmitHandler.java:59) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	2026/08/31 16:00:23	6 s	10/10	89.9 MiB	24.7 MiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

2846

GenerateSubscriptionSnapshot parquet at CreateSubscriptionService.java:124 +details

org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)
com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.executeTyped(CreateSubscriptionService.java:124)
com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.execute(CreateSubscriptionService.java:80)
com.adb.hdh.spark.job.server.JobDispatcher.dispatch(JobDispatcher.java:78)
com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.runJob(JobSubmitHandler.java:76)
com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.lambda$handle$0(JobSubmitHandler.java:59)
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
java.base/java.util.concurrent.FutureTask.run(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.base/java.lang.Thread.run(Unknown Source)

2026/08/31 16:00:23

6 s

10/10

89.9 MiB

24.7 MiB

Page:

Skipped Stages (5)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
2845	count at CreateSubscriptionService.java:123 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.executeTyped(CreateSubscriptionService.java:123) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.execute(CreateSubscriptionService.java:80) com.adb.hdh.spark.job.server.JobDispatcher.dispatch(JobDispatcher.java:78) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.runJob(JobSubmitHandler.java:76) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.lambda$handle$0(JobSubmitHandler.java:59) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/3
2844	count at CreateSubscriptionService.java:123 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.executeTyped(CreateSubscriptionService.java:123) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.execute(CreateSubscriptionService.java:80) com.adb.hdh.spark.job.server.JobDispatcher.dispatch(JobDispatcher.java:78) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.runJob(JobSubmitHandler.java:76) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.lambda$handle$0(JobSubmitHandler.java:59) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/11
2843	count at CreateSubscriptionService.java:123 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.executeTyped(CreateSubscriptionService.java:123) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.execute(CreateSubscriptionService.java:80) com.adb.hdh.spark.job.server.JobDispatcher.dispatch(JobDispatcher.java:78) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.runJob(JobSubmitHandler.java:76) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.lambda$handle$0(JobSubmitHandler.java:59) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/3
2842	count at CreateSubscriptionService.java:123 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.executeTyped(CreateSubscriptionService.java:123) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.execute(CreateSubscriptionService.java:80) com.adb.hdh.spark.job.server.JobDispatcher.dispatch(JobDispatcher.java:78) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.runJob(JobSubmitHandler.java:76) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.lambda$handle$0(JobSubmitHandler.java:59) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/3
2841	count at CreateSubscriptionService.java:123 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.executeTyped(CreateSubscriptionService.java:123) com.adb.hdh.spark.job.service.dataframe.CreateSubscriptionService.execute(CreateSubscriptionService.java:80) com.adb.hdh.spark.job.server.JobDispatcher.dispatch(JobDispatcher.java:78) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.runJob(JobSubmitHandler.java:76) com.adb.hdh.spark.job.server.servlet.JobSubmitHandler.lambda$handle$0(JobSubmitHandler.java:59) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/4

Page: