hdh-spark-job - Details for Query 1498

Submitted Time: 2026/10/23 14:48:14
Duration: 0.4 s
Succeeded Jobs: 1499 1504

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 4 ms"; 2 [labelType="html" label=" Project "]; 3 [labelType="html" label=" Project "]; 4 [labelType="html" label="HashAggregate time in aggregation build: 0 ms number of output rows: 1"]; } 5 [labelType="html" label="Exchange shuffle records written: 4 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 3 ms (0 ms, 0 ms, 1 ms (stage 2185.0: task 1910)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 4 local bytes read: 234.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 4 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 96.0 B (24.0 B, 24.0 B, 24.0 B (stage 2185.0: task 1909)) local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 234.0 B (57.0 B, 57.0 B, 63.0 B (stage 2185.0: task 1910))"]; subgraph cluster6 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n32 ms (2 ms, 7 ms, 17 ms (stage 2185.0: task 1910))"; 7 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 31 ms (2 ms, 7 ms, 17 ms (stage 2185.0: task 1910)) number of output rows: 4"]; 8 [labelType="html" label=" Project "]; 9 [labelType="html" label="ColumnarToRow number of output rows: 4,446 number of input batches: 2"]; } 10 [labelType="html" label="Scan parquet number of files read: 1 scan time total (min, med, max (stageId: taskId)) 29 ms (2 ms, 7 ms, 15 ms (stage 2185.0: task 1910)) metadata time: 0 ms size of files read: 12.7 MiB number of output rows: 4,446"]; 2->0; 3->2; 4->3; 5->4; 7->5; 8->7; 9->8; 10->9; }

AdaptiveSparkPlan isFinalPlan=true

Project [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L, str_concat(array(dec_to_str(dec_round(dec_mult(array(dec_div(array(COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L)), 100.0)), 2)), %)) AS COL_61BB4A02_C239_41D4_B4B7_7981EDCC7541#788888]

Project [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, bigint_sum(array(COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L)) AS COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L]

HashAggregate(keys=[], functions=[sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418), sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)])

WholeStageCodegen (2)

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=41069]

HashAggregate(keys=[], functions=[partial_sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418), partial_sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)])

Project [COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D#788196 AS COL_1263040A_A749_48DF_AD51_BA31B527706A#788418, COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2#788195 AS COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419]

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2#788195,COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D#788196] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/DATABLOCK_INFOR_326796], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2:int,COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D:int>

Details

== Physical Plan ==
AdaptiveSparkPlan (16)
+- == Final Plan ==
   * Project (9)
   +- * Project (8)
      +- * HashAggregate (7)
         +- ShuffleQueryStage (6), Statistics(sizeInBytes=96.0 B, rowCount=4)
            +- Exchange (5)
               +- * HashAggregate (4)
                  +- * Project (3)
                     +- * ColumnarToRow (2)
                        +- Scan parquet  (1)
+- == Initial Plan ==
   Project (15)
   +- Project (14)
      +- HashAggregate (13)
         +- Exchange (12)
            +- HashAggregate (11)
               +- Project (10)
                  +- Scan parquet  (1)


(1) Scan parquet 
Output [2]: [COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2#788195, COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D#788196]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/DATABLOCK_INFOR_326796]
ReadSchema: struct<COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2:int,COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D:int>

(2) ColumnarToRow [codegen id : 1]
Input [2]: [COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2#788195, COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D#788196]

(3) Project [codegen id : 1]
Output [2]: [COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D#788196 AS COL_1263040A_A749_48DF_AD51_BA31B527706A#788418, COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2#788195 AS COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419]
Input [2]: [COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2#788195, COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D#788196]

(4) HashAggregate [codegen id : 1]
Input [2]: [COL_1263040A_A749_48DF_AD51_BA31B527706A#788418, COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419]
Keys: []
Functions [2]: [partial_sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418), partial_sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)]
Aggregate Attributes [2]: [sum#789099L, sum#789100L]
Results [2]: [sum#789101L, sum#789102L]

(5) Exchange
Input [2]: [sum#789101L, sum#789102L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=41069]

(6) ShuffleQueryStage
Output [2]: [sum#789101L, sum#789102L]
Arguments: 0

(7) HashAggregate [codegen id : 2]
Input [2]: [sum#789101L, sum#789102L]
Keys: []
Functions [2]: [sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418), sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)]
Aggregate Attributes [2]: [sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418)#788456L, sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)#788458L]
Results [2]: [sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418)#788456L AS COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)#788458L AS COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L]

(8) Project [codegen id : 2]
Output [3]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, bigint_sum(array(COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L)) AS COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L]
Input [2]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L]

(9) Project [codegen id : 2]
Output [4]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L, str_concat(array(dec_to_str(dec_round(dec_mult(array(dec_div(array(COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L)), 100.0)), 2)),  %)) AS COL_61BB4A02_C239_41D4_B4B7_7981EDCC7541#788888]
Input [3]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L]

(10) Project
Output [2]: [COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D#788196 AS COL_1263040A_A749_48DF_AD51_BA31B527706A#788418, COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2#788195 AS COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419]
Input [2]: [COL_3F93A8A9_7638_4426_BC96_D5D5D9455BE2#788195, COL_CDBA4FAB_0E59_4EF2_B84E_D15485DF766D#788196]

(11) HashAggregate
Input [2]: [COL_1263040A_A749_48DF_AD51_BA31B527706A#788418, COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419]
Keys: []
Functions [2]: [partial_sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418), partial_sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)]
Aggregate Attributes [2]: [sum#789099L, sum#789100L]
Results [2]: [sum#789101L, sum#789102L]

(12) Exchange
Input [2]: [sum#789101L, sum#789102L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=41050]

(13) HashAggregate
Input [2]: [sum#789101L, sum#789102L]
Keys: []
Functions [2]: [sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418), sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)]
Aggregate Attributes [2]: [sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418)#788456L, sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)#788458L]
Results [2]: [sum(COL_1263040A_A749_48DF_AD51_BA31B527706A#788418)#788456L AS COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, sum(COL_F9F08576_CCE5_45C2_9043_2445776D64A5#788419)#788458L AS COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L]

(14) Project
Output [3]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, bigint_sum(array(COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L)) AS COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L]
Input [2]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L]

(15) Project
Output [4]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L, str_concat(array(dec_to_str(dec_round(dec_mult(array(dec_div(array(COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L)), 100.0)), 2)),  %)) AS COL_61BB4A02_C239_41D4_B4B7_7981EDCC7541#788888]
Input [3]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L]

(16) AdaptiveSparkPlan
Output [4]: [COL_AA75A0DD_4A42_4D47_A543_47C26773E444#788457L, COL_98B52DCA_A81E_4842_8A7C_DC69B34EC88C#788459L, COL_9FDF501F_D7C0_440A_AF3D_9835B2478C83#788504L, COL_61BB4A02_C239_41D4_B4B7_7981EDCC7541#788888]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet