SP1 - Details for Query 201

Submitted Time: 2026/02/20 08:08:04
Duration: 0.4 s
Succeeded Jobs: 294 299

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label=" TakeOrderedAndProject "]; subgraph cluster2 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 14 ms"; 3 [labelType="html" label="HashAggregate spill size: 0.0 B time in aggregation build: 1 ms peak memory: 16.3 MiB number of output rows: 6 number of sort fallback tasks: 0 avg hash probes per key: 1"]; } 4 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 661.0 B number of coalesced partitions: 1"]; 5 [labelType="html" label="Exchange shuffle records written: 6 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 6 ms (1 ms, 2 ms, 2 ms (stage 423.0: task 550)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 6 local bytes read: 635.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 3 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 408.0 B (120.0 B, 136.0 B, 152.0 B (stage 423.0: task 550)) local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 635.0 B (197.0 B, 214.0 B, 224.0 B (stage 423.0: task 550))"]; subgraph cluster6 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n285 ms (88 ms, 98 ms, 99 ms (stage 423.0: task 551))"; 7 [labelType="html" label="HashAggregate spill size: 0.0 B time in aggregation build total (min, med, max (stageId: taskId)) 22 ms (6 ms, 8 ms, 8 ms (stage 423.0: task 550)) peak memory total (min, med, max (stageId: taskId)) 768.0 KiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 423.0: task 550)) number of output rows: 6 number of sort fallback tasks: 0 avg hash probes per key: 0"]; 8 [labelType="html" label=" Project "]; 9 [labelType="html" label="ColumnarToRow number of output rows: 72 number of input batches: 3"]; } 10 [labelType="html" label="Scan parquet number of files read: 3 scan time total (min, med, max (stageId: taskId)) 16 ms (5 ms, 5 ms, 6 ms (stage 423.0: task 550)) metadata time: 0 ms size of files read: 4.5 MiB number of output rows: 72"]; 1->0; 3->1; 4->3; 5->4; 7->5; 8->7; 9->8; 10->9; }

AdaptiveSparkPlan isFinalPlan=true

TakeOrderedAndProject(limit=500000, orderBy=[COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022 ASC NULLS FIRST,COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023 ASC NULLS FIRST], output=[COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022,COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023,COL_2EFA973F_431A_4F09_BCF1_53BEBA35E885#62034])

HashAggregate(keys=[COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983], functions=[last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990, false)])

WholeStageCodegen (2)

AQEShuffleRead coalesced

Exchange hashpartitioning(COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, 10), ENSURE_REQUIREMENTS, [plan_id=7091]

HashAggregate(keys=[COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983], functions=[partial_last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990, false)])

Project [COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0#61805 AS COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3#61806 AS COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4#61812 AS COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990]

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0#61805,COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3#61806,COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4#61812] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/REPARTITION_PRO_14440], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0:string,COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3:s...

Details

== Physical Plan ==
AdaptiveSparkPlan (15)
+- == Final Plan ==
   TakeOrderedAndProject (9)
   +- * HashAggregate (8)
      +- AQEShuffleRead (7)
         +- ShuffleQueryStage (6), Statistics(sizeInBytes=408.0 B, rowCount=6)
            +- Exchange (5)
               +- * HashAggregate (4)
                  +- * Project (3)
                     +- * ColumnarToRow (2)
                        +- Scan parquet  (1)
+- == Initial Plan ==
   TakeOrderedAndProject (14)
   +- HashAggregate (13)
      +- Exchange (12)
         +- HashAggregate (11)
            +- Project (10)
               +- Scan parquet  (1)


(1) Scan parquet 
Output [3]: [COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0#61805, COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3#61806, COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4#61812]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/REPARTITION_PRO_14440]
ReadSchema: struct<COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0:string,COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3:string,COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4:double>

(2) ColumnarToRow [codegen id : 1]
Input [3]: [COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0#61805, COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3#61806, COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4#61812]

(3) Project [codegen id : 1]
Output [3]: [COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0#61805 AS COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3#61806 AS COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4#61812 AS COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990]
Input [3]: [COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0#61805, COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3#61806, COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4#61812]

(4) HashAggregate [codegen id : 1]
Input [3]: [COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990]
Keys [2]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983]
Functions [1]: [partial_last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990, false)]
Aggregate Attributes [2]: [last#62239, valueSet#62240]
Results [4]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, last#62241, valueSet#62242]

(5) Exchange
Input [4]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, last#62241, valueSet#62242]
Arguments: hashpartitioning(COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, 10), ENSURE_REQUIREMENTS, [plan_id=7091]

(6) ShuffleQueryStage
Output [4]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, last#62241, valueSet#62242]
Arguments: 0

(7) AQEShuffleRead
Input [4]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, last#62241, valueSet#62242]
Arguments: coalesced

(8) HashAggregate [codegen id : 2]
Input [4]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, last#62241, valueSet#62242]
Keys [2]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983]
Functions [1]: [last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990, false)]
Aggregate Attributes [1]: [last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990)()#62033]
Results [3]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984 AS COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983 AS COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023, last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990)()#62033 AS COL_2EFA973F_431A_4F09_BCF1_53BEBA35E885#62034]

(9) TakeOrderedAndProject
Input [3]: [COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022, COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023, COL_2EFA973F_431A_4F09_BCF1_53BEBA35E885#62034]
Arguments: 500000, [COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022 ASC NULLS FIRST, COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023 ASC NULLS FIRST], [COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022, COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023, COL_2EFA973F_431A_4F09_BCF1_53BEBA35E885#62034]

(10) Project
Output [3]: [COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0#61805 AS COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3#61806 AS COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4#61812 AS COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990]
Input [3]: [COL_0EFAFD75_A8DE_42EF_8A2B_87C19F240FF0#61805, COL_BD34C176_D5D8_497C_AC6E_1ECE0068FFA3#61806, COL_208CED73_0168_41C9_A2D0_B5ED3BB68CC4#61812]

(11) HashAggregate
Input [3]: [COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990]
Keys [2]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983]
Functions [1]: [partial_last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990, false)]
Aggregate Attributes [2]: [last#62239, valueSet#62240]
Results [4]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, last#62241, valueSet#62242]

(12) Exchange
Input [4]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, last#62241, valueSet#62242]
Arguments: hashpartitioning(COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, 10), ENSURE_REQUIREMENTS, [plan_id=7072]

(13) HashAggregate
Input [4]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983, last#62241, valueSet#62242]
Keys [2]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983]
Functions [1]: [last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990, false)]
Aggregate Attributes [1]: [last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990)()#62033]
Results [3]: [COL_4111D15F_0155_4952_A5EA_D3F00CBFBFF4#61984 AS COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022, COL_9E90D5BD_D839_4D90_8B55_455534803284#61983 AS COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023, last(COL_46DE2263_5485_4C86_AC85_9E0799BDE25E#61990)()#62033 AS COL_2EFA973F_431A_4F09_BCF1_53BEBA35E885#62034]

(14) TakeOrderedAndProject
Input [3]: [COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022, COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023, COL_2EFA973F_431A_4F09_BCF1_53BEBA35E885#62034]
Arguments: 500000, [COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022 ASC NULLS FIRST, COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023 ASC NULLS FIRST], [COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022, COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023, COL_2EFA973F_431A_4F09_BCF1_53BEBA35E885#62034]

(15) AdaptiveSparkPlan
Output [3]: [COL_1F905013_30B0_41D1_BE4A_3FB85D278D0E#62022, COL_E93C1ADA_3903_4D41_A3F4_4D9E2C0056A3#62023, COL_2EFA973F_431A_4F09_BCF1_53BEBA35E885#62034]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet