SP1 - Details for Query 142

Submitted Time: 2026/06/25 12:32:32
Duration: 12 s
Succeeded Jobs: 184 185

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand task commit time: 1 ms number of written files: 1 job commit time: 13 ms number of output rows: 1,904,713 number of dynamic part: 0 written output: 84.0 MiB"]; 2 [labelType="html" label=" WriteFiles "]; 3 [labelType="html" label="Window spill size: 0.0 B"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 5.4 s"; 5 [labelType="html" label=" Project "]; 6 [labelType="html" label="Filter number of output rows: 1,904,713"]; } 7 [labelType="html" label="Window spill size: 0.0 B"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 2.3 s"; 9 [labelType="html" label="Sort sort time: 171 ms peak memory: 496.0 MiB spill size: 0.0 B"]; } 10 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 87.8 MiB number of coalesced partitions: 1"]; 11 [labelType="html" label="Exchange shuffle records written: 1,904,714 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 246 ms (20 ms, 25 ms, 26 ms (stage 287.0: task 270)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 1,904,714 local bytes read: 82.9 MiB fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 10 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 412.5 MiB (37.3 MiB, 41.6 MiB, 42.1 MiB (stage 287.0: task 277)) local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 82.9 MiB (6.6 MiB, 8.6 MiB, 8.8 MiB (stage 287.0: task 272))"]; subgraph cluster12 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n10.0 s (863 ms, 1.0 s, 1.1 s (stage 287.0: task 270))"; 13 [labelType="html" label=" Project "]; } 14 [labelType="html" label="Scan csv number of output rows: 1,904,714 number of files read: 1 metadata time: 0 ms size of files read: 347.0 MiB"]; 1->0; 2->1; 3->2; 5->3; 6->5; 7->6; 9->7; 10->9; 11->10; 13->11; 14->13; }

AdaptiveSparkPlan isFinalPlan=true

Execute InsertIntoHadoopFsRelationCommand file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411692, false, Parquet, [path=file:///data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411692], Overwrite, [COL_30863531_1D7E_428B_B583_E41A16C72ABB, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09]

WriteFiles

Window [last(COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016, true) windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#35038], [1], [1 DESC NULLS LAST]

Project [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_join(array(str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_complete(bigint_to_str(COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940), 0, 5, LEFT), ), str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, 40, 9), )), -), null) AS COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016]

Filter (COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940 > 1)

WholeStageCodegen (3)

Window [row_number() windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940], [1], [1 DESC NULLS LAST]

Sort [1 ASC NULLS FIRST, 1 DESC NULLS LAST], false, 0

WholeStageCodegen (2)

AQEShuffleRead coalesced

Exchange hashpartitioning(1, 10), ENSURE_REQUIREMENTS, [plan_id=8499]

Project [_c0#34913 AS COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, str_extract(_c0#34913, LEFT, 1) AS COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]

WholeStageCodegen (1)

FileScan csv [_c0#34913] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/execution/RSF_SOURCE_LOCA_411658/current/RS..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string>

Details

== Physical Plan ==
AdaptiveSparkPlan (22)
+- == Final Plan ==
   Execute InsertIntoHadoopFsRelationCommand (12)
   +- WriteFiles (11)
      +- Window (10)
         +- * Project (9)
            +- * Filter (8)
               +- Window (7)
                  +- * Sort (6)
                     +- AQEShuffleRead (5)
                        +- ShuffleQueryStage (4), Statistics(sizeInBytes=412.5 MiB, rowCount=1.90E+6)
                           +- Exchange (3)
                              +- * Project (2)
                                 +- Scan csv  (1)
+- == Initial Plan ==
   Execute InsertIntoHadoopFsRelationCommand (21)
   +- WriteFiles (20)
      +- Window (19)
         +- Project (18)
            +- Filter (17)
               +- Window (16)
                  +- Sort (15)
                     +- Exchange (14)
                        +- Project (13)
                           +- Scan csv  (1)


(1) Scan csv 
Output [1]: [_c0#34913]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/execution/RSF_SOURCE_LOCA_411658/current/RSF-MCO-2025-M12-Final.txt]
ReadSchema: struct<_c0:string>

(2) Project [codegen id : 1]
Output [2]: [_c0#34913 AS COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, str_extract(_c0#34913, LEFT, 1) AS COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Input [1]: [_c0#34913]

(3) Exchange
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Arguments: hashpartitioning(1, 10), ENSURE_REQUIREMENTS, [plan_id=8499]

(4) ShuffleQueryStage
Output [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Arguments: 0

(5) AQEShuffleRead
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Arguments: coalesced

(6) Sort [codegen id : 2]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Arguments: [1 ASC NULLS FIRST, 1 DESC NULLS LAST], false, 0

(7) Window
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Arguments: [row_number() windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940], [1], [1 DESC NULLS LAST]

(8) Filter [codegen id : 3]
Input [3]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940]
Condition : (COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940 > 1)

(9) Project [codegen id : 3]
Output [4]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_join(array(str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_complete(bigint_to_str(COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940), 0, 5, LEFT), ), str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, 40, 9), )), -), null) AS COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016]
Input [3]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940]

(10) Window
Input [4]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016]
Arguments: [last(COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016, true) windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#35038], [1], [1 DESC NULLS LAST]

(11) WriteFiles
Input [5]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#35038]

(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411692, false, Parquet, [path=file:///data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411692], Overwrite, [COL_30863531_1D7E_428B_B583_E41A16C72ABB, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09]

(13) Project
Output [2]: [_c0#34913 AS COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, str_extract(_c0#34913, LEFT, 1) AS COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Input [1]: [_c0#34913]

(14) Exchange
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Arguments: hashpartitioning(1, 10), ENSURE_REQUIREMENTS, [plan_id=8484]

(15) Sort
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Arguments: [1 ASC NULLS FIRST, 1 DESC NULLS LAST], false, 0

(16) Window
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932]
Arguments: [row_number() windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940], [1], [1 DESC NULLS LAST]

(17) Filter
Input [3]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940]
Condition : (COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940 > 1)

(18) Project
Output [4]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_join(array(str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_complete(bigint_to_str(COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940), 0, 5, LEFT), ), str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932 <=> A), str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, 40, 9), )), -), null) AS COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016]
Input [3]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940]

(19) Window
Input [4]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016]
Arguments: [last(COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016, true) windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#35038], [1], [1 DESC NULLS LAST]

(20) WriteFiles
Input [5]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#34930, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#34940, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#34932, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#35016, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#35038]

(21) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411692, false, Parquet, [path=file:///data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411692], Overwrite, [COL_30863531_1D7E_428B_B583_E41A16C72ABB, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09]

(22) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet