SP1 - Details for Query 108

Submitted Time: 2026/06/25 10:31:02
Duration: 4 s
Succeeded Jobs: 129 130

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand task commit time: 1 ms number of written files: 1 job commit time: 14 ms number of output rows: 509,703 number of dynamic part: 0 written output: 25.4 MiB"]; 2 [labelType="html" label=" WriteFiles "]; 3 [labelType="html" label="Window spill size: 0.0 B"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 1.7 s"; 5 [labelType="html" label=" Project "]; 6 [labelType="html" label="Filter number of output rows: 509,703"]; } 7 [labelType="html" label="Window spill size: 0.0 B"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 883 ms"; 9 [labelType="html" label="Sort sort time: 61 ms peak memory: 144.0 MiB spill size: 0.0 B"]; } 10 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 26.6 MiB number of coalesced partitions: 1"]; 11 [labelType="html" label="Exchange shuffle records written: 509,704 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 89 ms (5 ms, 9 ms, 11 ms (stage 184.0: task 141)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 509,704 local bytes read: 25.9 MiB fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 10 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 111.3 MiB (6.8 MiB, 11.6 MiB, 11.6 MiB (stage 184.0: task 140)) local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 25.9 MiB (1659.3 KiB, 2.7 MiB, 2.7 MiB (stage 184.0: task 146))"]; subgraph cluster12 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n3.7 s (252 ms, 377 ms, 404 ms (stage 184.0: task 148))"; 13 [labelType="html" label=" Project "]; } 14 [labelType="html" label="Scan csv number of output rows: 509,704 number of files read: 1 metadata time: 0 ms size of files read: 93.7 MiB"]; 1->0; 2->1; 3->2; 5->3; 6->5; 7->6; 9->7; 10->9; 11->10; 13->11; 14->13; }

AdaptiveSparkPlan isFinalPlan=true

Execute InsertIntoHadoopFsRelationCommand file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411360, false, Parquet, [path=file:///data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411360], Overwrite, [COL_30863531_1D7E_428B_B583_E41A16C72ABB, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09]

WriteFiles

Window [last(COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677, true) windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#19699], [1], [1 DESC NULLS LAST]

Project [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_join(array(str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_complete(bigint_to_str(COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601), 0, 5, LEFT), ), str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, 40, 9), )), -), null) AS COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677]

Filter (COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601 > 1)

WholeStageCodegen (3)

Window [row_number() windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601], [1], [1 DESC NULLS LAST]

Sort [1 ASC NULLS FIRST, 1 DESC NULLS LAST], false, 0

WholeStageCodegen (2)

AQEShuffleRead coalesced

Exchange hashpartitioning(1, 10), ENSURE_REQUIREMENTS, [plan_id=5410]

Project [_c0#19574 AS COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, str_extract(_c0#19574, LEFT, 1) AS COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]

WholeStageCodegen (1)

FileScan csv [_c0#19574] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/execution/RSF_SOURCE_LOCA_411326/current/rs..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string>

Details

== Physical Plan ==
AdaptiveSparkPlan (22)
+- == Final Plan ==
   Execute InsertIntoHadoopFsRelationCommand (12)
   +- WriteFiles (11)
      +- Window (10)
         +- * Project (9)
            +- * Filter (8)
               +- Window (7)
                  +- * Sort (6)
                     +- AQEShuffleRead (5)
                        +- ShuffleQueryStage (4), Statistics(sizeInBytes=111.3 MiB, rowCount=5.10E+5)
                           +- Exchange (3)
                              +- * Project (2)
                                 +- Scan csv  (1)
+- == Initial Plan ==
   Execute InsertIntoHadoopFsRelationCommand (21)
   +- WriteFiles (20)
      +- Window (19)
         +- Project (18)
            +- Filter (17)
               +- Window (16)
                  +- Sort (15)
                     +- Exchange (14)
                        +- Project (13)
                           +- Scan csv  (1)


(1) Scan csv 
Output [1]: [_c0#19574]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/execution/RSF_SOURCE_LOCA_411326/current/rsf_Bergerac_M12_2025.txt]
ReadSchema: struct<_c0:string>

(2) Project [codegen id : 1]
Output [2]: [_c0#19574 AS COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, str_extract(_c0#19574, LEFT, 1) AS COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Input [1]: [_c0#19574]

(3) Exchange
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Arguments: hashpartitioning(1, 10), ENSURE_REQUIREMENTS, [plan_id=5410]

(4) ShuffleQueryStage
Output [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Arguments: 0

(5) AQEShuffleRead
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Arguments: coalesced

(6) Sort [codegen id : 2]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Arguments: [1 ASC NULLS FIRST, 1 DESC NULLS LAST], false, 0

(7) Window
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Arguments: [row_number() windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601], [1], [1 DESC NULLS LAST]

(8) Filter [codegen id : 3]
Input [3]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601]
Condition : (COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601 > 1)

(9) Project [codegen id : 3]
Output [4]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_join(array(str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_complete(bigint_to_str(COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601), 0, 5, LEFT), ), str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, 40, 9), )), -), null) AS COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677]
Input [3]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601]

(10) Window
Input [4]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677]
Arguments: [last(COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677, true) windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#19699], [1], [1 DESC NULLS LAST]

(11) WriteFiles
Input [5]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#19699]

(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411360, false, Parquet, [path=file:///data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411360], Overwrite, [COL_30863531_1D7E_428B_B583_E41A16C72ABB, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09]

(13) Project
Output [2]: [_c0#19574 AS COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, str_extract(_c0#19574, LEFT, 1) AS COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Input [1]: [_c0#19574]

(14) Exchange
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Arguments: hashpartitioning(1, 10), ENSURE_REQUIREMENTS, [plan_id=5395]

(15) Sort
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Arguments: [1 ASC NULLS FIRST, 1 DESC NULLS LAST], false, 0

(16) Window
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593]
Arguments: [row_number() windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601], [1], [1 DESC NULLS LAST]

(17) Filter
Input [3]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601]
Condition : (COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601 > 1)

(18) Project
Output [4]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_join(array(str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_complete(bigint_to_str(COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601), 0, 5, LEFT), ), str_if_with_rule((COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593 <=> A), str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, 40, 9), )), -), null) AS COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677]
Input [3]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601]

(19) Window
Input [4]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677]
Arguments: [last(COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677, true) windowspecdefinition(1, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#19699], [1], [1 DESC NULLS LAST]

(20) WriteFiles
Input [5]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#19591, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F#19601, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#19593, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763#19677, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09#19699]

(21) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411360, false, Parquet, [path=file:///data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411360], Overwrite, [COL_30863531_1D7E_428B_B583_E41A16C72ABB, COL_68EAEFC1_2326_4E45_A372_A7E0B2A1C23F, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9, COL_EA2F05A2_A6F1_4291_910C_47CB1DD38763, COL_1C2F8BFA_DEC1_42A6_B904_489BF5DA6E09]

(22) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet