SP1 - Details for Query 43

Submitted Time: 2026/08/26 14:00:42
Duration: 11 s
Succeeded Jobs: 48 49

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand task commit time: 1 ms number of written files: 1 job commit time: 13 ms number of output rows: 4 number of dynamic part: 0 written output: 1031.0 B"]; 2 [labelType="html" label=" WriteFiles "]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 36 ms"; 4 [labelType="html" label=" Project "]; 5 [labelType="html" label="Generate number of output rows: 4"]; 6 [labelType="html" label="HashAggregate time in aggregation build: 1 ms number of output rows: 1"]; } 7 [labelType="html" label="Exchange shuffle records written: 10 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 5 ms (0 ms, 0 ms, 0 ms (stage 88.0: task 147)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 10 local bytes read: 605.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 10 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 400.0 B (40.0 B, 40.0 B, 40.0 B (stage 88.0: task 147)) local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 605.0 B (58.0 B, 58.0 B, 83.0 B (stage 88.0: task 144))"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n10.8 s (5 ms, 13 ms, 10.7 s (stage 88.0: task 144))"; 9 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 10.8 s (5 ms, 12 ms, 10.7 s (stage 88.0: task 144)) number of output rows: 10"]; 10 [labelType="html" label=" Project "]; 11 [labelType="html" label="Filter number of output rows: 334,650"]; 12 [labelType="html" label="ColumnarToRow number of output rows: 2,401,930 number of input batches: 587"]; } 13 [labelType="html" label="Scan parquet number of files read: 1 scan time total (min, med, max (stageId: taskId)) 712 ms (5 ms, 12 ms, 619 ms (stage 88.0: task 144)) metadata time: 0 ms size of files read: 91.4 MiB number of output rows: 2,401,930"]; 1->0; 2->1; 4->2; 5->4; 6->5; 7->6; 9->7; 10->9; 11->10; 12->11; 13->12; }

AdaptiveSparkPlan isFinalPlan=true

Execute InsertIntoHadoopFsRelationCommand file:/data/output/cache/parquet/datablock/de327d0f_8baf_45d3_8fe2_34303adf69b2, false, Parquet, [path=file:///data/output/cache/parquet/datablock/de327d0f_8baf_45d3_8fe2_34303adf69b2], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

WriteFiles

Project [col0#17526 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#17573, col1#17527 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#17574]

Generate stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#17229, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#17230, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#17231, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#17232), false, [col0#17526, col1#17527]

HashAggregate(keys=[], functions=[min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)])

WholeStageCodegen (2)

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2772]

HashAggregate(keys=[], functions=[partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)])

Project [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938]

Filter (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321 <=> A)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321] Batched: true, DataFilters: [(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321 <=> A)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_425007], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9,A)], ReadSchema: struct<COL_30863531_1D7E_428B_B583_E41A16C72ABB:string,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9:s...

Details

== Physical Plan ==
AdaptiveSparkPlan (22)
+- == Final Plan ==
   Execute InsertIntoHadoopFsRelationCommand (12)
   +- WriteFiles (11)
      +- * Project (10)
         +- * Generate (9)
            +- * HashAggregate (8)
               +- ShuffleQueryStage (7), Statistics(sizeInBytes=400.0 B, rowCount=10)
                  +- Exchange (6)
                     +- * HashAggregate (5)
                        +- * Project (4)
                           +- * Filter (3)
                              +- * ColumnarToRow (2)
                                 +- Scan parquet  (1)
+- == Initial Plan ==
   Execute InsertIntoHadoopFsRelationCommand (21)
   +- WriteFiles (20)
      +- Project (19)
         +- Generate (18)
            +- HashAggregate (17)
               +- Exchange (16)
                  +- HashAggregate (15)
                     +- Project (14)
                        +- Filter (13)
                           +- Scan parquet  (1)


(1) Scan parquet 
Output [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_425007]
PushedFilters: [EqualNullSafe(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9,A)]
ReadSchema: struct<COL_30863531_1D7E_428B_B583_E41A16C72ABB:string,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9:string>

(2) ColumnarToRow [codegen id : 1]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321]

(3) Filter [codegen id : 1]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321]
Condition : (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321 <=> A)

(4) Project [codegen id : 1]
Output [2]: [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321]

(5) HashAggregate [codegen id : 1]
Input [2]: [COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937, COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938]
Keys: []
Functions [4]: [partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)]
Aggregate Attributes [4]: [min#17683, max#17684, min#17685, max#17686]
Results [4]: [min#17687, max#17688, min#17689, max#17690]

(6) Exchange
Input [4]: [min#17687, max#17688, min#17689, max#17690]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2772]

(7) ShuffleQueryStage
Output [4]: [min#17687, max#17688, min#17689, max#17690]
Arguments: 0

(8) HashAggregate [codegen id : 2]
Input [4]: [min#17687, max#17688, min#17689, max#17690]
Keys: []
Functions [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)]
Aggregate Attributes [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937)#17022, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937)#17024, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)#17026, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)#17028]
Results [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937)#17022 AS COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#17229, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937)#17024 AS COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#17230, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)#17026 AS COL_DBB8940D_7041_41A8_9448_C56C73EB8044#17231, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)#17028 AS COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#17232]

(9) Generate [codegen id : 2]
Input [4]: [COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#17229, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#17230, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#17231, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#17232]
Arguments: stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#17229, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#17230, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#17231, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#17232), false, [col0#17526, col1#17527]

(10) Project [codegen id : 2]
Output [2]: [col0#17526 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#17573, col1#17527 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#17574]
Input [2]: [col0#17526, col1#17527]

(11) WriteFiles
Input [2]: [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#17573, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#17574]

(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/de327d0f_8baf_45d3_8fe2_34303adf69b2, false, Parquet, [path=file:///data/output/cache/parquet/datablock/de327d0f_8baf_45d3_8fe2_34303adf69b2], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

(13) Filter
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321]
Condition : (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321 <=> A)

(14) Project
Output [2]: [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#11319, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#11321]

(15) HashAggregate
Input [2]: [COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937, COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938]
Keys: []
Functions [4]: [partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)]
Aggregate Attributes [4]: [min#17683, max#17684, min#17685, max#17686]
Results [4]: [min#17687, max#17688, min#17689, max#17690]

(16) Exchange
Input [4]: [min#17687, max#17688, min#17689, max#17690]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2747]

(17) HashAggregate
Input [4]: [min#17687, max#17688, min#17689, max#17690]
Keys: []
Functions [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)]
Aggregate Attributes [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937)#17022, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937)#17024, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)#17026, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)#17028]
Results [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937)#17022 AS COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#17229, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#16937)#17024 AS COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#17230, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)#17026 AS COL_DBB8940D_7041_41A8_9448_C56C73EB8044#17231, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#16938)#17028 AS COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#17232]

(18) Generate
Input [4]: [COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#17229, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#17230, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#17231, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#17232]
Arguments: stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#17229, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#17230, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#17231, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#17232), false, [col0#17526, col1#17527]

(19) Project
Output [2]: [col0#17526 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#17573, col1#17527 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#17574]
Input [2]: [col0#17526, col1#17527]

(20) WriteFiles
Input [2]: [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#17573, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#17574]

(21) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/de327d0f_8baf_45d3_8fe2_34303adf69b2, false, Parquet, [path=file:///data/output/cache/parquet/datablock/de327d0f_8baf_45d3_8fe2_34303adf69b2], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

(22) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet