SP1 - Details for Query 163

Submitted Time: 2026/06/25 12:33:57
Duration: 13 s
Succeeded Jobs: 220 221

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand task commit time: 1 ms number of written files: 1 job commit time: 15 ms number of output rows: 4 number of dynamic part: 0 written output: 1031.0 B"]; 2 [labelType="html" label=" WriteFiles "]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 30 ms"; 4 [labelType="html" label=" Project "]; 5 [labelType="html" label="Generate number of output rows: 4"]; 6 [labelType="html" label="HashAggregate time in aggregation build: 0 ms number of output rows: 1"]; } 7 [labelType="html" label="Exchange shuffle records written: 10 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 25 ms (0 ms, 2 ms, 5 ms (stage 363.0: task 389)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 10 local bytes read: 605.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 10 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 400.0 B (40.0 B, 40.0 B, 40.0 B (stage 363.0: task 394)) local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 605.0 B (58.0 B, 58.0 B, 83.0 B (stage 363.0: task 392))"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n12.9 s (3 ms, 6 ms, 12.9 s (stage 363.0: task 392))"; 9 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 12.9 s (3 ms, 6 ms, 12.9 s (stage 363.0: task 392)) number of output rows: 10"]; 10 [labelType="html" label=" Project "]; 11 [labelType="html" label="Filter number of output rows: 394,501"]; 12 [labelType="html" label="ColumnarToRow number of output rows: 1,904,713 number of input batches: 466"]; } 13 [labelType="html" label="Scan parquet number of files read: 1 scan time total (min, med, max (stageId: taskId)) 659 ms (3 ms, 6 ms, 612 ms (stage 363.0: task 392)) metadata time: 0 ms size of files read: 84.0 MiB number of output rows: 1,904,713"]; 1->0; 2->1; 4->2; 5->4; 6->5; 7->6; 9->7; 10->9; 11->10; 12->11; 13->12; }

AdaptiveSparkPlan isFinalPlan=true

Execute InsertIntoHadoopFsRelationCommand file:/data/output/cache/parquet/datablock/ff89ba3c_9f5a_4602_8cc1_d69a51de8cd9, false, Parquet, [path=file:///data/output/cache/parquet/datablock/ff89ba3c_9f5a_4602_8cc1_d69a51de8cd9], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

WriteFiles

Project [col0#47258 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#47285, col1#47259 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#47286]

Generate stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#47231, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#47232, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#47233, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#47234), false, [col0#47258, col1#47259]

HashAggregate(keys=[], functions=[min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)])

WholeStageCodegen (2)

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10745]

HashAggregate(keys=[], functions=[partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)])

Project [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072]

Filter (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143 <=> A)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143] Batched: true, DataFilters: [(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143 <=> A)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411692], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9,A)], ReadSchema: struct<COL_30863531_1D7E_428B_B583_E41A16C72ABB:string,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9:s...

Details

== Physical Plan ==
AdaptiveSparkPlan (22)
+- == Final Plan ==
   Execute InsertIntoHadoopFsRelationCommand (12)
   +- WriteFiles (11)
      +- * Project (10)
         +- * Generate (9)
            +- * HashAggregate (8)
               +- ShuffleQueryStage (7), Statistics(sizeInBytes=400.0 B, rowCount=10)
                  +- Exchange (6)
                     +- * HashAggregate (5)
                        +- * Project (4)
                           +- * Filter (3)
                              +- * ColumnarToRow (2)
                                 +- Scan parquet  (1)
+- == Initial Plan ==
   Execute InsertIntoHadoopFsRelationCommand (21)
   +- WriteFiles (20)
      +- Project (19)
         +- Generate (18)
            +- HashAggregate (17)
               +- Exchange (16)
                  +- HashAggregate (15)
                     +- Project (14)
                        +- Filter (13)
                           +- Scan parquet  (1)


(1) Scan parquet 
Output [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411692]
PushedFilters: [EqualNullSafe(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9,A)]
ReadSchema: struct<COL_30863531_1D7E_428B_B583_E41A16C72ABB:string,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9:string>

(2) ColumnarToRow [codegen id : 1]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143]

(3) Filter [codegen id : 1]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143]
Condition : (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143 <=> A)

(4) Project [codegen id : 1]
Output [2]: [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143]

(5) HashAggregate [codegen id : 1]
Input [2]: [COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071, COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072]
Keys: []
Functions [4]: [partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)]
Aggregate Attributes [4]: [min#47356, max#47357, min#47358, max#47359]
Results [4]: [min#47360, max#47361, min#47362, max#47363]

(6) Exchange
Input [4]: [min#47360, max#47361, min#47362, max#47363]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10745]

(7) ShuffleQueryStage
Output [4]: [min#47360, max#47361, min#47362, max#47363]
Arguments: 0

(8) HashAggregate [codegen id : 2]
Input [4]: [min#47360, max#47361, min#47362, max#47363]
Keys: []
Functions [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)]
Aggregate Attributes [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071)#47174, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071)#47176, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)#47178, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)#47180]
Results [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071)#47174 AS COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#47231, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071)#47176 AS COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#47232, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)#47178 AS COL_DBB8940D_7041_41A8_9448_C56C73EB8044#47233, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)#47180 AS COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#47234]

(9) Generate [codegen id : 2]
Input [4]: [COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#47231, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#47232, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#47233, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#47234]
Arguments: stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#47231, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#47232, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#47233, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#47234), false, [col0#47258, col1#47259]

(10) Project [codegen id : 2]
Output [2]: [col0#47258 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#47285, col1#47259 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#47286]
Input [2]: [col0#47258, col1#47259]

(11) WriteFiles
Input [2]: [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#47285, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#47286]

(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/ff89ba3c_9f5a_4602_8cc1_d69a51de8cd9, false, Parquet, [path=file:///data/output/cache/parquet/datablock/ff89ba3c_9f5a_4602_8cc1_d69a51de8cd9], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

(13) Filter
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143]
Condition : (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143 <=> A)

(14) Project
Output [2]: [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#41141, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#41143]

(15) HashAggregate
Input [2]: [COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071, COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072]
Keys: []
Functions [4]: [partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)]
Aggregate Attributes [4]: [min#47356, max#47357, min#47358, max#47359]
Results [4]: [min#47360, max#47361, min#47362, max#47363]

(16) Exchange
Input [4]: [min#47360, max#47361, min#47362, max#47363]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10720]

(17) HashAggregate
Input [4]: [min#47360, max#47361, min#47362, max#47363]
Keys: []
Functions [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)]
Aggregate Attributes [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071)#47174, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071)#47176, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)#47178, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)#47180]
Results [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071)#47174 AS COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#47231, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#47071)#47176 AS COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#47232, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)#47178 AS COL_DBB8940D_7041_41A8_9448_C56C73EB8044#47233, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#47072)#47180 AS COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#47234]

(18) Generate
Input [4]: [COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#47231, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#47232, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#47233, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#47234]
Arguments: stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#47231, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#47232, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#47233, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#47234), false, [col0#47258, col1#47259]

(19) Project
Output [2]: [col0#47258 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#47285, col1#47259 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#47286]
Input [2]: [col0#47258, col1#47259]

(20) WriteFiles
Input [2]: [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#47285, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#47286]

(21) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/ff89ba3c_9f5a_4602_8cc1_d69a51de8cd9, false, Parquet, [path=file:///data/output/cache/parquet/datablock/ff89ba3c_9f5a_4602_8cc1_d69a51de8cd9], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

(22) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet