SP1 - Details for Query 129

Submitted Time: 2026/06/25 10:31:36
Duration: 4 s
Succeeded Jobs: 165 166

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand task commit time: 1 ms number of written files: 1 job commit time: 14 ms number of output rows: 4 number of dynamic part: 0 written output: 1031.0 B"]; 2 [labelType="html" label=" WriteFiles "]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 32 ms"; 4 [labelType="html" label=" Project "]; 5 [labelType="html" label="Generate number of output rows: 4"]; 6 [labelType="html" label="HashAggregate time in aggregation build: 1 ms number of output rows: 1"]; } 7 [labelType="html" label="Exchange shuffle records written: 7 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 3 ms (0 ms, 0 ms, 0 ms (stage 260.0: task 238)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 7 local bytes read: 431.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 7 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 280.0 B (40.0 B, 40.0 B, 40.0 B (stage 260.0: task 235)) local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 431.0 B (58.0 B, 58.0 B, 83.0 B (stage 260.0: task 238))"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n3.6 s (3 ms, 5 ms, 3.5 s (stage 260.0: task 238))"; 9 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 3.6 s (3 ms, 5 ms, 3.5 s (stage 260.0: task 238)) number of output rows: 7"]; 10 [labelType="html" label=" Project "]; 11 [labelType="html" label="Filter number of output rows: 114,063"]; 12 [labelType="html" label="ColumnarToRow number of output rows: 509,703 number of input batches: 125"]; } 13 [labelType="html" label="Scan parquet number of files read: 1 scan time total (min, med, max (stageId: taskId)) 218 ms (3 ms, 5 ms, 182 ms (stage 260.0: task 238)) metadata time: 0 ms size of files read: 25.4 MiB number of output rows: 509,703"]; 1->0; 2->1; 4->2; 5->4; 6->5; 7->6; 9->7; 10->9; 11->10; 12->11; 13->12; }

AdaptiveSparkPlan isFinalPlan=true

Execute InsertIntoHadoopFsRelationCommand file:/data/output/cache/parquet/datablock/8b597b57_4d7e_4d11_8940_fa31fbf31f2f, false, Parquet, [path=file:///data/output/cache/parquet/datablock/8b597b57_4d7e_4d11_8940_fa31fbf31f2f], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

WriteFiles

Project [col0#31353 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#31534, col1#31354 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#31535]

Generate stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#31140, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#31141, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#31142, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#31143), false, [col0#31353, col1#31354]

HashAggregate(keys=[], functions=[min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)])

WholeStageCodegen (2)

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7631]

HashAggregate(keys=[], functions=[partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)])

Project [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935]

Filter (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304 <=> A)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304] Batched: true, DataFilters: [(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304 <=> A)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411360], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9,A)], ReadSchema: struct<COL_30863531_1D7E_428B_B583_E41A16C72ABB:string,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9:s...

Details

== Physical Plan ==
AdaptiveSparkPlan (22)
+- == Final Plan ==
   Execute InsertIntoHadoopFsRelationCommand (12)
   +- WriteFiles (11)
      +- * Project (10)
         +- * Generate (9)
            +- * HashAggregate (8)
               +- ShuffleQueryStage (7), Statistics(sizeInBytes=280.0 B, rowCount=7)
                  +- Exchange (6)
                     +- * HashAggregate (5)
                        +- * Project (4)
                           +- * Filter (3)
                              +- * ColumnarToRow (2)
                                 +- Scan parquet  (1)
+- == Initial Plan ==
   Execute InsertIntoHadoopFsRelationCommand (21)
   +- WriteFiles (20)
      +- Project (19)
         +- Generate (18)
            +- HashAggregate (17)
               +- Exchange (16)
                  +- HashAggregate (15)
                     +- Project (14)
                        +- Filter (13)
                           +- Scan parquet  (1)


(1) Scan parquet 
Output [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/01_A_PRE_DECOUP_411360]
PushedFilters: [EqualNullSafe(COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9,A)]
ReadSchema: struct<COL_30863531_1D7E_428B_B583_E41A16C72ABB:string,COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9:string>

(2) ColumnarToRow [codegen id : 1]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304]

(3) Filter [codegen id : 1]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304]
Condition : (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304 <=> A)

(4) Project [codegen id : 1]
Output [2]: [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304]

(5) HashAggregate [codegen id : 1]
Input [2]: [COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934, COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935]
Keys: []
Functions [4]: [partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)]
Aggregate Attributes [4]: [min#31927, max#31928, min#31929, max#31930]
Results [4]: [min#31935, max#31936, min#31937, max#31938]

(6) Exchange
Input [4]: [min#31935, max#31936, min#31937, max#31938]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7631]

(7) ShuffleQueryStage
Output [4]: [min#31935, max#31936, min#31937, max#31938]
Arguments: 0

(8) HashAggregate [codegen id : 2]
Input [4]: [min#31935, max#31936, min#31937, max#31938]
Keys: []
Functions [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)]
Aggregate Attributes [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934)#30957, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934)#30959, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)#30961, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)#30963]
Results [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934)#30957 AS COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#31140, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934)#30959 AS COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#31141, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)#30961 AS COL_DBB8940D_7041_41A8_9448_C56C73EB8044#31142, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)#30963 AS COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#31143]

(9) Generate [codegen id : 2]
Input [4]: [COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#31140, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#31141, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#31142, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#31143]
Arguments: stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#31140, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#31141, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#31142, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#31143), false, [col0#31353, col1#31354]

(10) Project [codegen id : 2]
Output [2]: [col0#31353 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#31534, col1#31354 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#31535]
Input [2]: [col0#31353, col1#31354]

(11) WriteFiles
Input [2]: [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#31534, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#31535]

(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/8b597b57_4d7e_4d11_8940_fa31fbf31f2f, false, Parquet, [path=file:///data/output/cache/parquet/datablock/8b597b57_4d7e_4d11_8940_fa31fbf31f2f], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

(13) Filter
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304]
Condition : (COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304 <=> A)

(14) Project
Output [2]: [str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, 85, 8), ddMMyyyy, Europe/Paris, null) AS COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934, str_to_datez(str_extract_middle(COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, 93, 8), ddMMyyyy, Europe/Paris, null) AS COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935]
Input [2]: [COL_30863531_1D7E_428B_B583_E41A16C72ABB#25302, COL_17B43B85_6EF2_4511_9F9E_84EDB43F71E9#25304]

(15) HashAggregate
Input [2]: [COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934, COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935]
Keys: []
Functions [4]: [partial_min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), partial_max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), partial_min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935), partial_max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)]
Aggregate Attributes [4]: [min#31927, max#31928, min#31929, max#31930]
Results [4]: [min#31935, max#31936, min#31937, max#31938]

(16) Exchange
Input [4]: [min#31935, max#31936, min#31937, max#31938]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7606]

(17) HashAggregate
Input [4]: [min#31935, max#31936, min#31937, max#31938]
Keys: []
Functions [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934), min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935), max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)]
Aggregate Attributes [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934)#30957, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934)#30959, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)#30961, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)#30963]
Results [4]: [min(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934)#30957 AS COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#31140, max(COL_D0465C8E_6B06_4A7A_8B80_515D022F7F71#30934)#30959 AS COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#31141, min(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)#30961 AS COL_DBB8940D_7041_41A8_9448_C56C73EB8044#31142, max(COL_3012EB13_C19A_4C0E_B9D6_1602956EDD6E#30935)#30963 AS COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#31143]

(18) Generate
Input [4]: [COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#31140, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#31141, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#31142, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#31143]
Arguments: stack(4, Date Entrée Minimum, COL_7BF3C93F_BA3F_4AF5_8B33_4CE84EE7D5E9#31140, Date Entrée Maximum, COL_DBBAFF90_70C2_4384_B0BB_D610BDCCAACB#31141, Date Sortie Minimum, COL_DBB8940D_7041_41A8_9448_C56C73EB8044#31142, Date Sortie Maximum, COL_99327D62_52F1_4F11_A1A9_9374D2F6D2FC#31143), false, [col0#31353, col1#31354]

(19) Project
Output [2]: [col0#31353 AS COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#31534, col1#31354 AS COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#31535]
Input [2]: [col0#31353, col1#31354]

(20) WriteFiles
Input [2]: [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A#31534, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A#31535]

(21) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/8b597b57_4d7e_4d11_8940_fa31fbf31f2f, false, Parquet, [path=file:///data/output/cache/parquet/datablock/8b597b57_4d7e_4d11_8940_fa31fbf31f2f], Overwrite, [COL_0C6E843F_84DB_4A2F_9715_BA558C46163A, COL_5D808C60_1732_48DB_A4EB_D68FAE81C26A]

(22) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet