hdh-spark-job - Details for Query 1533

Submitted Time: 2026/10/23 14:48:20
Duration: 83 ms
Succeeded Jobs: 1555 1557

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 0 ms"; 2 [labelType="html" label=" Project "]; 3 [labelType="html" label=" Project "]; 4 [labelType="html" label="HashAggregate time in aggregation build: 0 ms number of output rows: 1"]; } 5 [labelType="html" label="Exchange shuffle records written: 1 local merged chunks fetched: 0 shuffle write time: 0 ms remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 1 local bytes read: 63.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 1 remote merged chunks fetched: 0 remote blocks read: 0 data size: 24.0 B local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written: 63.0 B"]; subgraph cluster6 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: 2 ms"; 7 [labelType="html" label="HashAggregate time in aggregation build: 2 ms number of output rows: 1"]; 8 [labelType="html" label=" Project "]; 9 [labelType="html" label="ColumnarToRow number of output rows: 270 number of input batches: 1"]; } 10 [labelType="html" label="Scan parquet number of files read: 1 scan time: 2 ms metadata time: 0 ms size of files read: 40.1 KiB number of output rows: 270"]; 2->0; 3->2; 4->3; 5->4; 7->5; 8->7; 9->8; 10->9; }

AdaptiveSparkPlan isFinalPlan=true

Project [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L, str_concat(array(dec_to_str(dec_round(dec_mult(array(dec_div(array(COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L)), 100.0)), 2)), %)) AS COL_D98DEF51_C432_4FAD_8FBD_F3196E94254A#793607]

Project [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, bigint_sum(array(COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L)) AS COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L]

HashAggregate(keys=[], functions=[sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503), sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)])

WholeStageCodegen (2)

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=42411]

HashAggregate(keys=[], functions=[partial_sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503), partial_sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)])

Project [COL_664D7B67_EB70_4B92_A103_5457F8C0C367#793212 AS COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503, COL_A983C304_F802_42B4_B60D_067460C7DD14#793213 AS COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505]

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_664D7B67_EB70_4B92_A103_5457F8C0C367#793212,COL_A983C304_F802_42B4_B60D_067460C7DD14#793213] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/LES_CONNECTEURS_326363], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_664D7B67_EB70_4B92_A103_5457F8C0C367:int,COL_A983C304_F802_42B4_B60D_067460C7DD14:int>

Details

== Physical Plan ==
AdaptiveSparkPlan (16)
+- == Final Plan ==
   * Project (9)
   +- * Project (8)
      +- * HashAggregate (7)
         +- ShuffleQueryStage (6), Statistics(sizeInBytes=24.0 B, rowCount=1)
            +- Exchange (5)
               +- * HashAggregate (4)
                  +- * Project (3)
                     +- * ColumnarToRow (2)
                        +- Scan parquet  (1)
+- == Initial Plan ==
   Project (15)
   +- Project (14)
      +- HashAggregate (13)
         +- Exchange (12)
            +- HashAggregate (11)
               +- Project (10)
                  +- Scan parquet  (1)


(1) Scan parquet 
Output [2]: [COL_664D7B67_EB70_4B92_A103_5457F8C0C367#793212, COL_A983C304_F802_42B4_B60D_067460C7DD14#793213]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/LES_CONNECTEURS_326363]
ReadSchema: struct<COL_664D7B67_EB70_4B92_A103_5457F8C0C367:int,COL_A983C304_F802_42B4_B60D_067460C7DD14:int>

(2) ColumnarToRow [codegen id : 1]
Input [2]: [COL_664D7B67_EB70_4B92_A103_5457F8C0C367#793212, COL_A983C304_F802_42B4_B60D_067460C7DD14#793213]

(3) Project [codegen id : 1]
Output [2]: [COL_664D7B67_EB70_4B92_A103_5457F8C0C367#793212 AS COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503, COL_A983C304_F802_42B4_B60D_067460C7DD14#793213 AS COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505]
Input [2]: [COL_664D7B67_EB70_4B92_A103_5457F8C0C367#793212, COL_A983C304_F802_42B4_B60D_067460C7DD14#793213]

(4) HashAggregate [codegen id : 1]
Input [2]: [COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503, COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505]
Keys: []
Functions [2]: [partial_sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503), partial_sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)]
Aggregate Attributes [2]: [sum#793773L, sum#793774L]
Results [2]: [sum#793775L, sum#793776L]

(5) Exchange
Input [2]: [sum#793775L, sum#793776L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=42411]

(6) ShuffleQueryStage
Output [2]: [sum#793775L, sum#793776L]
Arguments: 0

(7) HashAggregate [codegen id : 2]
Input [2]: [sum#793775L, sum#793776L]
Keys: []
Functions [2]: [sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503), sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)]
Aggregate Attributes [2]: [sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503)#793543L, sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)#793545L]
Results [2]: [sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503)#793543L AS COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)#793545L AS COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L]

(8) Project [codegen id : 2]
Output [3]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, bigint_sum(array(COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L)) AS COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L]
Input [2]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L]

(9) Project [codegen id : 2]
Output [4]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L, str_concat(array(dec_to_str(dec_round(dec_mult(array(dec_div(array(COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L)), 100.0)), 2)),  %)) AS COL_D98DEF51_C432_4FAD_8FBD_F3196E94254A#793607]
Input [3]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L]

(10) Project
Output [2]: [COL_664D7B67_EB70_4B92_A103_5457F8C0C367#793212 AS COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503, COL_A983C304_F802_42B4_B60D_067460C7DD14#793213 AS COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505]
Input [2]: [COL_664D7B67_EB70_4B92_A103_5457F8C0C367#793212, COL_A983C304_F802_42B4_B60D_067460C7DD14#793213]

(11) HashAggregate
Input [2]: [COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503, COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505]
Keys: []
Functions [2]: [partial_sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503), partial_sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)]
Aggregate Attributes [2]: [sum#793773L, sum#793774L]
Results [2]: [sum#793775L, sum#793776L]

(12) Exchange
Input [2]: [sum#793775L, sum#793776L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=42392]

(13) HashAggregate
Input [2]: [sum#793775L, sum#793776L]
Keys: []
Functions [2]: [sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503), sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)]
Aggregate Attributes [2]: [sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503)#793543L, sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)#793545L]
Results [2]: [sum(COL_86E8C454_268D_4B9D_B6BC_96D71825FDD6#793503)#793543L AS COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, sum(COL_D9680567_70AC_4D1E_B623_BEDBD5ED98FA#793505)#793545L AS COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L]

(14) Project
Output [3]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, bigint_sum(array(COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L)) AS COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L]
Input [2]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L]

(15) Project
Output [4]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L, str_concat(array(dec_to_str(dec_round(dec_mult(array(dec_div(array(COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L)), 100.0)), 2)),  %)) AS COL_D98DEF51_C432_4FAD_8FBD_F3196E94254A#793607]
Input [3]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L]

(16) AdaptiveSparkPlan
Output [4]: [COL_EB894600_7CAD_4E70_A293_21CC57C83BAD#793544L, COL_048017A2_98AB_4321_B7E1_B3E1F1DFEEEC#793546L, COL_181EC327_E1E8_4053_90E1_267EE70B9ED4#793568L, COL_D98DEF51_C432_4FAD_8FBD_F3196E94254A#793607]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet