SP1 - Details for Query 486

Submitted Time: 2026/02/23 14:36:45
Duration: 0.3 s
Succeeded Jobs: 652 653

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 2 ms"; 2 [labelType="html" label="HashAggregate spill size: 0.0 B time in aggregation build: 0 ms peak memory: 16.3 MiB number of output rows: 2 number of sort fallback tasks: 0 avg hash probes per key: 1"]; } 3 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 212.0 B number of coalesced partitions: 1"]; 4 [labelType="html" label="Exchange shuffle records written: 2 local merged chunks fetched: 0 shuffle write time: 0 ms remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 2 local bytes read: 198.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 1 remote merged chunks fetched: 0 remote blocks read: 0 data size: 128.0 B local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written: 198.0 B"]; subgraph cluster5 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: 71 ms"; 6 [labelType="html" label="HashAggregate spill size: 0.0 B time in aggregation build: 5 ms peak memory: 256.0 KiB number of output rows: 2 number of sort fallback tasks: 0 avg hash probes per key: 0"]; 7 [labelType="html" label=" Project "]; 8 [labelType="html" label="Filter number of output rows: 14"]; 9 [labelType="html" label="ColumnarToRow number of output rows: 8,480 number of input batches: 3"]; } 10 [labelType="html" label="Scan parquet number of files read: 1 scan time: 4 ms metadata time: 0 ms size of files read: 76.3 KiB number of output rows: 8,480"]; 2->0; 3->2; 4->3; 6->4; 7->6; 8->7; 9->8; 10->9; }

AdaptiveSparkPlan isFinalPlan=true

HashAggregate(keys=[COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625], functions=[sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)])

WholeStageCodegen (2)

AQEShuffleRead coalesced

Exchange hashpartitioning(COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, 10), ENSURE_REQUIREMENTS, [plan_id=14247]

HashAggregate(keys=[COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625], functions=[partial_sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)])

Project [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283 AS COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284 AS COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285 AS COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286 AS COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287 AS COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626]

Filter ((COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290 <=> 350000055) AND (COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291 <=> 2025-07))

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283,COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284,COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285,COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286,COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287,COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290,COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291] Batched: true, DataFilters: [(COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290 <=> 350000055), (COL_B648FE91_DE45_4C69_9C69_31..., Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/1_D_1_CDEM_DEMO_307387], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_687BAF68_8527_4100_B69B_339A76FFCD88,350000055), EqualNullSafe(COL_B648FE91_DE..., ReadSchema: struct<COL_3FC9CF25_115F_47C2_8706_BC734AAA7617:int,COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA:int,...

Details

== Physical Plan ==
AdaptiveSparkPlan (16)
+- == Final Plan ==
   * HashAggregate (9)
   +- AQEShuffleRead (8)
      +- ShuffleQueryStage (7), Statistics(sizeInBytes=128.0 B, rowCount=2)
         +- Exchange (6)
            +- * HashAggregate (5)
               +- * Project (4)
                  +- * Filter (3)
                     +- * ColumnarToRow (2)
                        +- Scan parquet  (1)
+- == Initial Plan ==
   CollectLimit (15)
   +- HashAggregate (14)
      +- Exchange (13)
         +- HashAggregate (12)
            +- Project (11)
               +- Filter (10)
                  +- Scan parquet  (1)


(1) Scan parquet 
Output [7]: [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287, COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290, COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/1_D_1_CDEM_DEMO_307387]
PushedFilters: [EqualNullSafe(COL_687BAF68_8527_4100_B69B_339A76FFCD88,350000055), EqualNullSafe(COL_B648FE91_DE45_4C69_9C69_3146520F0D91,2025-07)]
ReadSchema: struct<COL_3FC9CF25_115F_47C2_8706_BC734AAA7617:int,COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA:int,COL_38963EBF_223A_49F6_9EE1_560E597035A9:string,COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C:string,COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794:int,COL_687BAF68_8527_4100_B69B_339A76FFCD88:string,COL_B648FE91_DE45_4C69_9C69_3146520F0D91:string>

(2) ColumnarToRow [codegen id : 1]
Input [7]: [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287, COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290, COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291]

(3) Filter [codegen id : 1]
Input [7]: [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287, COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290, COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291]
Condition : ((COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290 <=> 350000055) AND (COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291 <=> 2025-07))

(4) Project [codegen id : 1]
Output [5]: [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283 AS COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284 AS COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285 AS COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286 AS COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287 AS COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626]
Input [7]: [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287, COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290, COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291]

(5) HashAggregate [codegen id : 1]
Input [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626]
Keys [4]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625]
Functions [1]: [partial_sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)]
Aggregate Attributes [1]: [sum#141008L]
Results [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, sum#141009L]

(6) Exchange
Input [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, sum#141009L]
Arguments: hashpartitioning(COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, 10), ENSURE_REQUIREMENTS, [plan_id=14247]

(7) ShuffleQueryStage
Output [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, sum#141009L]
Arguments: 0

(8) AQEShuffleRead
Input [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, sum#141009L]
Arguments: coalesced

(9) HashAggregate [codegen id : 2]
Input [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, sum#141009L]
Keys [4]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625]
Functions [1]: [sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)]
Aggregate Attributes [1]: [sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)#140699L]
Results [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622 AS COL_10AD9900_35D4_4FB3_9659_E4CAD8D0CAA2#140674, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623 AS COL_06C698A1_4BDA_4148_B279_B7A4E3F35A4A#140675, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624 AS COL_E9491A4D_C5D5_4A34_9641_6DD9A4665A16#140676, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625 AS COL_DBCF8546_BD3D_4E01_B93F_A181408D8A6F#140677, sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)#140699L AS COL_B5BF8810_F4CC_42E3_BE4B_4542493F02DC#140700L]

(10) Filter
Input [7]: [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287, COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290, COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291]
Condition : ((COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290 <=> 350000055) AND (COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291 <=> 2025-07))

(11) Project
Output [5]: [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283 AS COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284 AS COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285 AS COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286 AS COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287 AS COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626]
Input [7]: [COL_3FC9CF25_115F_47C2_8706_BC734AAA7617#140283, COL_943EA533_0089_4A3D_83A8_F2F6FEA02FEA#140284, COL_38963EBF_223A_49F6_9EE1_560E597035A9#140285, COL_3A733BA3_8EA8_4CC2_939F_8B0CF2121D6C#140286, COL_5F49C39D_0EB7_459A_A9B6_51E79C2C2794#140287, COL_687BAF68_8527_4100_B69B_339A76FFCD88#140290, COL_B648FE91_DE45_4C69_9C69_3146520F0D91#140291]

(12) HashAggregate
Input [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626]
Keys [4]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625]
Functions [1]: [partial_sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)]
Aggregate Attributes [1]: [sum#141008L]
Results [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, sum#141009L]

(13) Exchange
Input [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, sum#141009L]
Arguments: hashpartitioning(COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, 10), ENSURE_REQUIREMENTS, [plan_id=14204]

(14) HashAggregate
Input [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625, sum#141009L]
Keys [4]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625]
Functions [1]: [sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)]
Aggregate Attributes [1]: [sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)#140699L]
Results [5]: [COL_CA45D555_D2E9_4F11_9728_A217359075B0#140622 AS COL_10AD9900_35D4_4FB3_9659_E4CAD8D0CAA2#140674, COL_293EC427_9EF1_4C92_8C2F_B441571AB863#140623 AS COL_06C698A1_4BDA_4148_B279_B7A4E3F35A4A#140675, COL_A2C94FDE_E7D9_47CD_B68F_AE3A8BA192A3#140624 AS COL_E9491A4D_C5D5_4A34_9641_6DD9A4665A16#140676, COL_48A9E01E_48AB_4DCD_8974_C2B499C3CA2C#140625 AS COL_DBCF8546_BD3D_4E01_B93F_A181408D8A6F#140677, sum(COL_4EDFFB3F_BCE5_4833_BCED_2B9F03D43B8D#140626)#140699L AS COL_B5BF8810_F4CC_42E3_BE4B_4542493F02DC#140700L]

(15) CollectLimit
Input [5]: [COL_10AD9900_35D4_4FB3_9659_E4CAD8D0CAA2#140674, COL_06C698A1_4BDA_4148_B279_B7A4E3F35A4A#140675, COL_E9491A4D_C5D5_4A34_9641_6DD9A4665A16#140676, COL_DBCF8546_BD3D_4E01_B93F_A181408D8A6F#140677, COL_B5BF8810_F4CC_42E3_BE4B_4542493F02DC#140700L]
Arguments: 500000

(16) AdaptiveSparkPlan
Output [5]: [COL_10AD9900_35D4_4FB3_9659_E4CAD8D0CAA2#140674, COL_06C698A1_4BDA_4148_B279_B7A4E3F35A4A#140675, COL_E9491A4D_C5D5_4A34_9641_6DD9A4665A16#140676, COL_DBCF8546_BD3D_4E01_B93F_A181408D8A6F#140677, COL_B5BF8810_F4CC_42E3_BE4B_4542493F02DC#140700L]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet