SP1 - Details for Query 222

Submitted Time: 2026/08/27 08:57:59
Duration: 0.5 s
Succeeded Jobs: 261 264

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label=" TakeOrderedAndProject "]; subgraph cluster2 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 16 ms"; 3 [labelType="html" label="HashAggregate spill size: 0.0 B time in aggregation build: 2 ms peak memory: 16.3 MiB number of output rows: 50 number of sort fallback tasks: 0 avg hash probes per key: 1"]; } 4 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 14.8 KiB number of coalesced partitions: 1"]; 5 [labelType="html" label="Exchange shuffle records written: 186 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 38 ms (6 ms, 7 ms, 10 ms (stage 370.0: task 467)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 186 local bytes read: 14.0 KiB fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 5 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 16.6 KiB (2.4 KiB, 3.5 KiB, 4.2 KiB (stage 370.0: task 464)) local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 14.0 KiB (2.2 KiB, 2.9 KiB, 3.3 KiB (stage 370.0: task 464))"]; subgraph cluster6 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n323 ms (63 ms, 64 ms, 68 ms (stage 370.0: task 465))"; 7 [labelType="html" label="HashAggregate spill size: 0.0 B time in aggregation build total (min, med, max (stageId: taskId)) 100 ms (18 ms, 19 ms, 24 ms (stage 370.0: task 465)) peak memory total (min, med, max (stageId: taskId)) 1280.0 KiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 370.0: task 466)) number of output rows: 186 number of sort fallback tasks: 0 avg hash probes per key: 0"]; 8 [labelType="html" label=" Project "]; 9 [labelType="html" label="Filter number of output rows: 4,038"]; 10 [labelType="html" label="ColumnarToRow number of output rows: 5,919 number of input batches: 5"]; } 11 [labelType="html" label="Scan parquet number of files read: 5 scan time total (min, med, max (stageId: taskId)) 26 ms (4 ms, 5 ms, 6 ms (stage 370.0: task 464)) metadata time: 0 ms size of files read: 88.8 KiB number of output rows: 5,919"]; 1->0; 3->1; 4->3; 5->4; 7->5; 8->7; 9->8; 10->9; 11->10; }

AdaptiveSparkPlan isFinalPlan=true

TakeOrderedAndProject(limit=500000, orderBy=[COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327 DESC NULLS LAST], output=[COL_8B6BBE02_8454_4784_91ED_D8089C0A5DD2#43268,COL_F4F446E9_78F7_4833_B94A_1B0E72E65F69#43333L,COL_99478157_CE04_4863_936B_33F871FEC0C9#43323L,COL_17B02F90_745F_457E_A5FC_DC1439812691#43325L,COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327])

HashAggregate(keys=[COL_3409B5EF_170B_42C0_893D_49B61679D364#43232], functions=[sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227), sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230), sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231), count(1)])

WholeStageCodegen (2)

AQEShuffleRead coalesced

Exchange hashpartitioning(COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, 10), ENSURE_REQUIREMENTS, [plan_id=8520]

HashAggregate(keys=[COL_3409B5EF_170B_42C0_893D_49B61679D364#43232], functions=[partial_sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227), partial_sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230), partial_sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231), partial_count(1)])

Project [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815 AS COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818 AS COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819 AS COL_603A9476_D6B3_486D_838A_B09E504BC177#43231, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820 AS COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]

Filter NOT (str_remove_accent(upper(COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825)) <=> 2)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815,COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818,COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819,COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820,COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825] Batched: true, DataFilters: [NOT (str_remove_accent(upper(COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825)) <=> 2)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/RHA_2021_CSARR__57211], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_969644EA_9245_4D2F_B985_FFA83A591910:int,COL_F0391FB9_833D_4445_9DF1_DDE2092461C8:int,...

Details

== Physical Plan ==
AdaptiveSparkPlan (17)
+- == Final Plan ==
   TakeOrderedAndProject (10)
   +- * HashAggregate (9)
      +- AQEShuffleRead (8)
         +- ShuffleQueryStage (7), Statistics(sizeInBytes=16.6 KiB, rowCount=186)
            +- Exchange (6)
               +- * HashAggregate (5)
                  +- * Project (4)
                     +- * Filter (3)
                        +- * ColumnarToRow (2)
                           +- Scan parquet  (1)
+- == Initial Plan ==
   TakeOrderedAndProject (16)
   +- HashAggregate (15)
      +- Exchange (14)
         +- HashAggregate (13)
            +- Project (12)
               +- Filter (11)
                  +- Scan parquet  (1)


(1) Scan parquet 
Output [5]: [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820, COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/RHA_2021_CSARR__57211]
ReadSchema: struct<COL_969644EA_9245_4D2F_B985_FFA83A591910:int,COL_F0391FB9_833D_4445_9DF1_DDE2092461C8:int,COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6:double,COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78:string,COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD:string>

(2) ColumnarToRow [codegen id : 1]
Input [5]: [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820, COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825]

(3) Filter [codegen id : 1]
Input [5]: [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820, COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825]
Condition : NOT (str_remove_accent(upper(COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825)) <=> 2)

(4) Project [codegen id : 1]
Output [4]: [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815 AS COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818 AS COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819 AS COL_603A9476_D6B3_486D_838A_B09E504BC177#43231, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820 AS COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]
Input [5]: [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820, COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825]

(5) HashAggregate [codegen id : 1]
Input [4]: [COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227, COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230, COL_603A9476_D6B3_486D_838A_B09E504BC177#43231, COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]
Keys [1]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]
Functions [4]: [partial_sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227), partial_sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230), partial_sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231), partial_count(1)]
Aggregate Attributes [4]: [sum#43603L, sum#43604L, sum#43605, count#43606L]
Results [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, sum#43607L, sum#43608L, sum#43609, count#43610L]

(6) Exchange
Input [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, sum#43607L, sum#43608L, sum#43609, count#43610L]
Arguments: hashpartitioning(COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, 10), ENSURE_REQUIREMENTS, [plan_id=8520]

(7) ShuffleQueryStage
Output [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, sum#43607L, sum#43608L, sum#43609, count#43610L]
Arguments: 0

(8) AQEShuffleRead
Input [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, sum#43607L, sum#43608L, sum#43609, count#43610L]
Arguments: coalesced

(9) HashAggregate [codegen id : 2]
Input [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, sum#43607L, sum#43608L, sum#43609, count#43610L]
Keys [1]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]
Functions [4]: [sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227), sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230), sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231), count(1)]
Aggregate Attributes [4]: [sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227)#43321L, sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230)#43324L, sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231)#43326, count(1)#43328L]
Results [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232 AS COL_8B6BBE02_8454_4784_91ED_D8089C0A5DD2#43268, sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227)#43321L AS COL_99478157_CE04_4863_936B_33F871FEC0C9#43323L, sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230)#43324L AS COL_17B02F90_745F_457E_A5FC_DC1439812691#43325L, sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231)#43326 AS COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327, count(1)#43328L AS COL_F4F446E9_78F7_4833_B94A_1B0E72E65F69#43333L]

(10) TakeOrderedAndProject
Input [5]: [COL_8B6BBE02_8454_4784_91ED_D8089C0A5DD2#43268, COL_99478157_CE04_4863_936B_33F871FEC0C9#43323L, COL_17B02F90_745F_457E_A5FC_DC1439812691#43325L, COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327, COL_F4F446E9_78F7_4833_B94A_1B0E72E65F69#43333L]
Arguments: 500000, [COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327 DESC NULLS LAST], [COL_8B6BBE02_8454_4784_91ED_D8089C0A5DD2#43268, COL_F4F446E9_78F7_4833_B94A_1B0E72E65F69#43333L, COL_99478157_CE04_4863_936B_33F871FEC0C9#43323L, COL_17B02F90_745F_457E_A5FC_DC1439812691#43325L, COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327]

(11) Filter
Input [5]: [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820, COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825]
Condition : NOT (str_remove_accent(upper(COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825)) <=> 2)

(12) Project
Output [4]: [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815 AS COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818 AS COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819 AS COL_603A9476_D6B3_486D_838A_B09E504BC177#43231, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820 AS COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]
Input [5]: [COL_969644EA_9245_4D2F_B985_FFA83A591910#42815, COL_F0391FB9_833D_4445_9DF1_DDE2092461C8#42818, COL_1C6DC50B_F3EB_4AC3_A3E7_648EA4B25CE6#42819, COL_C3A1E34C_FD52_40F9_8F3B_3571D0AECB78#42820, COL_023941C4_6DC9_4F47_BDB9_32BBA36217CD#42825]

(13) HashAggregate
Input [4]: [COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227, COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230, COL_603A9476_D6B3_486D_838A_B09E504BC177#43231, COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]
Keys [1]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]
Functions [4]: [partial_sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227), partial_sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230), partial_sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231), partial_count(1)]
Aggregate Attributes [4]: [sum#43603L, sum#43604L, sum#43605, count#43606L]
Results [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, sum#43607L, sum#43608L, sum#43609, count#43610L]

(14) Exchange
Input [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, sum#43607L, sum#43608L, sum#43609, count#43610L]
Arguments: hashpartitioning(COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, 10), ENSURE_REQUIREMENTS, [plan_id=8454]

(15) HashAggregate
Input [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232, sum#43607L, sum#43608L, sum#43609, count#43610L]
Keys [1]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232]
Functions [4]: [sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227), sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230), sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231), count(1)]
Aggregate Attributes [4]: [sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227)#43321L, sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230)#43324L, sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231)#43326, count(1)#43328L]
Results [5]: [COL_3409B5EF_170B_42C0_893D_49B61679D364#43232 AS COL_8B6BBE02_8454_4784_91ED_D8089C0A5DD2#43268, sum(COL_EC44D9BF_F630_4643_96AD_96BFB4B179DD#43227)#43321L AS COL_99478157_CE04_4863_936B_33F871FEC0C9#43323L, sum(COL_6D9C1BAD_462B_4648_95CA_EEA558923EBD#43230)#43324L AS COL_17B02F90_745F_457E_A5FC_DC1439812691#43325L, sum(COL_603A9476_D6B3_486D_838A_B09E504BC177#43231)#43326 AS COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327, count(1)#43328L AS COL_F4F446E9_78F7_4833_B94A_1B0E72E65F69#43333L]

(16) TakeOrderedAndProject
Input [5]: [COL_8B6BBE02_8454_4784_91ED_D8089C0A5DD2#43268, COL_99478157_CE04_4863_936B_33F871FEC0C9#43323L, COL_17B02F90_745F_457E_A5FC_DC1439812691#43325L, COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327, COL_F4F446E9_78F7_4833_B94A_1B0E72E65F69#43333L]
Arguments: 500000, [COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327 DESC NULLS LAST], [COL_8B6BBE02_8454_4784_91ED_D8089C0A5DD2#43268, COL_F4F446E9_78F7_4833_B94A_1B0E72E65F69#43333L, COL_99478157_CE04_4863_936B_33F871FEC0C9#43323L, COL_17B02F90_745F_457E_A5FC_DC1439812691#43325L, COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327]

(17) AdaptiveSparkPlan
Output [5]: [COL_8B6BBE02_8454_4784_91ED_D8089C0A5DD2#43268, COL_F4F446E9_78F7_4833_B94A_1B0E72E65F69#43333L, COL_99478157_CE04_4863_936B_33F871FEC0C9#43323L, COL_17B02F90_745F_457E_A5FC_DC1439812691#43325L, COL_259D63E7_D459_4DCB_80AC_A4896FDE2AED#43327]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet