SP1 - Details for Query 524

Submitted Time: 2026/02/24 07:34:37
Duration: 52 s
Succeeded Jobs: 691 693 694

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: 0 ms"; 2 [labelType="html" label="HashAggregate time in aggregation build: 0 ms number of output rows: 1"]; 3 [labelType="html" label="HashAggregate time in aggregation build: 0 ms number of output rows: 1"]; } 4 [labelType="html" label="Exchange shuffle records written: 200 local merged chunks fetched: 0 shuffle write time: 0 ms remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 200 local bytes read: 56.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 1 remote merged chunks fetched: 0 remote blocks read: 0 data size: 0.0 B local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written: 56.0 B"]; subgraph cluster5 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 27 ms"; 6 [labelType="html" label=" LocalLimit "]; 7 [labelType="html" label="HashAggregate spill size: 0.0 B time in aggregation build: 21 ms peak memory: 16.3 MiB number of output rows: 200 number of sort fallback tasks: 0 avg hash probes per key: 1"]; } 8 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 448.2 KiB number of coalesced partitions: 1"]; 9 [labelType="html" label="Exchange shuffle records written: 31,919 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 10 ms (1 ms, 1 ms, 2 ms (stage 908.0: task 1243)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 31,919 local bytes read: 429.2 KiB fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 5 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 853.3 KiB (10.9 KiB, 190.3 KiB, 281.2 KiB (stage 908.0: task 1242)) local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 429.2 KiB (6.6 KiB, 91.8 KiB, 145.2 KiB (stage 908.0: task 1242))"]; subgraph cluster10 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n2.1 m (876 ms, 20.8 s, 51.7 s (stage 908.0: task 1244))"; 11 [labelType="html" label="HashAggregate spill size: 0.0 B time in aggregation build total (min, med, max (stageId: taskId)) 2.1 m (864 ms, 20.8 s, 51.7 s (stage 908.0: task 1244)) peak memory total (min, med, max (stageId: taskId)) 65.3 MiB (256.0 KiB, 16.3 MiB, 16.3 MiB (stage 908.0: task 1245)) number of output rows: 31,919 number of sort fallback tasks: 0 avg hash probes per key (min, med, max (stageId: taskId)): (1, 1, 1 (stage 908.0: task 1245))"]; 12 [labelType="html" label=" Project "]; } 13 [labelType="html" label=" Project "]; subgraph cluster14 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.1 m (930 ms, 20.9 s, 51.7 s (stage 908.0: task 1244))"; 15 [labelType="html" label="Generate number of output rows: 316,675"]; } 16 [labelType="html" label=" Project "]; 17 [labelType="html" label="Filter number of output rows: 5"]; 18 [labelType="html" label="Scan binaryFile number of output rows: 5 number of files read: 5 metadata time: 0 ms size of files read: 34.6 MiB"]; 2->0; 3->2; 4->3; 6->4; 7->6; 8->7; 9->8; 11->9; 12->11; 13->12; 15->13; 16->15; 17->16; 18->17; }

AdaptiveSparkPlan isFinalPlan=true

HashAggregate(keys=[], functions=[count(1)])

HashAggregate(keys=[], functions=[partial_count(1)])

WholeStageCodegen (4)

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15161]

LocalLimit 200

HashAggregate(keys=[COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568], functions=[])

WholeStageCodegen (3)

AQEShuffleRead coalesced

Exchange hashpartitioning(COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568, 10), ENSURE_REQUIREMENTS, [plan_id=15028]

HashAggregate(keys=[COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568], functions=[])

Project [str_if_with_rule((((CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ,)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ,) END OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ;)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ;) END) OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, :)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, :) END) OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, .)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, .) END), str_extract(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, LEFT, dec_to_int(dec_operator(DIFFERENCE, array(cast(str_size(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467) as double), 1.0)))), COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467) AS COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]

WholeStageCodegen (2)

Project [str_transform_case(str_remove_accent(get_json_object(COL_21E39881_03B7_4478_981D_330734A120C0#147453, $.term)), UPPER) AS COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467]

Generate explode(COL_C3F4AA07_85F2_497E_9135_2A1292466B75#147449), false, [COL_21E39881_03B7_4478_981D_330734A120C0#147453]

WholeStageCodegen (1)

Project [from_json(ArrayType(StringType,false), to_json(bin_content_words(pdf, content#147382), Some(Etc/UTC)), Some(Etc/UTC)) AS COL_C3F4AA07_85F2_497E_9135_2A1292466B75#147449]

Filter ((size(from_json(ArrayType(StringType,false), to_json(bin_content_words(pdf, content#147382), Some(Etc/UTC)), Some(Etc/UTC)), true) > 0) AND isnotnull(from_json(ArrayType(StringType,false), to_json(bin_content_words(pdf, content#147382), Some(Etc/UTC)), Some(Etc/UTC))))

FileScan binaryFile [content#147382] Batched: false, DataFilters: [(size(from_json(ArrayType(StringType,false), to_json(bin_content_words(pdf, content#147382), Som..., Format: org.apache.spark.sql.execution.datasources.binaryfile.BinaryFileFormat@7e993be, Location: InMemoryFileIndex(5 paths)[file:/data/input/depot/binary/execution/454FFE78_6FB4_4FFA_AB5C_590D50..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<content:binary>

Details

== Physical Plan ==
AdaptiveSparkPlan (28)
+- == Final Plan ==
   * HashAggregate (16)
   +- * HashAggregate (15)
      +- ShuffleQueryStage (14), Statistics(sizeInBytes=0.0 B, rowCount=200)
         +- Exchange (13)
            +- * LocalLimit (12)
               +- * HashAggregate (11)
                  +- AQEShuffleRead (10)
                     +- ShuffleQueryStage (9), Statistics(sizeInBytes=853.3 KiB, rowCount=3.19E+4)
                        +- Exchange (8)
                           +- * HashAggregate (7)
                              +- * Project (6)
                                 +- Project (5)
                                    +- * Generate (4)
                                       +- Project (3)
                                          +- Filter (2)
                                             +- Scan binaryFile  (1)
+- == Initial Plan ==
   HashAggregate (27)
   +- HashAggregate (26)
      +- GlobalLimit (25)
         +- Exchange (24)
            +- LocalLimit (23)
               +- HashAggregate (22)
                  +- Exchange (21)
                     +- HashAggregate (20)
                        +- Project (19)
                           +- Project (18)
                              +- Generate (17)
                                 +- Project (3)
                                    +- Filter (2)
                                       +- Scan binaryFile  (1)


(1) Scan binaryFile 
Output [1]: [content#147382]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/binary/execution/454FFE78_6FB4_4FFA_AB5C_590D50B78C20/current/BENARD_Clement_these_2021.pdf, ... 4 entries]
ReadSchema: struct<content:binary>

(2) Filter
Input [1]: [content#147382]
Condition : ((size(from_json(ArrayType(StringType,false), to_json(bin_content_words(pdf, content#147382), Some(Etc/UTC)), Some(Etc/UTC)), true) > 0) AND isnotnull(from_json(ArrayType(StringType,false), to_json(bin_content_words(pdf, content#147382), Some(Etc/UTC)), Some(Etc/UTC))))

(3) Project
Output [1]: [from_json(ArrayType(StringType,false), to_json(bin_content_words(pdf, content#147382), Some(Etc/UTC)), Some(Etc/UTC)) AS COL_C3F4AA07_85F2_497E_9135_2A1292466B75#147449]
Input [1]: [content#147382]

(4) Generate [codegen id : 1]
Input [1]: [COL_C3F4AA07_85F2_497E_9135_2A1292466B75#147449]
Arguments: explode(COL_C3F4AA07_85F2_497E_9135_2A1292466B75#147449), false, [COL_21E39881_03B7_4478_981D_330734A120C0#147453]

(5) Project
Output [1]: [str_transform_case(str_remove_accent(get_json_object(COL_21E39881_03B7_4478_981D_330734A120C0#147453, $.term)), UPPER) AS COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467]
Input [1]: [COL_21E39881_03B7_4478_981D_330734A120C0#147453]

(6) Project [codegen id : 2]
Output [1]: [str_if_with_rule((((CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ,)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ,) END OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ;)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ;) END) OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, :)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, :) END) OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, .)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, .) END), str_extract(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, LEFT, dec_to_int(dec_operator(DIFFERENCE, array(cast(str_size(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467) as double), 1.0)))), COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467) AS COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Input [1]: [COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467]

(7) HashAggregate [codegen id : 2]
Input [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Keys [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Functions: []
Aggregate Attributes: []
Results [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]

(8) Exchange
Input [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Arguments: hashpartitioning(COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568, 10), ENSURE_REQUIREMENTS, [plan_id=15028]

(9) ShuffleQueryStage
Output [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Arguments: 0

(10) AQEShuffleRead
Input [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Arguments: coalesced

(11) HashAggregate [codegen id : 3]
Input [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Keys [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Functions: []
Aggregate Attributes: []
Results: []

(12) LocalLimit [codegen id : 3]
Input: []
Arguments: 200

(13) Exchange
Input: []
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15161]

(14) ShuffleQueryStage
Output: []
Arguments: 1

(15) HashAggregate [codegen id : 4]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#147651L]
Results [1]: [count#147652L]

(16) HashAggregate [codegen id : 4]
Input [1]: [count#147652L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#147648L]
Results [1]: [count(1)#147648L AS count#147649L]

(17) Generate
Input [1]: [COL_C3F4AA07_85F2_497E_9135_2A1292466B75#147449]
Arguments: explode(COL_C3F4AA07_85F2_497E_9135_2A1292466B75#147449), false, [COL_21E39881_03B7_4478_981D_330734A120C0#147453]

(18) Project
Output [1]: [str_transform_case(str_remove_accent(get_json_object(COL_21E39881_03B7_4478_981D_330734A120C0#147453, $.term)), UPPER) AS COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467]
Input [1]: [COL_21E39881_03B7_4478_981D_330734A120C0#147453]

(19) Project
Output [1]: [str_if_with_rule((((CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ,)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ,) END OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ;)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, ;) END) OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, :)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, :) END) OR CASE WHEN isnull(EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, .)) THEN false ELSE EndsWith(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, .) END), str_extract(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467, LEFT, dec_to_int(dec_operator(DIFFERENCE, array(cast(str_size(COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467) as double), 1.0)))), COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467) AS COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Input [1]: [COL_8024C6FB_6C34_46E0_91CB_695910BF0C1B#147467]

(20) HashAggregate
Input [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Keys [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Functions: []
Aggregate Attributes: []
Results [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]

(21) Exchange
Input [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Arguments: hashpartitioning(COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568, 10), ENSURE_REQUIREMENTS, [plan_id=14993]

(22) HashAggregate
Input [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Keys [1]: [COL_BAC7322B_FA9B_430E_B73B_810A78B5874D#147568]
Functions: []
Aggregate Attributes: []
Results: []

(23) LocalLimit
Input: []
Arguments: 200

(24) Exchange
Input: []
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=14997]

(25) GlobalLimit
Input: []
Arguments: 200, 0

(26) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#147651L]
Results [1]: [count#147652L]

(27) HashAggregate
Input [1]: [count#147652L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#147648L]
Results [1]: [count(1)#147648L AS count#147649L]

(28) AdaptiveSparkPlan
Output [1]: [count#147649L]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet