digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 2 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 64.2 MiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
}
3 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 72.0 B<br>number of coalesced partitions: 1"];
4 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 72.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 32.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 72.0 B"];
subgraph cluster5 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: 60 ms";
6 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 10 ms<br>peak memory: 256.0 KiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
7 [labelType="html" label="<br><b>Project</b><br><br>"];
8 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1,216"];
}
9 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 1,216<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 111.6 KiB"];
2->0;
3->2;
4->3;
6->4;
7->6;
8->7;
9->8;
}
10
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848], functions=[count(1)])
WholeStageCodegen (2)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, 10), ENSURE_REQUIREMENTS, [plan_id=3087]
HashAggregate(keys=[COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848], functions=[partial_count(1)])
Project [str_extract_middle(_c0#17785, 10, 2) AS COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Filter isnotnull(str_extract_middle(_c0#17785, 10, 2))
WholeStageCodegen (1)
FileScan csv [_c0#17785] Batched: false, DataFilters: [isnotnull(str_extract_middle(_c0#17785, 10, 2))], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/DMI_M12_2025_440461/DMI-M12-2025 - BIS...., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string>
== Physical Plan ==
AdaptiveSparkPlan (15)
+- == Final Plan ==
* HashAggregate (8)
+- AQEShuffleRead (7)
+- ShuffleQueryStage (6), Statistics(sizeInBytes=32.0 B, rowCount=1)
+- Exchange (5)
+- * HashAggregate (4)
+- * Project (3)
+- * Filter (2)
+- Scan csv (1)
+- == Initial Plan ==
TakeOrderedAndProject (14)
+- HashAggregate (13)
+- Exchange (12)
+- HashAggregate (11)
+- Project (10)
+- Filter (9)
+- Scan csv (1)
(1) Scan csv
Output [1]: [_c0#17785]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/tmp/DMI_M12_2025_440461/DMI-M12-2025 - BIS.txt]
ReadSchema: struct<_c0:string>
(2) Filter [codegen id : 1]
Input [1]: [_c0#17785]
Condition : isnotnull(str_extract_middle(_c0#17785, 10, 2))
(3) Project [codegen id : 1]
Output [1]: [str_extract_middle(_c0#17785, 10, 2) AS COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Input [1]: [_c0#17785]
(4) HashAggregate [codegen id : 1]
Input [1]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Keys [1]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#17880L]
Results [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, count#17881L]
(5) Exchange
Input [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, count#17881L]
Arguments: hashpartitioning(COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, 10), ENSURE_REQUIREMENTS, [plan_id=3087]
(6) ShuffleQueryStage
Output [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, count#17881L]
Arguments: 0
(7) AQEShuffleRead
Input [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, count#17881L]
Arguments: coalesced
(8) HashAggregate [codegen id : 2]
Input [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, count#17881L]
Keys [1]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#17871L]
Results [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848 AS COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F581#17862, count(1)#17871L AS COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F582#17873L]
(9) Filter
Input [1]: [_c0#17785]
Condition : isnotnull(str_extract_middle(_c0#17785, 10, 2))
(10) Project
Output [1]: [str_extract_middle(_c0#17785, 10, 2) AS COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Input [1]: [_c0#17785]
(11) HashAggregate
Input [1]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Keys [1]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#17880L]
Results [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, count#17881L]
(12) Exchange
Input [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, count#17881L]
Arguments: hashpartitioning(COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, 10), ENSURE_REQUIREMENTS, [plan_id=3070]
(13) HashAggregate
Input [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848, count#17881L]
Keys [1]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848]
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#17871L]
Results [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F58#17848 AS COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F581#17862, count(1)#17871L AS COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F582#17873L]
(14) TakeOrderedAndProject
Input [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F581#17862, COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F582#17873L]
Arguments: 10, [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F582#17873L DESC NULLS LAST], [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F581#17862, COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F582#17873L]
(15) AdaptiveSparkPlan
Output [2]: [COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F581#17862, COL_5F7ED7D9_198B_43FF_AA77_F43BC93B4F582#17873L]
Arguments: isFinalPlan=true