digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 1 ms";
2 [labelType="html" label="<br><b>Project</b><br><br>"];
3 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 59<br>spill size: 0.0 B"];
}
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 3 ms";
5 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
}
6 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 1162.0 B<br>number of coalesced partitions: 1"];
7 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 59<br>local merged chunks fetched: 0<br>shuffle write time: 1 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 59<br>local bytes read: 1107.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 1888.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 1107.0 B"];
subgraph cluster8 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: 5 ms";
9 [labelType="html" label="<br><b>LocalLimit</b><br><br>"];
10 [labelType="html" label="<br><b>Project</b><br><br>"];
11 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 59<br>number of input batches: 1"];
}
12 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time: 2 ms<br>metadata time: 0 ms<br>size of files read: 3.6 KiB<br>number of output rows: 59"];
subgraph cluster13 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: 0 ms";
14 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
}
15 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 972.0 B<br>number of coalesced partitions: 1"];
16 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 17<br>local merged chunks fetched: 0<br>shuffle write time: 1 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 17<br>local bytes read: 923.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 712.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 923.0 B"];
subgraph cluster17 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 5 ms";
18 [labelType="html" label="<br><b>Project</b><br><br>"];
19 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 17"];
20 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 17<br>number of input batches: 1"];
}
21 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time: 3 ms<br>metadata time: 0 ms<br>size of files read: 5.0 KiB<br>number of output rows: 17"];
2->0;
3->2;
5->3;
6->5;
7->6;
9->7;
10->9;
11->10;
12->11;
14->3;
15->14;
16->15;
18->16;
19->18;
20->19;
21->20;
}
22
AdaptiveSparkPlan isFinalPlan=true
Project [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694 AS COL_B6A3E50E_EDBF_43BB_8BC5_E616E4275F41#286735]
SortMergeJoin [COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L], [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L], LeftOuter
WholeStageCodegen (5)
Sort [COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L ASC NULLS FIRST], false, 0
WholeStageCodegen (3)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L, 10), ENSURE_REQUIREMENTS, [plan_id=39551]
LocalLimit 1500
Project [duration#286625L AS COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, cast(date_duration#286626 as timestamp) AS COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, id_expo#286627L AS COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [duration#286625L,date_duration#286626,id_expo#286627L] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/parquet/execution/HIST_ALL_ANALYSE_API/current/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<duration:bigint,date_duration:timestamp_ntz,id_expo:bigint>
Sort [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
AQEShuffleRead coalesced
Exchange hashpartitioning(DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, 10), ENSURE_REQUIREMENTS, [plan_id=39578]
Project [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L AS DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615 AS DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Filter isnotnull(COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L)
ColumnarToRow
WholeStageCodegen (2)
FileScan parquet [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L,COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615] Batched: true, DataFilters: [isnotnull(COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/DB_META_ALL_ANA_398939], PartitionFilters: [], PushedFilters: [IsNotNull(COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8)], ReadSchema: struct<COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8:bigint,COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4:s...
== Physical Plan ==
AdaptiveSparkPlan (30)
+- == Final Plan ==
* Project (18)
+- * SortMergeJoin LeftOuter (17)
:- * Sort (8)
: +- AQEShuffleRead (7)
: +- ShuffleQueryStage (6), Statistics(sizeInBytes=1888.0 B, rowCount=59)
: +- Exchange (5)
: +- * LocalLimit (4)
: +- * Project (3)
: +- * ColumnarToRow (2)
: +- Scan parquet (1)
+- * Sort (16)
+- AQEShuffleRead (15)
+- ShuffleQueryStage (14), Statistics(sizeInBytes=712.0 B, rowCount=17)
+- Exchange (13)
+- * Project (12)
+- * Filter (11)
+- * ColumnarToRow (10)
+- Scan parquet (9)
+- == Initial Plan ==
CollectLimit (29)
+- Project (28)
+- SortMergeJoin LeftOuter (27)
:- Sort (22)
: +- Exchange (21)
: +- LocalLimit (20)
: +- Project (19)
: +- Scan parquet (1)
+- Sort (26)
+- Exchange (25)
+- Project (24)
+- Filter (23)
+- Scan parquet (9)
(1) Scan parquet
Output [3]: [duration#286625L, date_duration#286626, id_expo#286627L]
Batched: true
Location: InMemoryFileIndex [file:/data/input/depot/parquet/execution/HIST_ALL_ANALYSE_API/current/pydc_2sx3r3oe.parquet]
ReadSchema: struct<duration:bigint,date_duration:timestamp_ntz,id_expo:bigint>
(2) ColumnarToRow [codegen id : 1]
Input [3]: [duration#286625L, date_duration#286626, id_expo#286627L]
(3) Project [codegen id : 1]
Output [3]: [duration#286625L AS COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, cast(date_duration#286626 as timestamp) AS COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, id_expo#286627L AS COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Input [3]: [duration#286625L, date_duration#286626, id_expo#286627L]
(4) LocalLimit [codegen id : 1]
Input [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Arguments: 1500
(5) Exchange
Input [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Arguments: hashpartitioning(COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L, 10), ENSURE_REQUIREMENTS, [plan_id=39551]
(6) ShuffleQueryStage
Output [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Arguments: 0
(7) AQEShuffleRead
Input [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Arguments: coalesced
(8) Sort [codegen id : 3]
Input [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Arguments: [COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L ASC NULLS FIRST], false, 0
(9) Scan parquet
Output [2]: [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/DB_META_ALL_ANA_398939]
PushedFilters: [IsNotNull(COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8)]
ReadSchema: struct<COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8:bigint,COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4:string>
(10) ColumnarToRow [codegen id : 2]
Input [2]: [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615]
(11) Filter [codegen id : 2]
Input [2]: [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615]
Condition : isnotnull(COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L)
(12) Project [codegen id : 2]
Output [2]: [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L AS DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615 AS DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Input [2]: [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615]
(13) Exchange
Input [2]: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Arguments: hashpartitioning(DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, 10), ENSURE_REQUIREMENTS, [plan_id=39578]
(14) ShuffleQueryStage
Output [2]: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Arguments: 1
(15) AQEShuffleRead
Input [2]: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Arguments: coalesced
(16) Sort [codegen id : 4]
Input [2]: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Arguments: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L ASC NULLS FIRST], false, 0
(17) SortMergeJoin [codegen id : 5]
Left keys [1]: [COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Right keys [1]: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L]
Join type: LeftOuter
Join condition: None
(18) Project [codegen id : 5]
Output [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694 AS COL_B6A3E50E_EDBF_43BB_8BC5_E616E4275F41#286735]
Input [5]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L, DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
(19) Project
Output [3]: [duration#286625L AS COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, cast(date_duration#286626 as timestamp) AS COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, id_expo#286627L AS COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Input [3]: [duration#286625L, date_duration#286626, id_expo#286627L]
(20) LocalLimit
Input [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Arguments: 1500
(21) Exchange
Input [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Arguments: hashpartitioning(COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L, 10), ENSURE_REQUIREMENTS, [plan_id=39529]
(22) Sort
Input [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Arguments: [COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L ASC NULLS FIRST], false, 0
(23) Filter
Input [2]: [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615]
Condition : isnotnull(COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L)
(24) Project
Output [2]: [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L AS DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615 AS DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Input [2]: [COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286607L, COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286615]
(25) Exchange
Input [2]: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Arguments: hashpartitioning(DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, 10), ENSURE_REQUIREMENTS, [plan_id=39530]
(26) Sort
Input [2]: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
Arguments: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L ASC NULLS FIRST], false, 0
(27) SortMergeJoin
Left keys [1]: [COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L]
Right keys [1]: [DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L]
Join type: LeftOuter
Join condition: None
(28) Project
Output [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694 AS COL_B6A3E50E_EDBF_43BB_8BC5_E616E4275F41#286735]
Input [5]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_346C834F_4ED0_48F3_BAA4_4CEE86C98F27#286673L, DB_META_ALL_ANA_398939_COL_3771B488_98F1_4DA6_9E11_BE7256EEF1E8#286686L, DB_META_ALL_ANA_398939_COL_6ED0E1C3_5350_4091_8240_CB8541E5E9A4#286694]
(29) CollectLimit
Input [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_B6A3E50E_EDBF_43BB_8BC5_E616E4275F41#286735]
Arguments: 1500
(30) AdaptiveSparkPlan
Output [3]: [COL_26752D67_E1AA_48C9_A46D_A9E1BA9C65FC#286671L, COL_7FDF7398_FCF6_40D3_B983_FF6950F2C4CA#286672, COL_B6A3E50E_EDBF_43BB_8BC5_E616E4275F41#286735]
Arguments: isFinalPlan=true