digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 5<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>1 ms (0 ms, 0 ms, 0 ms (stage 1020.0: task 1307))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 5<br>local bytes read: 295.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 5<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>80.0 B (16.0 B, 16.0 B, 16.0 B (stage 1020.0: task 1307))<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>295.0 B (59.0 B, 59.0 B, 59.0 B (stage 1020.0: task 1307))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n1.4 s (262 ms, 269 ms, 282 ms (stage 1020.0: task 1306))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build total (min, med, max (stageId: taskId))<br>1.4 s (261 ms, 269 ms, 282 ms (stage 1020.0: task 1306))<br>peak memory: 0.0 B<br>number of output rows: 5<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
7 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 602,754<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 1020.0: task 1307))"];
}
subgraph cluster8 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n1.7 s (327 ms, 333 ms, 345 ms (stage 1020.0: task 1308))";
9 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>82 ms (16 ms, 16 ms, 18 ms (stage 1020.0: task 1305))<br>peak memory total (min, med, max (stageId: taskId))<br>100.0 MiB (20.0 MiB, 20.0 MiB, 20.0 MiB (stage 1020.0: task 1307))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 1020.0: task 1307))"];
}
10 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 5<br>partition data size total (min, med, max (stageId: taskId))<br>4.8 MiB (981.7 KiB, 981.7 KiB, 981.7 KiB (driver))<br>number of coalesced partitions: 5"];
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 602,754<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>17 ms (4 ms, 12 ms, 12 ms (stage 1016.0: task 1296))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 602,754<br>local bytes read total (min, med, max (stageId: taskId))<br>4.7 MiB (965.0 KiB, 972.4 KiB, 973.6 KiB (stage 1020.0: task 1307))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 1020.0: task 1307))<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 10<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>18.4 MiB (2.8 MiB, 15.6 MiB, 15.6 MiB (stage 1016.0: task 1296))<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>4.7 MiB (730.8 KiB, 4.0 MiB, 4.0 MiB (stage 1016.0: task 1296))"];
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n884 ms (5 ms, 7 ms, 651 ms (stage 1016.0: task 1296))";
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 602,754"];
15 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 2,996,009<br>number of input batches: 732"];
}
16 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time total (min, med, max (stageId: taskId))<br>101 ms (4 ms, 6 ms, 35 ms (stage 1016.0: task 1296))<br>metadata time: 0 ms<br>size of files read: 150.9 MiB<br>number of output rows: 2,996,009"];
subgraph cluster17 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 0 ms";
18 [labelType="html" label="<br><b>Project</b><br><br>"];
19 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 88,319"];
}
20 [labelType="html" label="<b>Window</b><br><br>spill size: 0.0 B"];
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: 0 ms";
22 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>5 ms (1 ms, 1 ms, 1 ms (stage 1020.0: task 1307))<br>peak memory total (min, med, max (stageId: taskId))<br>82.5 MiB (16.5 MiB, 16.5 MiB, 16.5 MiB (stage 1020.0: task 1307))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 1020.0: task 1307))"];
}
23 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 5<br>partition data size total (min, med, max (stageId: taskId))<br>780.9 KiB (152.2 KiB, 156.4 KiB, 159.6 KiB (driver))<br>number of coalesced partitions: 5"];
24 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 93,005<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>5 ms (2 ms, 3 ms, 3 ms (stage 1017.0: task 1302))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 93,005<br>local bytes read total (min, med, max (stageId: taskId))<br>736.8 KiB (145.8 KiB, 147.4 KiB, 149.7 KiB (stage 1020.0: task 1307))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 1020.0: task 1307))<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 10<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>2.8 MiB (1195.3 KiB, 1711.1 KiB, 1711.1 KiB (stage 1017.0: task 1302))<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>736.8 KiB (308.6 KiB, 428.2 KiB, 428.2 KiB (stage 1017.0: task 1302))"];
subgraph cluster25 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n264 ms (108 ms, 156 ms, 156 ms (stage 1017.0: task 1302))";
26 [labelType="html" label="<br><b>Project</b><br><br>"];
27 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 93,005"];
}
28 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 93,005<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 6.4 MiB"];
2->0;
3->2;
5->3;
6->5;
7->6;
9->7;
10->9;
11->10;
13->11;
14->13;
15->14;
16->15;
18->7;
19->18;
20->19;
22->20;
23->22;
24->23;
26->24;
27->26;
28->27;
}
29
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (7)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=25357]
HashAggregate(keys=[], functions=[partial_count(1)])
Project
SortMergeJoin [COL_136DCAA3_2545_47C9_A61E_86ED7914066D#96161], [00_C_PASSAGE_PA_345648_COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#96370], LeftOuter
WholeStageCodegen (6)
Sort [COL_136DCAA3_2545_47C9_A61E_86ED7914066D#96161 ASC NULLS FIRST], false, 0
WholeStageCodegen (3)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_136DCAA3_2545_47C9_A61E_86ED7914066D#96161, 10), ENSURE_REQUIREMENTS, [plan_id=25206]
Project [COL_1CCAED33_3E69_435E_B522_DAE3B9E63F9D#96057 AS COL_136DCAA3_2545_47C9_A61E_86ED7914066D#96161]
Filter (COL_1ADA21FB_DB08_4A1F_986E_E8E363B14F38#96054 <=> A)
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [COL_1ADA21FB_DB08_4A1F_986E_E8E363B14F38#96054,COL_1CCAED33_3E69_435E_B522_DAE3B9E63F9D#96057] Batched: true, DataFilters: [(COL_1ADA21FB_DB08_4A1F_986E_E8E363B14F38#96054 <=> A)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/00_A_LIGNES_RSF_345253], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_1ADA21FB_DB08_4A1F_986E_E8E363B14F38,A)], ReadSchema: struct<COL_1ADA21FB_DB08_4A1F_986E_E8E363B14F38:string,COL_1CCAED33_3E69_435E_B522_DAE3B9E63F9D:s...
Project [COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#95995 AS 00_C_PASSAGE_PA_345648_COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#96370]
Filter (COL_6B340AEB_6908_4A50_8AD6_FE561D33FBE0#96011 <=> 1)
WholeStageCodegen (5)
Window [row_number() windowspecdefinition(COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#95995, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_6B340AEB_6908_4A50_8AD6_FE561D33FBE0#96011], [COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#95995], [1 DESC NULLS LAST]
Sort [COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#95995 ASC NULLS FIRST, 1 DESC NULLS LAST], false, 0
WholeStageCodegen (4)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#95995, 10), ENSURE_REQUIREMENTS, [plan_id=25222]
Project [Numéro Séjour#95874 AS COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#95995]
Filter (NOT CASE WHEN isnull(StartsWith(Numéro Séjour#95874, #)) THEN false ELSE StartsWith(Numéro Séjour#95874, #) END AND isnotnull(Numéro Séjour#95874))
WholeStageCodegen (2)
FileScan csv [Numéro Séjour#95874] Batched: false, DataFilters: [NOT CASE WHEN isnull(StartsWith(Numéro Séjour#95874, #)) THEN false ELSE StartsWith(Numéro Séjou..., Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/02_PASSAGE_PATI_344985/passage_pationt_..., PartitionFilters: [], PushedFilters: [IsNotNull(Numéro Séjour)], ReadSchema: struct<Numéro Séjour:string>