SP1 - Details for Query 507

Submitted Time: 2026/02/23 14:47:18
Duration: 0.3 s
Succeeded Jobs: 679

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" TakeOrderedAndProject "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n235 ms (4 ms, 8 ms, 106 ms (stage 892.0: task 1205))"; 2 [labelType="html" label=" Project "]; 3 [labelType="html" label="Filter number of output rows: 1,631"]; 4 [labelType="html" label="ColumnarToRow number of output rows: 60,000 number of input batches: 15"]; } 5 [labelType="html" label="Scan parquet number of files read: 7 scan time total (min, med, max (stageId: taskId)) 63 ms (4 ms, 5 ms, 25 ms (stage 892.0: task 1205)) metadata time: 0 ms size of files read: 11.5 MiB number of output rows: 60,000"]; 2->0; 3->2; 4->3; 5->4; }

TakeOrderedAndProject(limit=500000, orderBy=[COL_8E27D2F3_0BFB_4247_B8CA_9E75BCDFCB1A#144191 DESC NULLS LAST], output=[COL_3D869D6E_882C_458B_8BF2_23F7EC16C280#144184,COL_D526C61D_BD06_4682_B4FE_C4C5D9D46273#144187,COL_36AAF550_B3A4_4A16_9465_4065A3B49599#144188,COL_8B924A7C_EC45_4472_8111_ED463DBFD9A6#144186,COL_8E27D2F3_0BFB_4247_B8CA_9E75BCDFCB1A#144191,COL_F6F5D4D3_F632_4F46_BEE6_544E3926EA8B#144192])

Project [COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39#144044 AS COL_3D869D6E_882C_458B_8BF2_23F7EC16C280#144184, COL_CD664456_DB8C_4BED_905A_70756B23954B#144046 AS COL_8B924A7C_EC45_4472_8111_ED463DBFD9A6#144186, COL_F233F4CF_34B5_4221_B542_E316610EDF54#144047 AS COL_D526C61D_BD06_4682_B4FE_C4C5D9D46273#144187, COL_EEB1E99F_E345_4082_89A6_7A8693991684#144048 AS COL_36AAF550_B3A4_4A16_9465_4065A3B49599#144188, COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051 AS COL_8E27D2F3_0BFB_4247_B8CA_9E75BCDFCB1A#144191, COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052 AS COL_F6F5D4D3_F632_4F46_BEE6_544E3926EA8B#144192]

Filter ((((COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB#144043 <=> 350000055) AND (COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052 <=> 2025-07)) AND isnotnull(COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051)) AND CASE WHEN isnull(StartsWith(COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051, Inférieur)) THEN false ELSE StartsWith(COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051, Inférieur) END)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB#144043,COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39#144044,COL_CD664456_DB8C_4BED_905A_70756B23954B#144046,COL_F233F4CF_34B5_4221_B542_E316610EDF54#144047,COL_EEB1E99F_E345_4082_89A6_7A8693991684#144048,COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051,COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052] Batched: true, DataFilters: [(COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB#144043 <=> 350000055), (COL_C64DEE10_05C3_46E2_953B_1F..., Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/1_D_2_CMR_CASE__307277], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB,350000055), EqualNullSafe(COL_C64DEE10_05..., ReadSchema: struct<COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB:string,COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39:s...

Details

== Physical Plan ==
TakeOrderedAndProject (5)
+- * Project (4)
   +- * Filter (3)
      +- * ColumnarToRow (2)
         +- Scan parquet  (1)


(1) Scan parquet 
Output [7]: [COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB#144043, COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39#144044, COL_CD664456_DB8C_4BED_905A_70756B23954B#144046, COL_F233F4CF_34B5_4221_B542_E316610EDF54#144047, COL_EEB1E99F_E345_4082_89A6_7A8693991684#144048, COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051, COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/1_D_2_CMR_CASE__307277]
PushedFilters: [EqualNullSafe(COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB,350000055), EqualNullSafe(COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9,2025-07), IsNotNull(COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9)]
ReadSchema: struct<COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB:string,COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39:string,COL_CD664456_DB8C_4BED_905A_70756B23954B:double,COL_F233F4CF_34B5_4221_B542_E316610EDF54:double,COL_EEB1E99F_E345_4082_89A6_7A8693991684:double,COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9:string,COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9:string>

(2) ColumnarToRow [codegen id : 1]
Input [7]: [COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB#144043, COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39#144044, COL_CD664456_DB8C_4BED_905A_70756B23954B#144046, COL_F233F4CF_34B5_4221_B542_E316610EDF54#144047, COL_EEB1E99F_E345_4082_89A6_7A8693991684#144048, COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051, COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052]

(3) Filter [codegen id : 1]
Input [7]: [COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB#144043, COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39#144044, COL_CD664456_DB8C_4BED_905A_70756B23954B#144046, COL_F233F4CF_34B5_4221_B542_E316610EDF54#144047, COL_EEB1E99F_E345_4082_89A6_7A8693991684#144048, COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051, COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052]
Condition : ((((COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB#144043 <=> 350000055) AND (COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052 <=> 2025-07)) AND isnotnull(COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051)) AND CASE WHEN isnull(StartsWith(COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051, Inférieur)) THEN false ELSE StartsWith(COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051, Inférieur) END)

(4) Project [codegen id : 1]
Output [6]: [COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39#144044 AS COL_3D869D6E_882C_458B_8BF2_23F7EC16C280#144184, COL_CD664456_DB8C_4BED_905A_70756B23954B#144046 AS COL_8B924A7C_EC45_4472_8111_ED463DBFD9A6#144186, COL_F233F4CF_34B5_4221_B542_E316610EDF54#144047 AS COL_D526C61D_BD06_4682_B4FE_C4C5D9D46273#144187, COL_EEB1E99F_E345_4082_89A6_7A8693991684#144048 AS COL_36AAF550_B3A4_4A16_9465_4065A3B49599#144188, COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051 AS COL_8E27D2F3_0BFB_4247_B8CA_9E75BCDFCB1A#144191, COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052 AS COL_F6F5D4D3_F632_4F46_BEE6_544E3926EA8B#144192]
Input [7]: [COL_36BA36BA_D1CC_4770_ABD4_69C675D079EB#144043, COL_637AA126_E43A_4B72_A7C5_6B8D1201BA39#144044, COL_CD664456_DB8C_4BED_905A_70756B23954B#144046, COL_F233F4CF_34B5_4221_B542_E316610EDF54#144047, COL_EEB1E99F_E345_4082_89A6_7A8693991684#144048, COL_A23DE8F0_C7A0_498D_AEB3_2AB5429550E9#144051, COL_C64DEE10_05C3_46E2_953B_1F903CDFA5C9#144052]

(5) TakeOrderedAndProject
Input [6]: [COL_3D869D6E_882C_458B_8BF2_23F7EC16C280#144184, COL_8B924A7C_EC45_4472_8111_ED463DBFD9A6#144186, COL_D526C61D_BD06_4682_B4FE_C4C5D9D46273#144187, COL_36AAF550_B3A4_4A16_9465_4065A3B49599#144188, COL_8E27D2F3_0BFB_4247_B8CA_9E75BCDFCB1A#144191, COL_F6F5D4D3_F632_4F46_BEE6_544E3926EA8B#144192]
Arguments: 500000, [COL_8E27D2F3_0BFB_4247_B8CA_9E75BCDFCB1A#144191 DESC NULLS LAST], [COL_3D869D6E_882C_458B_8BF2_23F7EC16C280#144184, COL_D526C61D_BD06_4682_B4FE_C4C5D9D46273#144187, COL_36AAF550_B3A4_4A16_9465_4065A3B49599#144188, COL_8B924A7C_EC45_4472_8111_ED463DBFD9A6#144186, COL_8E27D2F3_0BFB_4247_B8CA_9E75BCDFCB1A#144191, COL_F6F5D4D3_F632_4F46_BEE6_544E3926EA8B#144192]

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet