hdh-spark-job - Details for Query 1071

Submitted Time: 2026/10/23 08:07:52
Duration: 75 ms
Succeeded Jobs: 1082 1083

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: 0 ms"; 2 [labelType="html" label="HashAggregate time in aggregation build: 0 ms number of output rows: 1"]; } 3 [labelType="html" label="Exchange shuffle records written: 2 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 1462.0: task 1315)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 2 local bytes read: 118.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 2 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 32.0 B (16.0 B, 16.0 B, 16.0 B (stage 1462.0: task 1315)) local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 118.0 B (59.0 B, 59.0 B, 59.0 B (stage 1462.0: task 1315))"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n13 ms (5 ms, 8 ms, 8 ms (stage 1462.0: task 1315))"; 5 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 12 ms (5 ms, 7 ms, 7 ms (stage 1462.0: task 1315)) number of output rows: 2"]; } 6 [labelType="html" label=" Union "]; subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: 5 ms"; 8 [labelType="html" label="Scan JDBCRelation((SELECT \"ot_numcot\", \"ot_numdem\", \"ot_numdos\", \"ot_numpco\", \"ot_primai\", \"ot_annval\", \"ot_codcha\", \"ot_codbas\", \"ot_rattac\", \"ot_lettre\", \"ot_coeff\", \"ot_cocal\", \"ot_platre\", \"ot_compar\", \"ot_codper\", \"ot_codser\", \"ot_qtebas\", \"ot_cdam\", \"ot_numsal\", \"ot_nomdos\", \"ot_cr\", \"ot_indbas\", \"ot_inftra\", \"ot_factur\", \"ot_monnaie\", \"ot_tarif\", \"ot_numero\", \"ot_t_red\", \"ot_datmaj\", \"ot_codsit\", \"ot_coduti\", \"ot_flrmo\", \"ot_nuit\", \"ot_majonuit\", \"ot_ferie\", \"ot_majoferie\", \"ot_flald\", \"ot_typexo\", \"ot_avisexo\", \"ot_basrmb\", \"ot_taucai\", \"ot_parcai\", \"ot_flid\", \"ot_mttid\", \"ot_tauid\", \"ot_amoid\", \"ot_mttik\", \"ot_tauik\", \"ot_amoik\", \"ot_nbrikm\", \"ot_mttikm\", \"ot_tauikm\", \"ot_amoikm\", \"ot_mttiks\", \"ot_tauiks\", \"ot_amoiks\", \"ot_taumut\", \"ot_parmut\", \"ot_nbriks\", \"ot_nbrik\", \"ot_numord\", \"ot_ligrgr\", \"ot_fldivise\", \"ot_codpre\", \"ot_majoage\", \"ot_faitlit\", \"ot_qualdp\", \"ot_priuni\", \"ot_denomb\", \"ot_amcid\", \"ot_codlis\", \"ot_coditm\", \"ot_numtli\", \"ot_depmnt\", \"ot_forc\", \"ot_formamc\", \"ot_numcot_ccam\", \"ot_coduam\", \"ot_mtdep\", \"ot_mtpenhps\", \"ot_mttori\", \"ot_codper2\", \"ot_letads\", \"ot_later\", \"ot_numexa\", \"ot_codapp\", \"ot_codper3\", \"ot_codtit\" FROM \"xplore\".\"h_cotdet\") as x) [numPartitions=1] number of output rows: 50,890"]; } subgraph cluster9 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 8 ms"; 10 [labelType="html" label="Scan JDBCRelation((SELECT \"ot_numcot\", \"ot_numdem\", \"ot_numdos\", \"ot_numpco\", \"ot_primai\", \"ot_annval\", \"ot_codcha\", \"ot_codbas\", \"ot_rattac\", \"ot_lettre\", \"ot_coeff\", \"ot_cocal\", \"ot_platre\", \"ot_compar\", \"ot_codper\", \"ot_codser\", \"ot_qtebas\", \"ot_cdam\", \"ot_numsal\", \"ot_nomdos\", \"ot_cr\", \"ot_indbas\", \"ot_inftra\", \"ot_factur\", \"ot_monnaie\", \"ot_tarif\", \"ot_numero\", \"ot_t_red\", \"ot_datmaj\", \"ot_codsit\", \"ot_coduti\", \"ot_flrmo\", \"ot_nuit\", \"ot_majonuit\", \"ot_ferie\", \"ot_majoferie\", \"ot_flald\", \"ot_typexo\", \"ot_avisexo\", \"ot_basrmb\", \"ot_taucai\", \"ot_parcai\", \"ot_flid\", \"ot_mttid\", \"ot_tauid\", \"ot_amoid\", \"ot_mttik\", \"ot_tauik\", \"ot_amoik\", \"ot_nbrikm\", \"ot_mttikm\", \"ot_tauikm\", \"ot_amoikm\", \"ot_mttiks\", \"ot_tauiks\", \"ot_amoiks\", \"ot_taumut\", \"ot_parmut\", \"ot_nbriks\", \"ot_nbrik\", \"ot_numord\", \"ot_ligrgr\", \"ot_fldivise\", \"ot_codpre\", \"ot_majoage\", \"ot_faitlit\", \"ot_qualdp\", \"ot_priuni\", \"ot_denomb\", \"ot_amcid\", \"ot_codlis\", \"ot_coditm\", \"ot_numtli\", \"ot_depmnt\", \"ot_forc\", \"ot_formamc\", \"ot_numcot_ccam\", \"ot_coduam\", \"ot_mtdep\", \"ot_mtpenhps\", \"ot_mttori\", \"ot_codper2\", \"ot_letads\", \"ot_later\", \"ot_numexa\", \"ot_codapp\", \"ot_codper3\", \"ot_codtit\" FROM \"xplore2\".\"h_cotdet\") as x) [numPartitions=1] number of output rows: 50,890"]; } 2->0; 3->2; 5->3; 6->5; 8->6; 10->6; }

AdaptiveSparkPlan isFinalPlan=true

HashAggregate(keys=[], functions=[count(1)])

WholeStageCodegen (4)

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=23267]

HashAggregate(keys=[], functions=[partial_count(1)])

WholeStageCodegen (3)

Union

Scan JDBCRelation((SELECT "ot_numcot", "ot_numdem", "ot_numdos", "ot_numpco", "ot_primai", "ot_annval", "ot_codcha", "ot_codbas", "ot_rattac", "ot_lettre", "ot_coeff", "ot_cocal", "ot_platre", "ot_compar", "ot_codper", "ot_codser", "ot_qtebas", "ot_cdam", "ot_numsal", "ot_nomdos", "ot_cr", "ot_indbas", "ot_inftra", "ot_factur", "ot_monnaie", "ot_tarif", "ot_numero", "ot_t_red", "ot_datmaj", "ot_codsit", "ot_coduti", "ot_flrmo", "ot_nuit", "ot_majonuit", "ot_ferie", "ot_majoferie", "ot_flald", "ot_typexo", "ot_avisexo", "ot_basrmb", "ot_taucai", "ot_parcai", "ot_flid", "ot_mttid", "ot_tauid", "ot_amoid", "ot_mttik", "ot_tauik", "ot_amoik", "ot_nbrikm", "ot_mttikm", "ot_tauikm", "ot_amoikm", "ot_mttiks", "ot_tauiks", "ot_amoiks", "ot_taumut", "ot_parmut", "ot_nbriks", "ot_nbrik", "ot_numord", "ot_ligrgr", "ot_fldivise", "ot_codpre", "ot_majoage", "ot_faitlit", "ot_qualdp", "ot_priuni", "ot_denomb", "ot_amcid", "ot_codlis", "ot_coditm", "ot_numtli", "ot_depmnt", "ot_forc", "ot_formamc", "ot_numcot_ccam", "ot_coduam", "ot_mtdep", "ot_mtpenhps", "ot_mttori", "ot_codper2", "ot_letads", "ot_later", "ot_numexa", "ot_codapp", "ot_codper3", "ot_codtit" FROM "xplore"."h_cotdet") as x) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>

WholeStageCodegen (1)

Scan JDBCRelation((SELECT "ot_numcot", "ot_numdem", "ot_numdos", "ot_numpco", "ot_primai", "ot_annval", "ot_codcha", "ot_codbas", "ot_rattac", "ot_lettre", "ot_coeff", "ot_cocal", "ot_platre", "ot_compar", "ot_codper", "ot_codser", "ot_qtebas", "ot_cdam", "ot_numsal", "ot_nomdos", "ot_cr", "ot_indbas", "ot_inftra", "ot_factur", "ot_monnaie", "ot_tarif", "ot_numero", "ot_t_red", "ot_datmaj", "ot_codsit", "ot_coduti", "ot_flrmo", "ot_nuit", "ot_majonuit", "ot_ferie", "ot_majoferie", "ot_flald", "ot_typexo", "ot_avisexo", "ot_basrmb", "ot_taucai", "ot_parcai", "ot_flid", "ot_mttid", "ot_tauid", "ot_amoid", "ot_mttik", "ot_tauik", "ot_amoik", "ot_nbrikm", "ot_mttikm", "ot_tauikm", "ot_amoikm", "ot_mttiks", "ot_tauiks", "ot_amoiks", "ot_taumut", "ot_parmut", "ot_nbriks", "ot_nbrik", "ot_numord", "ot_ligrgr", "ot_fldivise", "ot_codpre", "ot_majoage", "ot_faitlit", "ot_qualdp", "ot_priuni", "ot_denomb", "ot_amcid", "ot_codlis", "ot_coditm", "ot_numtli", "ot_depmnt", "ot_forc", "ot_formamc", "ot_numcot_ccam", "ot_coduam", "ot_mtdep", "ot_mtpenhps", "ot_mttori", "ot_codper2", "ot_letads", "ot_later", "ot_numexa", "ot_codapp", "ot_codper3", "ot_codtit" FROM "xplore2"."h_cotdet") as x) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>

WholeStageCodegen (2)

Details

== Physical Plan ==
AdaptiveSparkPlan (12)
+- == Final Plan ==
   * HashAggregate (7)
   +- ShuffleQueryStage (6), Statistics(sizeInBytes=32.0 B, rowCount=2)
      +- Exchange (5)
         +- * HashAggregate (4)
            +- Union (3)
               :- * Scan JDBCRelation((SELECT "ot_numcot", "ot_numdem", "ot_numdos", "ot_numpco", "ot_primai", "ot_annval", "ot_codcha", "ot_codbas", "ot_rattac", "ot_lettre", "ot_coeff", "ot_cocal", "ot_platre", "ot_compar", "ot_codper", "ot_codser", "ot_qtebas", "ot_cdam", "ot_numsal", "ot_nomdos", "ot_cr", "ot_indbas", "ot_inftra", "ot_factur", "ot_monnaie", "ot_tarif", "ot_numero", "ot_t_red", "ot_datmaj", "ot_codsit", "ot_coduti", "ot_flrmo", "ot_nuit", "ot_majonuit", "ot_ferie", "ot_majoferie", "ot_flald", "ot_typexo", "ot_avisexo", "ot_basrmb", "ot_taucai", "ot_parcai", "ot_flid", "ot_mttid", "ot_tauid", "ot_amoid", "ot_mttik", "ot_tauik", "ot_amoik", "ot_nbrikm", "ot_mttikm", "ot_tauikm", "ot_amoikm", "ot_mttiks", "ot_tauiks", "ot_amoiks", "ot_taumut", "ot_parmut", "ot_nbriks", "ot_nbrik", "ot_numord", "ot_ligrgr", "ot_fldivise", "ot_codpre", "ot_majoage", "ot_faitlit", "ot_qualdp", "ot_priuni", "ot_denomb", "ot_amcid", "ot_codlis", "ot_coditm", "ot_numtli", "ot_depmnt", "ot_forc", "ot_formamc", "ot_numcot_ccam", "ot_coduam", "ot_mtdep", "ot_mtpenhps", "ot_mttori", "ot_codper2", "ot_letads", "ot_later", "ot_numexa", "ot_codapp", "ot_codper3", "ot_codtit" FROM "xplore"."h_cotdet") as x) [numPartitions=1]  (1)
               +- * Scan JDBCRelation((SELECT "ot_numcot", "ot_numdem", "ot_numdos", "ot_numpco", "ot_primai", "ot_annval", "ot_codcha", "ot_codbas", "ot_rattac", "ot_lettre", "ot_coeff", "ot_cocal", "ot_platre", "ot_compar", "ot_codper", "ot_codser", "ot_qtebas", "ot_cdam", "ot_numsal", "ot_nomdos", "ot_cr", "ot_indbas", "ot_inftra", "ot_factur", "ot_monnaie", "ot_tarif", "ot_numero", "ot_t_red", "ot_datmaj", "ot_codsit", "ot_coduti", "ot_flrmo", "ot_nuit", "ot_majonuit", "ot_ferie", "ot_majoferie", "ot_flald", "ot_typexo", "ot_avisexo", "ot_basrmb", "ot_taucai", "ot_parcai", "ot_flid", "ot_mttid", "ot_tauid", "ot_amoid", "ot_mttik", "ot_tauik", "ot_amoik", "ot_nbrikm", "ot_mttikm", "ot_tauikm", "ot_amoikm", "ot_mttiks", "ot_tauiks", "ot_amoiks", "ot_taumut", "ot_parmut", "ot_nbriks", "ot_nbrik", "ot_numord", "ot_ligrgr", "ot_fldivise", "ot_codpre", "ot_majoage", "ot_faitlit", "ot_qualdp", "ot_priuni", "ot_denomb", "ot_amcid", "ot_codlis", "ot_coditm", "ot_numtli", "ot_depmnt", "ot_forc", "ot_formamc", "ot_numcot_ccam", "ot_coduam", "ot_mtdep", "ot_mtpenhps", "ot_mttori", "ot_codper2", "ot_letads", "ot_later", "ot_numexa", "ot_codapp", "ot_codper3", "ot_codtit" FROM "xplore2"."h_cotdet") as x) [numPartitions=1]  (2)
+- == Initial Plan ==
   HashAggregate (11)
   +- Exchange (10)
      +- HashAggregate (9)
         +- Union (8)
            :- Scan JDBCRelation((SELECT "ot_numcot", "ot_numdem", "ot_numdos", "ot_numpco", "ot_primai", "ot_annval", "ot_codcha", "ot_codbas", "ot_rattac", "ot_lettre", "ot_coeff", "ot_cocal", "ot_platre", "ot_compar", "ot_codper", "ot_codser", "ot_qtebas", "ot_cdam", "ot_numsal", "ot_nomdos", "ot_cr", "ot_indbas", "ot_inftra", "ot_factur", "ot_monnaie", "ot_tarif", "ot_numero", "ot_t_red", "ot_datmaj", "ot_codsit", "ot_coduti", "ot_flrmo", "ot_nuit", "ot_majonuit", "ot_ferie", "ot_majoferie", "ot_flald", "ot_typexo", "ot_avisexo", "ot_basrmb", "ot_taucai", "ot_parcai", "ot_flid", "ot_mttid", "ot_tauid", "ot_amoid", "ot_mttik", "ot_tauik", "ot_amoik", "ot_nbrikm", "ot_mttikm", "ot_tauikm", "ot_amoikm", "ot_mttiks", "ot_tauiks", "ot_amoiks", "ot_taumut", "ot_parmut", "ot_nbriks", "ot_nbrik", "ot_numord", "ot_ligrgr", "ot_fldivise", "ot_codpre", "ot_majoage", "ot_faitlit", "ot_qualdp", "ot_priuni", "ot_denomb", "ot_amcid", "ot_codlis", "ot_coditm", "ot_numtli", "ot_depmnt", "ot_forc", "ot_formamc", "ot_numcot_ccam", "ot_coduam", "ot_mtdep", "ot_mtpenhps", "ot_mttori", "ot_codper2", "ot_letads", "ot_later", "ot_numexa", "ot_codapp", "ot_codper3", "ot_codtit" FROM "xplore"."h_cotdet") as x) [numPartitions=1]  (1)
            +- Scan JDBCRelation((SELECT "ot_numcot", "ot_numdem", "ot_numdos", "ot_numpco", "ot_primai", "ot_annval", "ot_codcha", "ot_codbas", "ot_rattac", "ot_lettre", "ot_coeff", "ot_cocal", "ot_platre", "ot_compar", "ot_codper", "ot_codser", "ot_qtebas", "ot_cdam", "ot_numsal", "ot_nomdos", "ot_cr", "ot_indbas", "ot_inftra", "ot_factur", "ot_monnaie", "ot_tarif", "ot_numero", "ot_t_red", "ot_datmaj", "ot_codsit", "ot_coduti", "ot_flrmo", "ot_nuit", "ot_majonuit", "ot_ferie", "ot_majoferie", "ot_flald", "ot_typexo", "ot_avisexo", "ot_basrmb", "ot_taucai", "ot_parcai", "ot_flid", "ot_mttid", "ot_tauid", "ot_amoid", "ot_mttik", "ot_tauik", "ot_amoik", "ot_nbrikm", "ot_mttikm", "ot_tauikm", "ot_amoikm", "ot_mttiks", "ot_tauiks", "ot_amoiks", "ot_taumut", "ot_parmut", "ot_nbriks", "ot_nbrik", "ot_numord", "ot_ligrgr", "ot_fldivise", "ot_codpre", "ot_majoage", "ot_faitlit", "ot_qualdp", "ot_priuni", "ot_denomb", "ot_amcid", "ot_codlis", "ot_coditm", "ot_numtli", "ot_depmnt", "ot_forc", "ot_formamc", "ot_numcot_ccam", "ot_coduam", "ot_mtdep", "ot_mtpenhps", "ot_mttori", "ot_codper2", "ot_letads", "ot_later", "ot_numexa", "ot_codapp", "ot_codper3", "ot_codtit" FROM "xplore2"."h_cotdet") as x) [numPartitions=1]  (2)


(1) Scan JDBCRelation((SELECT "ot_numcot", "ot_numdem", "ot_numdos", "ot_numpco", "ot_primai", "ot_annval", "ot_codcha", "ot_codbas", "ot_rattac", "ot_lettre", "ot_coeff", "ot_cocal", "ot_platre", "ot_compar", "ot_codper", "ot_codser", "ot_qtebas", "ot_cdam", "ot_numsal", "ot_nomdos", "ot_cr", "ot_indbas", "ot_inftra", "ot_factur", "ot_monnaie", "ot_tarif", "ot_numero", "ot_t_red", "ot_datmaj", "ot_codsit", "ot_coduti", "ot_flrmo", "ot_nuit", "ot_majonuit", "ot_ferie", "ot_majoferie", "ot_flald", "ot_typexo", "ot_avisexo", "ot_basrmb", "ot_taucai", "ot_parcai", "ot_flid", "ot_mttid", "ot_tauid", "ot_amoid", "ot_mttik", "ot_tauik", "ot_amoik", "ot_nbrikm", "ot_mttikm", "ot_tauikm", "ot_amoikm", "ot_mttiks", "ot_tauiks", "ot_amoiks", "ot_taumut", "ot_parmut", "ot_nbriks", "ot_nbrik", "ot_numord", "ot_ligrgr", "ot_fldivise", "ot_codpre", "ot_majoage", "ot_faitlit", "ot_qualdp", "ot_priuni", "ot_denomb", "ot_amcid", "ot_codlis", "ot_coditm", "ot_numtli", "ot_depmnt", "ot_forc", "ot_formamc", "ot_numcot_ccam", "ot_coduam", "ot_mtdep", "ot_mtpenhps", "ot_mttori", "ot_codper2", "ot_letads", "ot_later", "ot_numexa", "ot_codapp", "ot_codper3", "ot_codtit" FROM "xplore"."h_cotdet") as x) [numPartitions=1]  [codegen id : 1]
Output: []
ReadSchema: struct<>

(2) Scan JDBCRelation((SELECT "ot_numcot", "ot_numdem", "ot_numdos", "ot_numpco", "ot_primai", "ot_annval", "ot_codcha", "ot_codbas", "ot_rattac", "ot_lettre", "ot_coeff", "ot_cocal", "ot_platre", "ot_compar", "ot_codper", "ot_codser", "ot_qtebas", "ot_cdam", "ot_numsal", "ot_nomdos", "ot_cr", "ot_indbas", "ot_inftra", "ot_factur", "ot_monnaie", "ot_tarif", "ot_numero", "ot_t_red", "ot_datmaj", "ot_codsit", "ot_coduti", "ot_flrmo", "ot_nuit", "ot_majonuit", "ot_ferie", "ot_majoferie", "ot_flald", "ot_typexo", "ot_avisexo", "ot_basrmb", "ot_taucai", "ot_parcai", "ot_flid", "ot_mttid", "ot_tauid", "ot_amoid", "ot_mttik", "ot_tauik", "ot_amoik", "ot_nbrikm", "ot_mttikm", "ot_tauikm", "ot_amoikm", "ot_mttiks", "ot_tauiks", "ot_amoiks", "ot_taumut", "ot_parmut", "ot_nbriks", "ot_nbrik", "ot_numord", "ot_ligrgr", "ot_fldivise", "ot_codpre", "ot_majoage", "ot_faitlit", "ot_qualdp", "ot_priuni", "ot_denomb", "ot_amcid", "ot_codlis", "ot_coditm", "ot_numtli", "ot_depmnt", "ot_forc", "ot_formamc", "ot_numcot_ccam", "ot_coduam", "ot_mtdep", "ot_mtpenhps", "ot_mttori", "ot_codper2", "ot_letads", "ot_later", "ot_numexa", "ot_codapp", "ot_codper3", "ot_codtit" FROM "xplore2"."h_cotdet") as x) [numPartitions=1]  [codegen id : 2]
Output: []
ReadSchema: struct<>

(3) Union

(4) HashAggregate [codegen id : 3]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#325168L]
Results [1]: [count#325169L]

(5) Exchange
Input [1]: [count#325169L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=23267]

(6) ShuffleQueryStage
Output [1]: [count#325169L]
Arguments: 0

(7) HashAggregate [codegen id : 4]
Input [1]: [count#325169L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#325076L]
Results [1]: [count(1)#325076L AS count#325077L]

(8) Union

(9) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#325168L]
Results [1]: [count#325169L]

(10) Exchange
Input [1]: [count#325169L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=23253]

(11) HashAggregate
Input [1]: [count#325169L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#325076L]
Results [1]: [count(1)#325076L AS count#325077L]

(12) AdaptiveSparkPlan
Output [1]: [count#325077L]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet