From d24281c3d462e067bdd8219f67f7704ba1411508 Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 10 Jul 2026 10:47:04 -0700 Subject: [PATCH] [TTS] Update multi-language eval report format Signed-off-by: Ryan --- examples/tts/magpietts_inference.py | 8 +++-- scripts/tts_comparison_report/README.md | 4 +-- .../reporting/constants.py | 35 +++++++++++++------ .../reporting/helpers.py | 4 +-- .../reporting/orchestrator.py | 9 +++-- .../templates/audio_report.jinja | 2 +- .../templates/eval_report.jinja | 2 +- 7 files changed, 43 insertions(+), 21 deletions(-) diff --git a/examples/tts/magpietts_inference.py b/examples/tts/magpietts_inference.py index b92236f1c6c4..81bf9023cd0b 100644 --- a/examples/tts/magpietts_inference.py +++ b/examples/tts/magpietts_inference.py @@ -216,7 +216,9 @@ def run_inference_and_evaluation( violin_plot_metrics.remove('utmosv2') # Build full checkpoint identifier (include MoE info if present) - full_checkpoint_name = f"{checkpoint_name}_{moe_info}{inference_config.build_identifier()}_SV_{eval_config.sv_model}_{eval_config.language}" + full_checkpoint_name = ( + f"{checkpoint_name}_{moe_info}{inference_config.build_identifier()}_SV_{eval_config.sv_model}" + ) # Tracking metrics across datasets ssim_per_dataset = [] @@ -261,7 +263,7 @@ def run_inference_and_evaluation( } # Setup output directories - eval_dir = os.path.join(out_dir, f"{full_checkpoint_name}_{dataset}") + eval_dir = os.path.join(out_dir, f"{full_checkpoint_name}_{language}_{dataset}") audio_dir = os.path.join(eval_dir, "audio") os.makedirs(eval_dir, exist_ok=True) @@ -341,7 +343,7 @@ def run_inference_and_evaluation( sorted_filewise = sorted(filewise_metrics, key=lambda x: x.get('cer', 0), reverse=True) with open(os.path.join(eval_dir, f"{dataset}_filewise_metrics_{repeat_idx}.json"), "w") as f: - json.dump(sorted_filewise, f, indent=4) + json.dump(sorted_filewise, f, indent=4, ensure_ascii=False) # Append to per-run CSV append_metrics_to_csv(per_run_csv, full_checkpoint_name, dataset, metrics) diff --git a/scripts/tts_comparison_report/README.md b/scripts/tts_comparison_report/README.md index a234c721a49a..807ff4f1be90 100644 --- a/scripts/tts_comparison_report/README.md +++ b/scripts/tts_comparison_report/README.md @@ -187,8 +187,8 @@ The default expiration time is one year. - The expiration time is also included as a suffix in the uploaded artifacts directory name, using the format `%Y-%m-%dT%H-%M-%SZ`, so uploaded reports can be filtered and deleted later if needed. -- Both generated reports include a clickable Jira link derived from `--task_id`. -If no task ID is specified, the link points to the Jira project page. +- Both generated reports include a clickable POR link derived from `--task_id`. +If no task ID is specified, the link points to the POR project page. ## Maintenance diff --git a/scripts/tts_comparison_report/reporting/constants.py b/scripts/tts_comparison_report/reporting/constants.py index d69c96855d5a..0c6c7239cdc3 100644 --- a/scripts/tts_comparison_report/reporting/constants.py +++ b/scripts/tts_comparison_report/reporting/constants.py @@ -17,15 +17,30 @@ _ROOT: Path = Path(__file__).parent.parent # Benchmark names supported by the comparison report pipeline. -SUPPORTED_BENCHMARK_NAMES: list[str] = [ - "libritts_seen", - "libritts_test_clean", - "riva_hard_digits", - "riva_hard_letters", - "riva_hard_money", - "riva_hard_short", - "vctk", -] +BENCHMARK_META = { + 'libritts': 'en', + 'riva_en': 'en', + 'riva_en_hard_sentences': 'en', + 'riva_en_short_sentences': 'en', + 'riva_en_qa': 'en', + 'riva_en_qa_longform': 'en', + 'King_ASR_sa_diacritics': 'ar', + 'King_ASR_sa_no_diacritics': 'ar', + 'King_ASR_uae_diacritics': 'ar', + 'King_ASR_uae_no_diacritics': 'ar', + 'cmltts_de': 'de', + 'cmltts_es': 'es', + 'cmltts_fr': 'fr', + 'AI4bharat': 'hi', + 'cmltts_it': 'it', + 'jvs_jsut': 'ja', + 'F5I9N7A1': 'ko', + 'cmltts_pt': 'pt', + 'vivos': 'vi', + 'mscenespeech': 'zh', +} + +SUPPORTED_BENCHMARK_NAMES = BENCHMARK_META.keys() # Default width of tqdm progress bars in terminal columns. TQDM_NCOLS: int = 80 @@ -52,4 +67,4 @@ DUMMY_TASK_ID: str = "NEMOTTS-0000" # URL prefix used to construct clickable Jira ticket links in reports. -JIRA_TICKET_URL_PREFIX: str = "https://jirasw.nvidia.com/browse" +TICKET_URL_PREFIX: str = "https://modelpor.ideas.aha.io/ideas" diff --git a/scripts/tts_comparison_report/reporting/helpers.py b/scripts/tts_comparison_report/reporting/helpers.py index 671c2a8444ff..d0455acddf4e 100644 --- a/scripts/tts_comparison_report/reporting/helpers.py +++ b/scripts/tts_comparison_report/reporting/helpers.py @@ -14,7 +14,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path -from scripts.tts_comparison_report.reporting.constants import DUMMY_TASK_ID, JIRA_TICKET_URL_PREFIX +from scripts.tts_comparison_report.reporting.constants import DUMMY_TASK_ID, TICKET_URL_PREFIX from scripts.tts_comparison_report.reporting.models import ExpirationInfo, TaskInfo @@ -46,7 +46,7 @@ def make_task_info(task_id: str) -> TaskInfo: Task information with the original task ID, derived Jira ID, and Jira URL. """ jira_id = task_id if task_id != DUMMY_TASK_ID else task_id.split("-")[0] - jira_url = f"{JIRA_TICKET_URL_PREFIX}/{jira_id}" + jira_url = f"{TICKET_URL_PREFIX}/{jira_id}" return TaskInfo( task_id=task_id, diff --git a/scripts/tts_comparison_report/reporting/orchestrator.py b/scripts/tts_comparison_report/reporting/orchestrator.py index 341c4c5ff4e7..23eb2ed81c94 100644 --- a/scripts/tts_comparison_report/reporting/orchestrator.py +++ b/scripts/tts_comparison_report/reporting/orchestrator.py @@ -22,6 +22,7 @@ prepare_eval_artifacts, ) from scripts.tts_comparison_report.reporting.constants import ( + BENCHMARK_META, S3_AUDIO_DIR, S3_IMAGES_DIR, S3_LINK_EXPIRES_IN, @@ -268,7 +269,9 @@ def _render_audio_report( pair_blocks=pair_blocks, ) benchmark_blocks.append(block) - benchmark_section_info.append((benchmark_name, benchmark_name)) + benchmark_language = BENCHMARK_META[benchmark_name] + name_info = f"{benchmark_name} ({benchmark_language})" + benchmark_section_info.append((benchmark_name, name_info)) report = self.renderer.render( name=TemplateName.audio_report, @@ -370,7 +373,9 @@ def _render_eval_report( image_block=image_block, ) benchmark_blocks.append(block) - benchmark_section_info.append((benchmark_name, benchmark_name)) + benchmark_language = BENCHMARK_META[benchmark_name] + name_info = f"{benchmark_name} ({benchmark_language})" + benchmark_section_info.append((benchmark_name, name_info)) report = self.renderer.render( name=TemplateName.eval_report, diff --git a/scripts/tts_comparison_report/templates/audio_report.jinja b/scripts/tts_comparison_report/templates/audio_report.jinja index 617452cdeb16..2d63908e481b 100644 --- a/scripts/tts_comparison_report/templates/audio_report.jinja +++ b/scripts/tts_comparison_report/templates/audio_report.jinja @@ -281,7 +281,7 @@

Links ↗

diff --git a/scripts/tts_comparison_report/templates/eval_report.jinja b/scripts/tts_comparison_report/templates/eval_report.jinja index 945476abd9d1..425e207f7122 100644 --- a/scripts/tts_comparison_report/templates/eval_report.jinja +++ b/scripts/tts_comparison_report/templates/eval_report.jinja @@ -384,7 +384,7 @@

Links ↗