diff --git a/examples/tts/magpietts_inference.py b/examples/tts/magpietts_inference.py
index b92236f1c6c4..2169050e3006 100644
--- a/examples/tts/magpietts_inference.py
+++ b/examples/tts/magpietts_inference.py
@@ -216,7 +216,9 @@ def run_inference_and_evaluation(
violin_plot_metrics.remove('utmosv2')
# Build full checkpoint identifier (include MoE info if present)
- full_checkpoint_name = f"{checkpoint_name}_{moe_info}{inference_config.build_identifier()}_SV_{eval_config.sv_model}_{eval_config.language}"
+ full_checkpoint_name = (
+ f"{checkpoint_name}_{moe_info}{inference_config.build_identifier()}_SV_{eval_config.sv_model}"
+ )
# Tracking metrics across datasets
ssim_per_dataset = []
@@ -261,7 +263,7 @@ def run_inference_and_evaluation(
}
# Setup output directories
- eval_dir = os.path.join(out_dir, f"{full_checkpoint_name}_{dataset}")
+ eval_dir = os.path.join(out_dir, f"{full_checkpoint_name}_{language}_{dataset}")
audio_dir = os.path.join(eval_dir, "audio")
os.makedirs(eval_dir, exist_ok=True)
@@ -336,12 +338,14 @@ def run_inference_and_evaluation(
filewise_metrics_all_repeats.extend(filewise_metrics)
# Save metrics
- with open(os.path.join(eval_dir, f"{dataset}_metrics_{repeat_idx}.json"), "w") as f:
+ metrics_path = os.path.join(eval_dir, f"{dataset}_metrics_{repeat_idx}.json")
+ with open(metrics_path, "w") as f:
json.dump(metrics, f, indent=4)
sorted_filewise = sorted(filewise_metrics, key=lambda x: x.get('cer', 0), reverse=True)
- with open(os.path.join(eval_dir, f"{dataset}_filewise_metrics_{repeat_idx}.json"), "w") as f:
- json.dump(sorted_filewise, f, indent=4)
+ filewise_metrics_path = os.path.join(eval_dir, f"{dataset}_filewise_metrics_{repeat_idx}.json")
+ with open(filewise_metrics_path, "w", encoding="utf-8") as f:
+ json.dump(sorted_filewise, f, indent=4, ensure_ascii=False)
# Append to per-run CSV
append_metrics_to_csv(per_run_csv, full_checkpoint_name, dataset, metrics)
diff --git a/scripts/tts_comparison_report/README.md b/scripts/tts_comparison_report/README.md
index a234c721a49a..e675b486e918 100644
--- a/scripts/tts_comparison_report/README.md
+++ b/scripts/tts_comparison_report/README.md
@@ -127,7 +127,7 @@ python scripts/tts_comparison_report/generate_report.py \
--s3_region us-west-2 \
--task_id NEMOTTS-2007 \
--audio_report \
- --audio_report_benchmarks libritts_test_clean,riva_hard_digits \
+ --audio_report_benchmarks libritts,riva_en \
--samples_per_benchmark 20
```
@@ -145,7 +145,7 @@ python scripts/tts_comparison_report/generate_report.py \
--s3_region us-west-2 \
--task_id NEMOTTS-2007 \
--audio_report \
- --audio_report_benchmarks libritts_test_clean,riva_hard_digits \
+ --audio_report_benchmarks libritts,riva_en \
--samples_per_benchmark 20 \
--remote_hostname your_remote_host \
--remote_username your_user
@@ -160,13 +160,13 @@ python scripts/tts_comparison_report/generate_report.py \
--baseline_path /workspace/NeMo/exp/buckets/baseline \
--candidate_name "Model B" \
--candidate_path /workspace/NeMo/exp/buckets/candidate \
- --benchmarks libritts_test_clean,riva_hard_digits,riva_hard_letters \
+ --benchmarks libritts,riva_en,riva_en_hard_sentences \
--s3_endpoint https://your-s3-endpoint \
--s3_bucket your_bucket_name \
--s3_region us-west-2 \
--task_id NEMOTTS-2007 \
--audio_report \
- --audio_report_benchmarks libritts_test_clean,riva_hard_digits \
+ --audio_report_benchmarks libritts,riva_en \
--samples_per_benchmark 20
```
@@ -187,8 +187,8 @@ The default expiration time is one year.
- The expiration time is also included as a suffix in the uploaded artifacts
directory name, using the format `%Y-%m-%dT%H-%M-%SZ`, so uploaded reports
can be filtered and deleted later if needed.
-- Both generated reports include a clickable Jira link derived from `--task_id`.
-If no task ID is specified, the link points to the Jira project page.
+- Both generated reports include a clickable POR link derived from `--task_id`.
+If no task ID is specified, the link points to the POR project page.
## Maintenance
diff --git a/scripts/tts_comparison_report/generate_report.py b/scripts/tts_comparison_report/generate_report.py
index 287731e29919..c6bfeee35109 100644
--- a/scripts/tts_comparison_report/generate_report.py
+++ b/scripts/tts_comparison_report/generate_report.py
@@ -41,7 +41,7 @@
_S3_ACCESS_KEY_ID: str = "S3_ACCESS_KEY_ID"
_S3_SECRET_ACCESS_KEY: str = "S3_SECRET_ACCESS_KEY"
-_DEFAULT_BENCHMARK_NAMES: str = ",".join(SUPPORTED_BENCHMARK_NAMES)
+_DEFAULT_BENCHMARK_NAMES: str = "libritts,riva_en"
def _create_argparser() -> ArgumentParser:
@@ -113,7 +113,7 @@ def _create_argparser() -> ArgumentParser:
"--task_id",
type=str,
default=DUMMY_TASK_ID,
- help="Jira task number associated with this report.",
+ help="Task ID associated with this report.",
)
parser.add_argument(
"--results_subdir",
@@ -129,7 +129,7 @@ def _create_argparser() -> ArgumentParser:
parser.add_argument(
"--audio_report_benchmarks",
type=str,
- default="libritts_test_clean,riva_hard_digits,riva_hard_letters",
+ default=_DEFAULT_BENCHMARK_NAMES,
help="Comma-separated list of benchmarks to include in the audio report.",
)
parser.add_argument(
diff --git a/scripts/tts_comparison_report/reporting/constants.py b/scripts/tts_comparison_report/reporting/constants.py
index d69c96855d5a..67d1b7b837f4 100644
--- a/scripts/tts_comparison_report/reporting/constants.py
+++ b/scripts/tts_comparison_report/reporting/constants.py
@@ -17,15 +17,30 @@
_ROOT: Path = Path(__file__).parent.parent
# Benchmark names supported by the comparison report pipeline.
-SUPPORTED_BENCHMARK_NAMES: list[str] = [
- "libritts_seen",
- "libritts_test_clean",
- "riva_hard_digits",
- "riva_hard_letters",
- "riva_hard_money",
- "riva_hard_short",
- "vctk",
-]
+BENCHMARK_META: dict[str, str] = {
+ 'libritts': 'en',
+ 'riva_en': 'en',
+ 'riva_en_hard_sentences': 'en',
+ 'riva_en_short_sentences': 'en',
+ 'riva_en_qa': 'en',
+ 'riva_en_qa_longform': 'en',
+ 'King_ASR_sa_diacritics': 'ar',
+ 'King_ASR_sa_no_diacritics': 'ar',
+ 'King_ASR_uae_diacritics': 'ar',
+ 'King_ASR_uae_no_diacritics': 'ar',
+ 'cmltts_de': 'de',
+ 'cmltts_es': 'es',
+ 'cmltts_fr': 'fr',
+ 'AI4bharat': 'hi',
+ 'cmltts_it': 'it',
+ 'jvs_jsut': 'ja',
+ 'F5I9N7A1': 'ko',
+ 'cmltts_pt': 'pt',
+ 'vivos': 'vi',
+ 'mscenespeech': 'zh',
+}
+
+SUPPORTED_BENCHMARK_NAMES: list[str] = list(BENCHMARK_META.keys())
# Default width of tqdm progress bars in terminal columns.
TQDM_NCOLS: int = 80
@@ -48,8 +63,8 @@
# Directory containing Jinja templates used for report rendering.
TEMPLATES_DIR: Path = _ROOT / "templates"
-# Fallback task id used when no real Jira ticket is provided.
-DUMMY_TASK_ID: str = "NEMOTTS-0000"
+# Fallback task id used when no real ticket is provided.
+DUMMY_TASK_ID: str = "NMP-I-000"
-# URL prefix used to construct clickable Jira ticket links in reports.
-JIRA_TICKET_URL_PREFIX: str = "https://jirasw.nvidia.com/browse"
+# URL prefix used to construct clickable ticket links in reports.
+TICKET_URL_PREFIX: str = "https://modelpor.ideas.aha.io/ideas"
diff --git a/scripts/tts_comparison_report/reporting/helpers.py b/scripts/tts_comparison_report/reporting/helpers.py
index 671c2a8444ff..1b4ba41e26c4 100644
--- a/scripts/tts_comparison_report/reporting/helpers.py
+++ b/scripts/tts_comparison_report/reporting/helpers.py
@@ -14,7 +14,7 @@
from datetime import UTC, datetime, timedelta
from pathlib import Path
-from scripts.tts_comparison_report.reporting.constants import DUMMY_TASK_ID, JIRA_TICKET_URL_PREFIX
+from scripts.tts_comparison_report.reporting.constants import TICKET_URL_PREFIX
from scripts.tts_comparison_report.reporting.models import ExpirationInfo, TaskInfo
@@ -37,21 +37,19 @@ def make_expiration_info(expires_in: int) -> ExpirationInfo:
def make_task_info(task_id: str) -> TaskInfo:
- """Create task metadata and the corresponding Jira link information.
+ """Create task metadata and the corresponding link information.
Args:
- task_id: Jira task identifier used for the report.
+ task_id: Task identifier used for the report.
Returns:
- Task information with the original task ID, derived Jira ID, and Jira URL.
+ Task information with the original task ID and task URL.
"""
- jira_id = task_id if task_id != DUMMY_TASK_ID else task_id.split("-")[0]
- jira_url = f"{JIRA_TICKET_URL_PREFIX}/{jira_id}"
+ task_url = f"{TICKET_URL_PREFIX}/{task_id}"
return TaskInfo(
task_id=task_id,
- jira_id=jira_id,
- jira_url=jira_url,
+ task_url=task_url,
)
diff --git a/scripts/tts_comparison_report/reporting/models.py b/scripts/tts_comparison_report/reporting/models.py
index 51ea36190d45..a1936cc61157 100644
--- a/scripts/tts_comparison_report/reporting/models.py
+++ b/scripts/tts_comparison_report/reporting/models.py
@@ -535,11 +535,10 @@ def get_benchmark_sample_meta(
@dataclass(frozen=True)
class TaskInfo:
- """Task identifiers and derived Jira link information used in reports."""
+ """Task identifiers and derived link information used in reports."""
task_id: str
- jira_id: str
- jira_url: str
+ task_url: str
@dataclass(frozen=True)
diff --git a/scripts/tts_comparison_report/reporting/orchestrator.py b/scripts/tts_comparison_report/reporting/orchestrator.py
index 341c4c5ff4e7..0e43fff791ab 100644
--- a/scripts/tts_comparison_report/reporting/orchestrator.py
+++ b/scripts/tts_comparison_report/reporting/orchestrator.py
@@ -22,6 +22,7 @@
prepare_eval_artifacts,
)
from scripts.tts_comparison_report.reporting.constants import (
+ BENCHMARK_META,
S3_AUDIO_DIR,
S3_IMAGES_DIR,
S3_LINK_EXPIRES_IN,
@@ -268,12 +269,14 @@ def _render_audio_report(
pair_blocks=pair_blocks,
)
benchmark_blocks.append(block)
- benchmark_section_info.append((benchmark_name, benchmark_name))
+ benchmark_language = BENCHMARK_META[benchmark_name]
+ name_info = f"{benchmark_name} ({benchmark_language})"
+ benchmark_section_info.append((benchmark_name, name_info))
report = self.renderer.render(
name=TemplateName.audio_report,
- jira_id=task_info.jira_id,
- jira_url=task_info.jira_url,
+ task_id=task_info.task_id,
+ task_url=task_info.task_url,
header_block=header_block,
benchmark_blocks=benchmark_blocks,
benchmark_section_info=benchmark_section_info,
@@ -370,13 +373,15 @@ def _render_eval_report(
image_block=image_block,
)
benchmark_blocks.append(block)
- benchmark_section_info.append((benchmark_name, benchmark_name))
+ benchmark_language = BENCHMARK_META[benchmark_name]
+ name_info = f"{benchmark_name} ({benchmark_language})"
+ benchmark_section_info.append((benchmark_name, name_info))
report = self.renderer.render(
name=TemplateName.eval_report,
is_self_comparison=eval_artifacts.is_self_comparison,
- jira_id=task_info.jira_id,
- jira_url=task_info.jira_url,
+ task_id=task_info.task_id,
+ task_url=task_info.task_url,
audio_report_url=audio_report_url,
configuration_block=configuration_block,
header_block=header_block,
@@ -414,7 +419,7 @@ def run(
generate_audio_report: Whether to generate the audio comparison report.
audio_report_benchmarks: Benchmark names to include in the audio report.
samples_per_benchmark: Number of audio pairs to sample per benchmark.
- task_id: Task identifier used for report metadata and Jira linking.
+ task_id: Task identifier used for report metadata and link.
Returns:
Tuple containing the evaluation report URL and the optional audio report URL.
diff --git a/scripts/tts_comparison_report/templates/audio_report.jinja b/scripts/tts_comparison_report/templates/audio_report.jinja
index 617452cdeb16..4e4ceb0f918b 100644
--- a/scripts/tts_comparison_report/templates/audio_report.jinja
+++ b/scripts/tts_comparison_report/templates/audio_report.jinja
@@ -281,7 +281,7 @@
Links ↗
diff --git a/scripts/tts_comparison_report/templates/eval_report.jinja b/scripts/tts_comparison_report/templates/eval_report.jinja
index 945476abd9d1..ebfab54793c4 100644
--- a/scripts/tts_comparison_report/templates/eval_report.jinja
+++ b/scripts/tts_comparison_report/templates/eval_report.jinja
@@ -384,7 +384,7 @@
Links ↗
{% if audio_report_url %}