diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ab20b80..2bd19d9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,6 +258,12 @@ add_library(engine_runtime STATIC src/models/omnivoice/assets.cpp src/models/omnivoice/tokenizer_text.cpp src/models/omnivoice/audio_tokenizer.cpp + src/models/outetts/assets.cpp + src/models/outetts/dac.cpp + src/models/outetts/llama.cpp + src/models/outetts/loader.cpp + src/models/outetts/session.cpp + src/models/outetts/tokenizer.cpp src/models/omnivoice/prompt_builder.cpp src/models/omnivoice/generator.cpp src/models/omnivoice/postprocess.cpp @@ -717,6 +723,7 @@ if (ENGINE_BUILD_WARMBENCH) add_engine_warmbench(moss_tts_local_warm_bench tests/moss_tts_local/moss_tts_local_warm_bench.cpp) add_engine_warmbench(nemotron_asr_warm_bench tests/nemotron_asr/nemotron_asr_warm_bench.cpp) add_engine_warmbench(omnivoice_warm_bench tests/omnivoice/omnivoice_warm_bench.cpp) + add_engine_warmbench(outetts_warm_bench tests/outetts/outetts_warm_bench.cpp) add_engine_warmbench(pocket_tts_warm_bench tests/pocket_tts/pocket_tts_warm_bench.cpp) add_engine_warmbench(qwen3_asr_warm_bench tests/qwen3_asr/qwen3_asr_warm_bench.cpp) add_engine_warmbench(qwen3_forced_aligner_warm_bench tests/qwen3_forced_aligner/qwen3_forced_aligner_warm_bench.cpp) @@ -786,6 +793,14 @@ if (ENGINE_BUILD_TESTS) COMMAND audio_chunking_test ) + add_engine_unittest(outetts_generation_budget_test tests/unittests/test_outetts_generation_budget.cpp) + target_include_directories(outetts_generation_budget_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) + + add_test( + NAME outetts_generation_budget_test + COMMAND outetts_generation_budget_test + ) + add_engine_unittest(subtitle_formatter_test tests/unittests/test_subtitle_formatter.cpp) add_test( diff --git a/README.md b/README.md index 3cacbca0..c9831ff6 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ audio.cpp would not be moving this quickly without generous contributors bringin | **miocodec** | audio codec, voice conversion backend | lang agnostic | MioCodec v2, 25 Hz, 44.1 kHz | | **miotts** | TTS, voice cloning | en, ja | MioTTS-1.7B | | **omnivoice** | TTS, voice cloning, voice design | 646+ langs | OmniVoice, Qwen3-0.6B based | +| **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Llama-OuteTTS-1.0-1B | | **pocket_tts** | TTS, voice cloning | en, de, it, pt, es | PocketTTS-100M | | **nemotron_asr** | ASR | 100+ ASR prompt codes incl. auto | Nemotron 3.5 ASR Streaming 0.6B | | **qwen3_asr** | ASR | zh, en, yue, ar, de, fr, es, pt, id, it, ko, ru, th, vi, ja, tr, hi, ms, nl, sv, da, fi, pl, cs, fil, fa, el, ro, hu, mk | Qwen3-ASR-0.6B, Qwen3-ASR-1.7B-hf | @@ -386,6 +387,7 @@ Recommended top-level install packages: | `moss_tts_local_v1_5` | MOSS-TTS-Local Transformer v1.5 | No | | `nemotron_asr` | Nemotron ASR | **Yes** | | `omnivoice` | OmniVoice | **Yes** | +| `outetts_1_0_1b` | OuteTTS 1.0 1B with IBM DAC codec and Qwen3-aligned voice cloning | No | | `parakeet_tdt_0_6b_v3` | Parakeet TDT 0.6B v3 | **Yes** | | `pocket_tts` | PocketTTS | **Yes** | | `qwen3_asr_0_6b` | Qwen3 ASR 0.6B | **Yes** | @@ -639,10 +641,10 @@ The framework also has a reusable GGUF tensor source and a streaming converter. container reader is shared by all model families; a family still has to list a `.gguf` checkpoint as one of its accepted assets because model configuration and tensor naming remain architecture-specific. Qwen3 ASR, Qwen3 Forced Aligner, Qwen3 TTS, Nemotron -3.5 ASR, VibeVoice-ASR, Higgs Audio STT, Hviske ASR, and Citrinet ASR currently accept +3.5 ASR, VibeVoice-ASR, Higgs Audio STT, Hviske ASR, Citrinet ASR, and OuteTTS currently accept `model.gguf` (including `speech_tokenizer/model.gguf` for TTS). The converter recursively embeds sidecar files up to 64 MiB by default using binary-safe metadata, including nested tokenizer models, -and Qwen3 ASR, Nemotron ASR, VibeVoice-ASR, Higgs Audio STT, Hviske ASR, and Citrinet ASR +and Qwen3 ASR, Nemotron ASR, VibeVoice-ASR, Higgs Audio STT, Hviske ASR, Citrinet ASR, and OuteTTS can load the resulting `model.gguf` as a standalone file. The converter embeds the selected package spec in new GGUF files. Standalone conversion with embedded sidecars is the default and fails if required package resources are missing. Pass `--no-sidecars` only to explicitly diff --git a/docs/gguf.md b/docs/gguf.md index d66840f1..68911b34 100644 --- a/docs/gguf.md +++ b/docs/gguf.md @@ -273,6 +273,7 @@ Status labels: | `moss_tts_nano` | Done | Pass | --- | Pass | No (similarity drift, frame drift, text large drift) | | `nemotron_asr` | Done | Pass | --- | Pass | Pass (minor filler drift) | | `omnivoice` | Done | Pass | --- | No (runtime assert, no audio) | No (runtime assert, no audio) | +| `outetts` | Done | Pass (TTS + clone) | --- | --- | Pass (TTS + clone) | | `pocket_tts` | No | --- | --- | --- | --- | | `qwen3_asr` | Done | Pass | --- | Pass | Pass | | `qwen3_forced_aligner` | Done | Pass | --- | Pass | Pass | diff --git a/docs/memory_saver.md b/docs/memory_saver.md index 658c2a96..044c250e 100644 --- a/docs/memory_saver.md +++ b/docs/memory_saver.md @@ -31,6 +31,7 @@ Native/default weights were used for all rows. - VoxCPM2 used the OpenAI-compatible offline speech endpoint with a 2048-character voice-design request, `seed=1234`, `max_tokens=512`, `num_inference_steps=10`, and `guidance_scale=2.0`. The default and `mem_saver` WAV outputs were byte-identical. - IndexTTS2 used a five-request server sequence with the same seeds and references for default and `mem_saver`: normal text, longer text, longer emotion-text request with a different reference, shorter text, then longer text. - Irodori TTS 500M used a five-request server sequence with the same seeds and options for default and `mem_saver`: reference text, longer reference text, longer no-reference emoji/style text, shorter reference text, then longer reference text. +- OuteTTS 1.0 1B Q8 used a five-request long-lived-session sequence on an RTX 3090: repeated fixed-seed TTS, four-chunk long-form TTS, then repeated fixed-seed cloning with the same reference. CUDA timing is the mean of three fresh processes per mode, alternated to reduce ordering bias; memory saver was 0.35-0.53% slower per request and 0.44% slower over the mean sequence total. This is close to run-to-run variance and is not evidence of a speed benefit. The default and `mem_saver` WAV outputs were byte-identical. VRAM is total-device usage with no other CUDA workload; resident VRAM was sampled during a five-second post-sequence hold. Per-request timing and RTF are recorded in `docs/outetts_validation.md`. | Model | Mode | Peak VRAM | Resident VRAM | Server wall | Audio | RTF | |---|---|---:|---:|---:|---:|---:| @@ -58,3 +59,5 @@ Native/default weights were used for all rows. | Irodori TTS 500M | mem_saver | 11222 MiB | 3570 MiB | 3276.203 ms | 95.4s | 0.0343418 | | Irodori TTS 500M 6000-char | default | 18693 MiB | 13367 MiB | 27185.7 ms | 777.92s | 0.0349466 | | Irodori TTS 500M 6000-char | mem_saver | 11588 MiB | 3609 MiB | 27828 ms | 777.92s | 0.0357724 | +| OuteTTS 1.0 1B Q8 | default | 17653 MiB | 294 MiB | 29911.38 ms | 11.237s | 2.662 | +| OuteTTS 1.0 1B Q8 | mem_saver | 5780 MiB | 294 MiB | 30043.25 ms | 11.237s | 2.674 | diff --git a/docs/outetts_validation.md b/docs/outetts_validation.md new file mode 100644 index 00000000..8595aa4c --- /dev/null +++ b/docs/outetts_validation.md @@ -0,0 +1,430 @@ +# OuteTTS validation + +This procedure exercises OuteTTS 1.0 1B in one long-lived audio.cpp session. +It covers normal TTS, framework long-form text chunking, voice cloning, repeated +reference-profile cache hits, cached-step graph reuse, stage timings, and +default-versus-`mem_saver` memory behavior. + +## Model setup + +Install the safetensors model, IBM DAC, and Qwen3 Forced Aligner resources: + +```bash +python tools/model_manager.py install outetts_1_0_1b --models-dir models +``` + +The model-manager package id is `outetts_1_0_1b`. It creates these model +roots: + +- `models/Llama-OuteTTS-1.0-1B` +- `models/DAC.speech.v1.0` +- `models/Qwen3-ForcedAligner-0.6B` + +The standalone GGUF command is documented in [TTS](tts.md#outetts). The packed +file used below contains the OuteTTS language model, IBM DAC, Qwen aligner, +tokenizers, configuration, and package specification. + +## Python reference setup + +The parity run used the official OuteTTS repository at commit +`f5eac6e70d792844c6a6959d900a47af2c061a5b` (`outetts` 0.4.4), Python +3.10, PyTorch 2.5.1+cu121, and Transformers 4.52.3: + +```powershell +git clone https://github.com/edwko/OuteTTS build\reference\OuteTTS +git -C build\reference\OuteTTS checkout f5eac6e70d792844c6a6959d900a47af2c061a5b +python -m venv --system-site-packages build\reference\venv +build\reference\venv\Scripts\python.exe -m pip install -e build\reference\OuteTTS --no-deps +build\reference\venv\Scripts\python.exe -m pip install transformers==4.52.3 llama-cpp-python==0.3.9 polars natsort mecab-python3 unidic-lite uroman openai-whisper ftfy pyloudnorm +``` + +The reference loaded `models/Llama-OuteTTS-1.0-1B` through +`outetts.Backend.HF` with FP32 CUDA weights and loaded the official IBM DAC +checkpoint from +`models/DAC.speech.v1.0/weights_24khz_1.5kbps_v1.0.pth`. Both implementations +used temperature `0.4`, repetition penalty `1.1` over the latest 64 tokens, +top-k `40`, top-p `0.9`, min-p `0.05`, the request-file seeds, and the same +maximum number of new tokens. + +Temperature zero was deliberately not used. OuteTTS 1.0's official +`generation_config.json`, README example, and Python API all select sampled +generation with temperature `0.4`; changing to greedy generation would test a +different inference contract. OuteTTS has no diffusion or latent-noise input. +Its only request-time randomness is language-model token sampling. + +For a controlled clone comparison, Qwen word timings were generated once and +passed to the official Python `AudioProcessor.create_speaker_from_dict` path: + +```powershell +build\windows-cuda-release\bin\audiocpp_cli.exe ` + --task align --family qwen3_forced_aligner ` + --model ..\models\Qwen3-ForcedAligner-0.6B_Q8\Qwen3-ForcedAligner-0.6B_Q8.gguf ` + --backend cuda --audio assets\resources\b.wav ` + --text "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you." ` + --language en --words-out build\reference\b_words.json +``` + +Python still performed the official DAC encoding and feature extraction. This +keeps different alignment algorithms from contaminating the TTS comparison. +The audio.cpp cold-clone timing includes its Qwen alignment, while the Python +number starts from the supplied word timings. + +Run the committed official-reference driver after creating the alignment JSON: + +```powershell +build\reference\venv\Scripts\python.exe ` + tools\audiocpp_cli\outetts_reference.py ` + --model ..\models\Llama-OuteTTS-1.0-1B ` + --dac ..\models\DAC.speech.v1.0\weights_24khz_1.5kbps_v1.0.pth ` + --alignment-json build\reference\b_words.json ` + --request-file tests\outetts\warm_bench_requests.json ` + --device cuda --dtype fp32 ` + --out-dir build\reference\official_script_fp32 +``` + +The driver resets both `torch.manual_seed` and +`torch.cuda.manual_seed_all` immediately before every `model.generate` call, +after all prompt, alignment, and DAC-profile work. It writes native 24 kHz +WAVs plus `boundaries.json`, `memory.json`, `stdout.log`, and `command.json`. + +## Exact build commands + +The Windows validation used the project build script to establish the compiler +and backend configuration, then explicitly enabled the warm-benchmark targets. + +Windows CPU: + +```powershell +.\scripts\build_windows.ps1 -Preset windows-cpu-release -ConfigureOnly +cmake -S . -B build\windows-cpu-release -DENGINE_BUILD_WARMBENCH=ON +cmake --build build\windows-cpu-release --parallel 4 --target audiocpp_cli audiocpp_server outetts_warm_bench +``` + +Windows CUDA 12.4: + +```powershell +$env:CUDA_PATH = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" +.\scripts\build_windows.ps1 -Preset windows-cuda-release -ConfigureOnly +cmake -S . -B build\windows-cuda-release -DENGINE_BUILD_WARMBENCH=ON +cmake --build build\windows-cuda-release --parallel 4 --target audiocpp_cli audiocpp_server outetts_warm_bench +``` + +## Exact standalone-GGUF path tests + +The tested GGUF was deliberately outside the PR worktree and its directory +contained only one file: + +```text +..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf +``` + +Normal TTS: + +```powershell +build\windows-cuda-release\bin\audiocpp_cli.exe ` + --task tts --family outetts ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` + --backend cuda ` + --text "This is the standalone GGUF path test." ` + --max-tokens 256 --request-option seed=1234 ` + --out ..\outputs\outetts_path_test_tts.wav +``` + +Voice cloning using the aligner and DAC embedded in the same GGUF: + +```powershell +build\windows-cuda-release\bin\audiocpp_cli.exe ` + --task clon --family outetts ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` + --backend cuda ` + --voice-ref assets\resources\b.wav ` + --reference-text "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you." ` + --request-option reference_language=en ` + --text "This is the standalone GGUF cloning path test." ` + --max-tokens 256 --request-option seed=42 ` + --out ..\outputs\outetts_path_test_clone.wav +``` + +Both commands loaded successfully without an external package spec, tokenizer, +DAC directory, aligner directory, or other sidecar. The generated 24 kHz mono +WAVs passed ffmpeg decoding: + +| Artifact | Duration | Bytes | SHA-256 | +|---|---:|---:|---| +| `..\outputs\outetts_path_test_tts.wav` | 1.399667s | 67228 | `92DA5D8438D4E6A79FBDDBD8EDA77D1AFD2A9B47F61085ECD3AD45C913102904` | +| `..\outputs\outetts_path_test_clone.wav` | 1.453000s | 69788 | `E5B46E62C2DB89A654F7EC21A7BE31F575D6F667F8578224386E7C552492B05E` | + +## Exact long-lived-session runs + +CUDA default: + +```powershell +build\windows-cuda-release\bin\outetts_warm_bench.exe ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` + --backend cuda --threads 8 ` + --request-file tests\outetts\warm_bench_requests.json ` + --hold-seconds 5 ` + --audio-out-dir ..\outputs\outetts_review_cuda_default ` + --log-file build\logs\warmbench\outetts-cuda-default.log +``` + +Expected trace evidence: + +- `outetts.text_chunk_count` is greater than one for `tts_longform`. +- `outetts.llama.step.graph_reused=1` appears after the first compatible + generation request or chunk. +- the second identical reference reports `outetts.reference_cache.hit=1`. +- `tts_cold` and `tts_repeat` are byte-identical, as are `clone_cold` + and `clone_repeat`; this verifies that warm graph/profile reuse does not + change deterministic output. +- `outetts.aligner.runtime_reused=1` is observable for uncached references + while the default session retains the aligner. +- only one active OuteTTS Llama runtime is retained; switching between native + TTS weights and the CUDA F32 cloning fallback replaces the previous runtime. + +Run the same request sequence in a fresh process with memory saver enabled: + +```powershell +build\windows-cuda-release\bin\outetts_warm_bench.exe ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` + --backend cuda --threads 8 ` + --request-file tests\outetts\warm_bench_requests.json ` + --session-option outetts.mem_saver=true ` + --hold-seconds 5 ` + --audio-out-dir ..\outputs\outetts_review_cuda_mem_saver ` + --log-file build\logs\warmbench\outetts-cuda-mem_saver.log +``` + +CPU default: + +```powershell +build\windows-cpu-release\bin\outetts_warm_bench.exe ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` + --backend cpu --threads 8 ` + --request-file tests\outetts\warm_bench_requests.json ` + --hold-seconds 5 ` + --audio-out-dir ..\outputs\outetts_review_cpu_default ` + --log-file build\logs\warmbench\outetts-cpu-default-final.log +``` + +Safetensors CUDA parity run (DAC is resolved by the package spec; the aligner +path is supplied because it is not embedded in the safetensors directory): + +```powershell +build\windows-cuda-release\bin\outetts_warm_bench.exe ` + --model ..\models\Llama-OuteTTS-1.0-1B ` + --backend cuda --threads 8 ` + --request-file tests\outetts\warm_bench_requests.json ` + --session-option outetts.weight_type=f32 ` + --session-option outetts.aligner_model_path=..\models\Qwen3-ForcedAligner-0.6B ` + --audio-out-dir build\reference\cpp_f32_final_explicit ` + --log-file build\reference\cpp_f32_final_explicit.log +``` + +The memory-saver trace reports a positive +`outetts.llama.step.released_cache_capacity` after generation and +`outetts.aligner.runtime_released=1` after an uncached reference. + +Sample total-device VRAM every 250 ms while each fresh benchmark runs. Ensure +that no unrelated CUDA workload is active: + +```powershell +nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -lms 250 +``` + +Compare peak and final resident VRAM, request wall time, output duration, and RTF +between the two runs. Keep model, backend, device, seed, requests, and +quantization identical. + +## Generated artifacts + +Each benchmark directory contains: + +```text +tts_cold_1.wav +tts_repeat_1.wav +tts_longform_1.wav +clone_cold_1.wav +clone_repeat_1.wav +``` + +The exact generated directories and logs were: + +- `..\outputs\outetts_review_cuda_default` +- `..\outputs\outetts_review_cuda_mem_saver` +- `..\outputs\outetts_review_cpu_default` +- `build\logs\warmbench\outetts-cuda-default.log` +- `build\logs\warmbench\outetts-cuda-mem_saver.log` +- `build\logs\warmbench\outetts-cpu-default-final.log` +- `build\logs\warmbench\outetts-ab-torch-sampler\default-{1,2,3}-audio` +- `build\logs\warmbench\outetts-ab-torch-sampler\mem_saver-{1,2,3}-audio` +- `build\reference\cpp_f32_final_explicit` +- `build\reference\cpp_q8_torch_sampler` +- `build\reference\official_script_fp32\outputs` +- `build\reference\official_script_fp32\boundaries.json` + +These validation artifacts are reproducible local outputs and are not committed +to the repository. + +## Measured validation + +The committed five-request sequence was measured with the packed Q8 GGUF on an +NVIDIA GeForce RTX 3090 (CUDA 12.4). Each CUDA mode was run in three fresh +processes in alternating order. Values below are mean +/- sample standard +deviation. The Python HF and audio.cpp safetensors columns are single loaded- +session runs; model load is excluded from every per-request measurement. + +| Request | Python HF FP32 wall / RTF | C++ FP32 wall / RTF | C++ Q8 wall / RTF | +|---|---:|---:|---:| +| `tts_cold` | 4599.12 ms / 3.485 | 5366.24 ms / 4.066 | 3211.84 +/- 17.30 ms / 2.434 | +| `tts_repeat` | 3777.69 ms / 2.863 | 3864.98 ms / 2.929 | 3030.79 +/- 18.60 ms / 2.297 | +| `tts_longform` | 15129.91 ms / 2.690 | 15547.80 ms / 2.764 | 13015.83 +/- 171.29 ms / 2.342 | +| `clone_cold` | 5762.51 ms / 3.792 | 5988.01 ms / 3.940 | 6432.68 +/- 65.11 ms / 4.233 | +| `clone_repeat` | 5522.67 ms / 3.634 | 4231.19 ms / 2.784 | 4220.23 +/- 44.07 ms / 2.777 | + +These are per-request timings from a single loaded session; model loading is +excluded. Python and C++ FP32 generated 1.31967, 1.31967, 5.62533, 1.51967, +and 1.51967 seconds respectively. Q8 generated 5.55867 seconds for the +long-form request and matched the other four durations. + +The memory-saver A/B is also reported per request rather than inferred from a +single aggregate run: + +| Request | Default CUDA Q8 | Memory saver CUDA Q8 | Wall delta | Default RTF | Memory-saver RTF | +|---|---:|---:|---:|---:|---:| +| `tts_cold` | 3211.84 +/- 17.30 ms | 3223.24 +/- 12.89 ms | +0.35% | 2.434 | 2.443 | +| `tts_repeat` | 3030.79 +/- 18.60 ms | 3044.58 +/- 18.76 ms | +0.45% | 2.297 | 2.307 | +| `tts_longform` | 13015.83 +/- 171.29 ms | 13070.17 +/- 105.32 ms | +0.42% | 2.342 | 2.351 | +| `clone_cold` | 6432.68 +/- 65.11 ms | 6467.00 +/- 46.38 ms | +0.53% | 4.233 | 4.256 | +| `clone_repeat` | 4220.23 +/- 44.07 ms | 4238.27 +/- 17.07 ms | +0.43% | 2.777 | 2.789 | + +Memory saver was slightly slower on every request in this controlled repeat: +0.35% to 0.53%, and 0.44% over the mean five-request wall total. These small +differences are near normal run-to-run variance, so there is no evidence that +memory saver is a speed optimization. It remains useful +because the independently sampled peak VRAM fell from 17653 MiB to 5780 MiB +(67.3%) with the same 294 MiB post-session resident VRAM. It is therefore a +memory-versus-speed option, not a speed optimization. + +The Windows CPU Q8 run is likewise reported per request: + +| Request | Wall | Audio | RTF | +|---|---:|---:|---:| +| `tts_cold` | 10517.2 ms | 1.31967s | 7.970 | +| `tts_repeat` | 10349.9 ms | 1.31967s | 7.843 | +| `tts_longform` | 41604.0 ms | 5.67867s | 7.326 | +| `clone_cold` | 22849.3 ms | 1.58633s | 14.404 | +| `clone_repeat` | 18205.6 ms | 1.58633s | 11.477 | + +All generated WAV files passed an ffmpeg decode check. The traces confirmed +four framework chunks for the long-form request, compatible step-graph reuse in +default mode, explicit graph release in memory-saver mode, and a reference- +profile cache hit on the repeated clone. In the original instrumented default +CUDA run, cold reference preparation took about 890 ms (109 ms alignment, +139 ms DAC encoding, and 214 ms profile construction), while the repeated +cached reference took 0.29 ms. + +## Path and parity results + +- The standalone path test loaded the one-file GGUF from outside the worktree; + no adjacent sidecars or `model_specs` directory were present. +- Within CUDA, cold and repeated TTS were byte-identical with SHA-256 + `9D1AF0D25212129B3C26880F7C928CFFA813A4B9EF22F12F390A4C372D51D406`. +- Within CUDA, cold and repeated cloning were byte-identical with SHA-256 + `EB7E6B5A1330F845F45D3DA21C8AC28490572F95218A6F0D3DA2A83CB797EFE6`. +- The CUDA default and memory-saver outputs had the same hashes. +- Within CPU, cold and repeated TTS were byte-identical with SHA-256 + `A315C0AD425C99910597FCC4BAB9D80CB5F4C893176EBD9099BA6C6AA7B66BFD`. +- Within CPU, cold and repeated cloning were byte-identical with SHA-256 + `1A2009C0D512773A4B5082BF37EDEDD6A52A37CC6D556772C62EA26550A83B4D`. +- CPU and CUDA WAV bytes are not identical because backend floating-point + execution differs; deterministic reuse parity was therefore checked within + each backend rather than asserted across backends. + +Python parity uses the repository's +`tools/audiocpp_cli/compare_audiocpp_cli_path_results.py` implementation: PCM +WAV cosine plus an 80-band `log1p` mel-spectrogram cosine (`n_fft=1024`, +`hop=256`). Both sides write the DAC-native 24 kHz signal directly. Raw WAV +cosine is highly phase-sensitive; log-mel cosine is the more informative value +when the DAC encoders are numerically close but not bit-identical. + +The CUDA sampler now uses the framework's PyTorch-compatible exponential-race +random stream, equivalent to the `torch.multinomial` path used by Transformers. +Temporary trace-level instrumentation compared prompt strings, prompt token +IDs, generated token IDs, codec codes, and WAV frame counts at component +boundaries. It was removed after validation to avoid normal-log spam. The +permanent Python driver records the Python side in `boundaries.json`. + +Boundary results under the controlled FP32 run: + +| Request | Chunks | Python/C++ prompt IDs | Generated IDs per chunk | Python/C++ frames | +|---|---:|---:|---:|---:| +| `tts_cold` | 1 | 22 / 22 | 256 | 31672 / 31672 | +| `tts_repeat` | 1 | 22 / 22 | 256 | 31672 / 31672 | +| `tts_longform` | 4 | 20,22,14,21 / same | 256 each | 135008 / 135008 | +| `clone_cold` | 1 | 2367 / 2367 | 256 | 36472 / 36472 | +| `clone_repeat` | 1 | 2367 / 2367 | 256 | 36472 / 36472 | + +The sampled prompt/generated boundary values for all six ordinary-TTS chunks +had zero mismatches. Clone prompt construction also matches the official word +labels and token count. The C++ DAC reference encoder still differs slightly +from Python: among 1046 reference-profile frames, C1 differs on 48 and C2 on +118. That numerical boundary explains why cloning is not waveform-identical. + +Primary native/FP32 parity, measured only after the controlled random stream +and frame counts matched: + +| Request | WAV cosine | Log-mel cosine | C++ / Python frames | +|---|---:|---:|---:| +| `tts_cold` | 0.999812247 | 0.999997481 | 31672 / 31672 | +| `tts_repeat` | 0.999812238 | 0.999997483 | 31672 / 31672 | +| `tts_longform` | 0.997324530 | 0.999294328 | 135008 / 135008 | +| `clone_cold` | -0.007753064 | 0.889403845 | 36472 / 36472 | +| `clone_repeat` | -0.007753064 | 0.889403845 | 36472 / 36472 | + +Only after that path was clean was the packed Q8 model compared with the same +FP32 Python reference. Quantization is expected to perturb autoregressive +sampling, so this is a secondary characterization rather than the correctness +gate: + +| Request | WAV cosine | Log-mel cosine | C++ Q8 / Python frames | +|---|---:|---:|---:| +| `tts_cold` | -0.016326171 | 0.696795792 | 31672 / 31672 | +| `tts_repeat` | -0.016326591 | 0.696792751 | 31672 / 31672 | +| `tts_longform` | 0.577927575 | 0.920741352 | 133408 / 135008 | +| `clone_cold` | -0.033990775 | 0.882715855 | 36472 / 36472 | +| `clone_repeat` | -0.033990775 | 0.882715855 | 36472 / 36472 | + +## Backend coverage + +| Backend | Coverage | +|---|---| +| Windows CPU | CLI/server/warm-benchmark build; normal TTS, long-form TTS, cloning, cache reuse, graph reuse, WAV decode, timing, and RSS run | +| Windows CUDA 12.4 | CLI/server/warm-benchmark build; normal TTS, long-form TTS, cloning, standalone-path loading, cache/graph reuse, three-run default/memory-saver A/B, Python HF parity, WAV decode, timing, RTF, and VRAM run | +| Linux CPU | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | +| Linux Vulkan | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | +| macOS CPU | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | +| Metal | Not enabled or runtime-tested in this validation | + +## Known limitations + +- OuteTTS is offline-only in this implementation. +- Voice cloning requires an accurate transcript of the reference WAV and rejects + references longer than 20 seconds. +- Quantized CUDA cloning expands the OuteTTS language-model weights to F32 for + generation correctness. Default mode can therefore have a high transient + peak while the aligner is retained; `outetts.mem_saver=true` releases the + aligner and cached-step graph between phases. +- Long-form output concatenates independently generated chunks. `max_tokens` + applies to each chunk; the runtime splits text further to fit an explicit cap + and retries a chunk that reaches the cap without EOS/audio-end as smaller + chunks rather than returning silently truncated speech. It reports an error + only when the remaining text cannot be split any further. +- CPU and CUDA outputs are deterministic within each tested backend but are not + expected to be byte-identical across backends. +- The controlled CUDA sampling stream is PyTorch-compatible. CPU sampling is + deterministic for a fixed seed but still uses the C++ standard-library + distribution and is not claimed to reproduce PyTorch token-for-token. +- Quantized weights can change sampled tokens and output length even when the + seed and sampler implementation match, as shown by the Q8 long-form frame + drift above. diff --git a/docs/tts.md b/docs/tts.md index d0ff8eee..ea768215 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -492,6 +492,130 @@ audiocpp_cli --task clon --family irodori_tts --model /path/to/Irodori-TTS-500M- | `--session-option irodori_tts.rf_weight_context_mb=` | MB | `768` | RF sampler weight context size. | | `--session-option irodori_tts.codec_weight_context_mb=` | MB | `512` | DACVAE codec weight context size. | +## OuteTTS + +OuteTTS 1.0 1B generates 24 kHz speech with a Llama text/audio-token model and the +IBM DAC 1.5 kbps codec. The integration supports both no-reference generation and +native voice cloning. For cloning, the DAC encoder turns a reference WAV into the two +codec-token streams used to condition the language model; no separate encoder model or +speaker-profile file is required. + +| Field | Value | +|---|---| +| Family | `outetts` | +| Model directory | `models/Llama-OuteTTS-1.0-1B` | +| Task | `tts`, `clon` | +| Modes | `offline` | +| Languages | `en`, `ar`, `zh`, `nl`, `fr`, `de`, `it`, `ja`, `ko`, `lt`, `ru`, `es`, plus moderate-data `pt`, `be`, `bn`, `ka`, `hu`, `lv`, `fa`, `pl`, `sw`, `ta`, `uk` | +| Voice input | Optional reference WAV plus its transcript | +| Output | mono 24 kHz WAV | + +Install both the language model and its DAC dependency: + +```bash +python tools/model_manager.py install outetts_1_0_1b --models-dir models +``` + +Run the safetensors package: + +```bash +audiocpp_cli --task tts --family outetts \ + --model models/Llama-OuteTTS-1.0-1B \ + --backend cuda --text "Hello from OuteTTS." \ + --max-tokens 1024 --out out.wav +``` + +Clone a voice with either the safetensors package or standalone GGUF. The reference +transcript must match the spoken reference audio. Around ten seconds of clean speech is +recommended; the maximum accepted reference length is twenty seconds. The standalone +GGUF described below contains Qwen3 Forced Aligner and activates it automatically for +accurate per-word codec conditioning: + +```bash +audiocpp_cli --task clon --family outetts \ + --model models/Llama-OuteTTS-1.0-1B-Q8_0/model.gguf \ + --backend cuda \ + --voice-ref reference.wav \ + --reference-text "The exact words spoken in reference.wav." \ + --request-option reference_language=en \ + --text "This sentence uses the cloned voice." \ + --max-tokens 1024 --out cloned.wav +``` + +`--task tts` with the same `--voice-ref` and `--reference-text` options also enables +speaker conditioning, which is useful for clients that expose one TTS route. +Safetensors packages and older OuteTTS GGUFs do not contain the aligner. For those +models, pass `outetts.aligner_model_path`; cloning fails clearly instead of using +unreliable estimated word boundaries. + +The installer places `DAC.speech.v1.0` and `Qwen3-ForcedAligner-0.6B` beside the +OuteTTS directory. It converts the official DAC checkpoint to a safe tensor source. +To do that conversion manually: + +```bash +python tools/convert_outetts_dac.py \ + models/DAC.speech.v1.0/weights_24khz_1.5kbps_v1.0.pth \ + models/DAC.speech.v1.0/model.safetensors +``` + +Pack the language model, DAC, Qwen3 Forced Aligner, and sidecars into one standalone +Q8 GGUF: + +```bash +audiocpp_gguf \ + --input model_weights=models/Llama-OuteTTS-1.0-1B/model.safetensors \ + --input dac_weights=models/DAC.speech.v1.0/model.safetensors \ + --input aligner_weights=models/Qwen3-ForcedAligner-0.6B/model.safetensors \ + --root models/Llama-OuteTTS-1.0-1B \ + --sidecar models/DAC.speech.v1.0/config.json=dac/config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/config.json=aligner/config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/generation_config.json=aligner/generation_config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/preprocessor_config.json=aligner/preprocessor_config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/tokenizer_config.json=aligner/tokenizer_config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/vocab.json=aligner/vocab.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/merges.txt=aligner/merges.txt \ + --output models/Llama-OuteTTS-1.0-1B-Q8_0/model.gguf \ + --type q8_0 +``` + +The resulting GGUF contains all three tensor groups, the model specification, and all +required sidecars. It does not need the original directories. Normal TTS does not run +the embedded aligner; it is initialized only when reference cloning is requested: + +```bash +audiocpp_cli --task tts --family outetts \ + --model models/Llama-OuteTTS-1.0-1B-Q8_0/model.gguf \ + --backend cuda --text "Hello from the standalone GGUF." \ + --max-tokens 1024 --out out.wav +``` + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--max-tokens` | integer | automatic | Maximum audio-token generation length per chunk. When omitted, OuteTTS estimates a safe budget from the chunk's word and character counts. An explicit smaller cap causes additional text splitting; a chunk that unexpectedly reaches the cap is retried as smaller chunks instead of silently truncating speech. | +| `--temperature` | float | `0.4` for cloning; model default for TTS | Sampling temperature. Voice cloning follows the official OuteTTS default without changing temperature between words. | +| `--top-k` | integer | `40` | Top-k sampling limit. | +| `--top-p` | float | `0.9` | Nucleus sampling limit. | +| `--request-option min_p=` | float | `0.05` | Minimum probability relative to the most likely token. | +| `--repetition-penalty` | float | `1.1` | Repetition penalty over the latest 64 tokens. | +| `--request-option seed=` | integer | native clone: `4099`; quantized clone: `42` | Deterministic sampling seed. The defaults were separately verified for the native and Q8 cloning paths. | +| `--text-chunk-size` | characters | `256` | Initial framework long-form text chunk size. Each chunk is split further when needed to fit `max_tokens`, generated and decoded in the same loaded session, then appended to the output WAV. | +| `--text-chunk-mode` | `default`, `tag_aware`, `japanese`, `endline` | `default` | Framework long-form text chunking mode. | +| `--reference-text` | text | none | Exact transcript of `--voice-ref`; required for voice cloning. | +| `--request-option reference_language=` | language code | `en` | Language used by the optional reference aligner. | +| `--session-option outetts.weight_type=native|f32|f16|bf16|q8_0` | enum | `native` | Language-model weight storage type. For CUDA voice cloning, quantized weights remain compact in the GGUF but are expanded to F32 in VRAM to avoid generation divergence over long reference-codec prompts. Normal TTS and CPU cloning keep the selected type. | +| `--session-option outetts.aligner_model_path=` | model path | embedded aligner | Optional external Qwen3 Forced Aligner override, required only for safetensors packages and older GGUFs without the embedded aligner. | +| `--session-option outetts.reference_cache_slots=` | integer | `1` | LRU slots for prepared reference profiles (alignment, DAC codes, and word features). Set `0` to disable reuse. | +| `--session-option outetts.mem_saver=true|false` | bool | `false` | Release the reusable Llama cached-step graph after each generated chunk and release the aligner runtime after preparing a reference. Model and DAC weights stay resident; later requests rebuild released state. | + +With logging enabled, OuteTTS reports framework chunk count and token budget, +per-chunk word/character counts, recommended and effective generation limits, +the natural stop reason, reference-profile cache hits/evictions, Llama runtime +and step-graph rebuild/reuse, released cache capacity, and timings for reference +alignment, DAC encode/decode, prompt construction, generation, and the complete +session request. See +[OuteTTS validation](outetts_validation.md) for the reproducible long-lived +session and memory test. + ## Supertonic Supertonic 3 is a preset-voice multilingual TTS model. It does not use external speaker references in the current integration. diff --git a/include/engine/models/outetts/assets.h b/include/engine/models/outetts/assets.h new file mode 100644 index 00000000..edd22bf0 --- /dev/null +++ b/include/engine/models/outetts/assets.h @@ -0,0 +1,68 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include + +namespace engine::models::qwen3_asr { +struct Qwen3ASRAssets; +} + +namespace engine::models::outetts { + +struct OuteTTSLlama3RopeConfig { + float factor = 32.0F; + float low_freq_factor = 1.0F; + float high_freq_factor = 4.0F; + int64_t original_max_position_embeddings = 8192; +}; + +struct OuteTTSConfig { + int64_t bos_token_id = 0; + int64_t eos_token_id = 0; + int64_t pad_token_id = 0; + int64_t hidden_size = 0; + int64_t intermediate_size = 0; + int64_t max_position_embeddings = 0; + int64_t num_attention_heads = 0; + int64_t num_hidden_layers = 0; + int64_t num_key_value_heads = 0; + int64_t head_dim = 0; + int64_t vocab_size = 0; + float rms_norm_eps = 1.0e-5F; + float rope_theta = 500000.0F; + OuteTTSLlama3RopeConfig rope_scaling; + int64_t sample_rate = 24000; + int64_t hop_length = 320; + int64_t codebook_size = 1024; + int64_t codebooks = 2; + int64_t dac_latent_dim = 1024; + int64_t dac_decoder_dim = 1536; +}; + +struct OuteTTSGenerationConfig { + float temperature = 0.4F; + float repetition_penalty = 1.1F; + int64_t repetition_window = 64; + int64_t top_k = 40; + float top_p = 0.9F; + float min_p = 0.05F; + int64_t max_length = 8192; +}; + +struct OuteTTSAssets { + assets::ResourceBundle resources; + OuteTTSConfig config; + OuteTTSGenerationConfig generation; + std::shared_ptr model_weights; + std::shared_ptr dac_weights; + std::shared_ptr + embedded_aligner; +}; + +std::shared_ptr load_outetts_assets(const std::filesystem::path & model_path); + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/dac.h b/include/engine/models/outetts/dac.h new file mode 100644 index 00000000..57ba009d --- /dev/null +++ b/include/engine/models/outetts/dac.h @@ -0,0 +1,41 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/runtime/session.h" +#include "engine/models/outetts/assets.h" + +#include +#include +#include +#include + +namespace engine::models::outetts { + +class OuteTTSDacDecoder final { +public: + struct EncodedReference { + std::vector samples; + std::vector codebook1; + std::vector codebook2; + }; + + OuteTTSDacDecoder(std::shared_ptr assets, + core::ExecutionContext &execution_context, + size_t weight_context_bytes = 1024ull * 1024ull * 1024ull, + size_t graph_context_bytes = 1536ull * 1024ull * 1024ull, + assets::TensorStorageType weight_storage_type = + assets::TensorStorageType::F32); + ~OuteTTSDacDecoder(); + + runtime::AudioBuffer decode(const std::vector &codebook1, + const std::vector &codebook2); + + EncodedReference encode_reference(const runtime::AudioBuffer &audio); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/llama.h b/include/engine/models/outetts/llama.h new file mode 100644 index 00000000..916e8473 --- /dev/null +++ b/include/engine/models/outetts/llama.h @@ -0,0 +1,70 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/models/outetts/assets.h" + +#include +#include +#include +#include +#include + +namespace engine::models::outetts { + +struct OuteTTSGenerateOptions { + int64_t max_new_tokens = 4096; + float temperature = 0.4F; + float repetition_penalty = 1.1F; + int64_t repetition_window = 64; + int64_t top_k = 40; + float top_p = 0.9F; + float min_p = 0.05F; + uint32_t seed = 0; +}; + +enum class OuteTTSStopReason { + Eos, + AudioEnd, + MaxTokens, + ContextLimit, +}; + +std::string_view outetts_stop_reason_name(OuteTTSStopReason reason) noexcept; + +struct OuteTTSGenerateResult { + std::vector tokens; + OuteTTSStopReason stop_reason = OuteTTSStopReason::MaxTokens; +}; + +class OuteTTSLlamaRuntime final { +public: + OuteTTSLlamaRuntime( + std::shared_ptr assets, + core::BackendType backend_type, + int device, + int threads, + size_t weight_context_bytes = 4ull * 1024ull * 1024ull * 1024ull, + size_t constant_context_bytes = 256ull * 1024ull * 1024ull, + assets::TensorStorageType weight_storage_type = assets::TensorStorageType::Native); + ~OuteTTSLlamaRuntime(); + + OuteTTSLlamaRuntime(const OuteTTSLlamaRuntime &) = delete; + OuteTTSLlamaRuntime & operator=(const OuteTTSLlamaRuntime &) = delete; + + OuteTTSGenerateResult generate( + const std::vector & prompt, + const OuteTTSGenerateOptions & options, + int32_t eos_id, + int32_t audio_end_id) const; + + // Releases the reusable cached-step graph while keeping model weights + // resident. Returns the released KV-cache capacity in tokens. + int64_t release_cached_step_graph(); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/loader.h b/include/engine/models/outetts/loader.h new file mode 100644 index 00000000..10430843 --- /dev/null +++ b/include/engine/models/outetts/loader.h @@ -0,0 +1,33 @@ +#pragma once + +#include "engine/framework/runtime/model.h" +#include "engine/models/outetts/assets.h" + +#include +#include + +namespace engine::models::outetts { + +class OuteTTSLoadedModel final : public runtime::ILoadedVoiceModel { +public: + OuteTTSLoadedModel(runtime::ModelMetadata metadata, + runtime::CapabilitySet capabilities, + std::shared_ptr assets); + + const runtime::ModelMetadata &metadata() const noexcept override; + const runtime::CapabilitySet &capabilities() const noexcept override; + std::unique_ptr + create_task_session(const runtime::TaskSpec &task, + const runtime::SessionOptions &options) const override; + +private: + runtime::ModelMetadata metadata_; + runtime::CapabilitySet capabilities_; + std::shared_ptr assets_; +}; + +std::unique_ptr +load_outetts_model(const std::filesystem::path &model_path); +std::shared_ptr make_outetts_loader(); + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/session.h b/include/engine/models/outetts/session.h new file mode 100644 index 00000000..af30dd47 --- /dev/null +++ b/include/engine/models/outetts/session.h @@ -0,0 +1,72 @@ +#pragma once + +#include "engine/framework/runtime/cache_slots.h" +#include "engine/framework/runtime/session_base.h" +#include "engine/models/outetts/assets.h" +#include "engine/models/outetts/dac.h" +#include "engine/models/outetts/llama.h" +#include "engine/models/outetts/tokenizer.h" + +#include +#include +#include +#include +#include + +namespace engine::models::qwen3_forced_aligner { +class Qwen3ForcedAlignerSession; +} + +namespace engine::models::outetts { + +class OuteTTSSession final : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession { +public: + OuteTTSSession(runtime::TaskSpec task, runtime::SessionOptions options, + std::shared_ptr assets); + ~OuteTTSSession() override; + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest &request) override; + runtime::TaskResult run(const runtime::TaskRequest &request) override; + +private: + struct ReferenceProfileCacheKey { + uint64_t audio_hash = 0; + int sample_rate = 0; + int channels = 0; + size_t sample_count = 0; + std::string text; + std::string language; + }; + + struct ReferenceProfileCacheKeyEqual { + bool operator()(const ReferenceProfileCacheKey &lhs, + const ReferenceProfileCacheKey &rhs) const; + }; + + OuteTTSLlamaRuntime &llama(bool voice_cloning); + OuteTTSVoiceProfile prepare_voice_profile( + const runtime::AudioBuffer &audio, const std::string &reference_text, + const std::string &language, bool &cache_hit); + + runtime::TaskSpec task_; + std::shared_ptr assets_; + OuteTTSTokenizer tokenizer_; + std::unique_ptr llama_; + std::optional llama_storage_type_; + std::unique_ptr< + engine::models::qwen3_forced_aligner::Qwen3ForcedAlignerSession> + aligner_session_; + std::shared_ptr + aligner_assets_; + OuteTTSDacDecoder dac_; + bool mem_saver_ = false; + runtime::CacheSlots + reference_profile_cache_; + std::optional voice_profile_; +}; + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/tokenizer.h b/include/engine/models/outetts/tokenizer.h new file mode 100644 index 00000000..54acbdd1 --- /dev/null +++ b/include/engine/models/outetts/tokenizer.h @@ -0,0 +1,65 @@ +#pragma once + +#include "engine/models/outetts/assets.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::outetts { + +struct OuteTTSVoiceFeatures { + int energy = 0; + int spectral_centroid = 0; + int pitch = 0; +}; + +struct OuteTTSVoiceWord { + std::string text; + double duration = 0.0; + OuteTTSVoiceFeatures features; + std::vector codebook1; + std::vector codebook2; +}; + +struct OuteTTSVoiceProfile { + std::string text; + OuteTTSVoiceFeatures global_features; + std::vector words; +}; + +struct OuteTTSTextGenerationBudget { + int64_t words = 0; + int64_t non_whitespace_codepoints = 0; + int64_t recommended_max_new_tokens = 0; +}; + +// Mirrors the sizing heuristic used by the upstream OuteTTS 1.0 runner. Audio +// code generation is much denser than text tokenization, so a character-sized +// chunk cannot safely use the same numeric value as its generated-token cap. +OuteTTSTextGenerationBudget +estimate_text_generation_budget(std::string_view text); + +class OuteTTSTokenizer { +public: + explicit OuteTTSTokenizer(std::shared_ptr assets); + + std::vector build_prompt(const std::string &text) const; + std::vector + build_clone_prompt(const std::string &text, + const OuteTTSVoiceProfile &profile) const; + bool is_stop_token(int32_t token) const noexcept; + int32_t eos_id() const noexcept; + int32_t audio_end_id() const noexcept; + bool append_audio_code(int32_t token, std::vector &codebook1, + std::vector &codebook2) const; + +private: + struct Impl; + std::shared_ptr impl_; +}; + +} // namespace engine::models::outetts diff --git a/include/engine/models/qwen3_asr/assets.h b/include/engine/models/qwen3_asr/assets.h index bf7d1c43..2ba86525 100644 --- a/include/engine/models/qwen3_asr/assets.h +++ b/include/engine/models/qwen3_asr/assets.h @@ -84,5 +84,7 @@ std::shared_ptr load_qwen3_asr_assets(const std::filesyste std::shared_ptr load_qwen3_asr_assets( const std::filesystem::path & model_path, std::string_view package_family); +std::shared_ptr load_qwen3_asr_assets( + assets::ResourceBundle resources); } // namespace engine::models::qwen3_asr diff --git a/model_specs/outetts.json b/model_specs/outetts.json new file mode 100644 index 00000000..05f859f3 --- /dev/null +++ b/model_specs/outetts.json @@ -0,0 +1,65 @@ +{ + "family": "outetts", + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer": "model:tokenizer.json", + "tokenizer_config": "model:tokenizer_config.json", + "special_tokens_map": "model:special_tokens_map.json", + "dac_config": "model:dac/config.json" + }, + "optional_files": { + "aligner_config": "model:aligner/config.json", + "aligner_generation_config": "model:aligner/generation_config.json", + "aligner_tokenizer_config": "model:aligner/tokenizer_config.json", + "aligner_preprocessor_config": "model:aligner/preprocessor_config.json", + "aligner_processor_config": "model:aligner/processor_config.json", + "aligner_chat_template": "model:aligner/chat_template.json", + "aligner_chat_template_jinja": "model:aligner/chat_template.jinja", + "aligner_vocab": "model:aligner/vocab.json", + "aligner_merges": "model:aligner/merges.txt", + "aligner_tokenizer_json": "model:aligner/tokenizer.json" + }, + "tensors": { + "model_weights": { + "source": "weights:", + "prefix": "model_weights" + }, + "dac_weights": { + "source": "weights:", + "prefix": "dac_weights" + }, + "aligner_weights": { + "source": "weights:", + "prefix": "aligner_weights" + } + } + }, + { + "format": "safetensors", + "roots": { + "model": ".", + "dac": "../DAC.speech.v1.0" + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer": "model:tokenizer.json", + "tokenizer_config": "model:tokenizer_config.json", + "special_tokens_map": "model:special_tokens_map.json", + "dac_config": "dac:config.json" + }, + "tensors": { + "model_weights": "model:model.safetensors", + "dac_weights": "dac:model.safetensors" + } + } + ] +} diff --git a/src/framework/runtime/registry.cpp b/src/framework/runtime/registry.cpp index ed05b1fb..2c28ed1c 100644 --- a/src/framework/runtime/registry.cpp +++ b/src/framework/runtime/registry.cpp @@ -24,6 +24,7 @@ #include "engine/models/moss/moss_tts_nano/loader.h" #include "engine/models/nemotron_asr/loader.h" #include "engine/models/omnivoice/loader.h" +#include "engine/models/outetts/loader.h" #include "engine/models/pocket_tts/loader.h" #include "engine/models/qwen3_asr/loader.h" #include "engine/models/qwen3_forced_aligner/loader.h" @@ -248,6 +249,7 @@ ModelRegistry make_default_registry(const std::optional & engine::models::demucs::make_htdemucs_loader(), engine::models::roformer::make_mel_band_roformer_loader(), engine::models::omnivoice::make_omnivoice_loader(), + engine::models::outetts::make_outetts_loader(), engine::models::miocodec::make_miocodec_loader(), engine::models::miotts::make_miotts_loader(), engine::models::moss_tts_local::make_moss_tts_local_loader(), diff --git a/src/models/outetts/assets.cpp b/src/models/outetts/assets.cpp new file mode 100644 index 00000000..fb43d4a7 --- /dev/null +++ b/src/models/outetts/assets.cpp @@ -0,0 +1,177 @@ +#include "engine/models/outetts/assets.h" + +#include "engine/framework/assets/model_package.h" +#include "engine/framework/io/config.h" +#include "engine/framework/io/json.h" +#include "engine/models/qwen3_asr/assets.h" + +#include + +namespace engine::models::outetts { +namespace json = engine::io::json; +namespace { + +OuteTTSConfig parse_config(const assets::ResourceBundle &resources) { + const auto root = resources.parse_json("config"); + if (json::require_string(root, "model_type") != "llama") { + throw std::runtime_error("OuteTTS expects a LlamaForCausalLM checkpoint"); + } + OuteTTSConfig out; + out.bos_token_id = json::require_i64(root, "bos_token_id"); + out.eos_token_id = json::require_i64(root, "eos_token_id"); + out.hidden_size = json::require_i64(root, "hidden_size"); + out.intermediate_size = json::require_i64(root, "intermediate_size"); + out.max_position_embeddings = + json::require_i64(root, "max_position_embeddings"); + out.num_attention_heads = json::require_i64(root, "num_attention_heads"); + out.num_hidden_layers = json::require_i64(root, "num_hidden_layers"); + out.num_key_value_heads = json::require_i64(root, "num_key_value_heads"); + out.head_dim = json::optional_i64(root, "head_dim", + out.hidden_size / out.num_attention_heads); + out.vocab_size = json::require_i64(root, "vocab_size"); + out.rms_norm_eps = json::optional_f32(root, "rms_norm_eps", out.rms_norm_eps); + out.rope_theta = json::optional_f32(root, "rope_theta", out.rope_theta); + const auto &rope = root.require("rope_scaling"); + if (json::require_string(rope, "rope_type") != "llama3") { + throw std::runtime_error("OuteTTS expects llama3 rope scaling"); + } + out.rope_scaling.factor = json::require_f32(rope, "factor"); + out.rope_scaling.low_freq_factor = json::require_f32(rope, "low_freq_factor"); + out.rope_scaling.high_freq_factor = + json::require_f32(rope, "high_freq_factor"); + out.rope_scaling.original_max_position_embeddings = + json::require_i64(rope, "original_max_position_embeddings"); + + const auto generation = resources.parse_json("generation_config"); + out.pad_token_id = json::require_i64(generation, "pad_token_id"); + engine::io::require_positive(out.hidden_size, "OuteTTS hidden_size"); + engine::io::require_positive(out.intermediate_size, + "OuteTTS intermediate_size"); + engine::io::require_positive(out.num_hidden_layers, "OuteTTS layer count"); + engine::io::require_positive(out.num_attention_heads, + "OuteTTS attention heads"); + engine::io::require_positive(out.num_key_value_heads, "OuteTTS KV heads"); + engine::io::require_divisible(out.num_attention_heads, + out.num_key_value_heads, + "OuteTTS attention heads"); + if (out.num_attention_heads * out.head_dim != out.hidden_size) { + throw std::runtime_error( + "OuteTTS hidden size does not match attention heads times head_dim"); + } + return out; +} + +OuteTTSGenerationConfig +parse_generation(const assets::ResourceBundle &resources) { + const auto root = resources.parse_json("generation_config"); + OuteTTSGenerationConfig out; + out.temperature = json::optional_f32(root, "temperature", out.temperature); + out.repetition_penalty = + json::optional_f32(root, "repetition_penalty", out.repetition_penalty); + out.top_k = json::optional_i64(root, "top_k", out.top_k); + out.top_p = json::optional_f32(root, "top_p", out.top_p); + out.min_p = json::optional_f32(root, "min_p", out.min_p); + return out; +} + +void validate_anchors(const OuteTTSAssets &assets) { + const auto &c = assets.config; + const auto &model = *assets.model_weights; + assets::require_tensor_shape(model, "model.embed_tokens.weight", + {c.vocab_size, c.hidden_size}); + assets::require_tensor_shape(model, "model.norm.weight", {c.hidden_size}); + assets::require_tensor_shape( + model, "model.layers.0.self_attn.q_proj.weight", + {c.num_attention_heads * c.head_dim, c.hidden_size}); + assets::require_tensor_shape( + model, "model.layers.0.self_attn.k_proj.weight", + {c.num_key_value_heads * c.head_dim, c.hidden_size}); + assets::require_tensor_shape(model, "model.layers.0.mlp.gate_proj.weight", + {c.intermediate_size, c.hidden_size}); + + const auto &dac = *assets.dac_weights; + assets::require_tensor_shape(dac, "quantizer.quantizers.0.codebook.weight", + {c.codebook_size, 8}); + assets::require_tensor_shape(dac, "quantizer.quantizers.1.codebook.weight", + {c.codebook_size, 8}); + assets::require_tensor_shape(dac, "quantizer.quantizers.0.in_proj.weight_v", + {8, c.dac_latent_dim, 1}); + assets::require_tensor_shape(dac, "quantizer.quantizers.1.in_proj.weight_v", + {8, c.dac_latent_dim, 1}); + assets::require_tensor_shape(dac, "encoder.block.0.weight_v", {64, 1, 7}); + assets::require_tensor_shape(dac, "encoder.block.6.weight_v", + {c.dac_latent_dim, c.dac_latent_dim, 3}); + assets::require_tensor_shape(dac, "decoder.model.0.weight_v", + {c.dac_decoder_dim, c.dac_latent_dim, 7}); + assets::require_tensor_shape(dac, "decoder.model.6.weight_v", {1, 96, 7}); +} + +std::shared_ptr +load_embedded_aligner(const assets::ResourceBundle &resources) { + if (!resources.has_file("aligner_config")) + return nullptr; + for (const char *id : + {"aligner_generation_config", "aligner_tokenizer_config"}) { + if (!resources.has_file(id)) { + throw std::runtime_error( + std::string("OuteTTS embedded aligner is missing resource: ") + id); + } + } + if (!resources.has_file("aligner_preprocessor_config") && + !resources.has_file("aligner_processor_config")) { + throw std::runtime_error( + "OuteTTS embedded aligner is missing its processor configuration"); + } + if (!(resources.has_file("aligner_tokenizer_json") || + (resources.has_file("aligner_vocab") && + resources.has_file("aligner_merges")))) { + throw std::runtime_error( + "OuteTTS embedded aligner is missing tokenizer resources"); + } + + const auto source = resources.open_tensor_source("aligner_weights"); + if (source->tensors().empty()) { + throw std::runtime_error( + "OuteTTS embedded aligner has no aligner_weights tensors"); + } + assets::ResourceBundle aligner(resources.model_root()); + const auto add_required = [&](const char *target, const char *source_id) { + aligner.add_file(target, resources.require_file(source_id)); + }; + const auto add_optional = [&](const char *target, const char *source_id) { + if (resources.has_file(source_id)) + aligner.add_file(target, resources.require_file(source_id)); + }; + add_required("config", "aligner_config"); + add_required("generation_config", "aligner_generation_config"); + add_required("tokenizer_config", "aligner_tokenizer_config"); + add_optional("preprocessor_config", "aligner_preprocessor_config"); + add_optional("processor_config", "aligner_processor_config"); + add_optional("chat_template", "aligner_chat_template"); + add_optional("chat_template_jinja", "aligner_chat_template_jinja"); + add_optional("vocab", "aligner_vocab"); + add_optional("merges", "aligner_merges"); + add_optional("tokenizer_json", "aligner_tokenizer_json"); + aligner.add_tensor_source( + "weights", resources.require_file("aligner_weights"), "aligner_weights"); + return engine::models::qwen3_asr::load_qwen3_asr_assets(std::move(aligner)); +} + +} // namespace + +std::shared_ptr +load_outetts_assets(const std::filesystem::path &model_path) { + auto resources = assets::load_resource_bundle_from_package_spec( + model_path, assets::default_model_package_spec_path("outetts")); + OuteTTSAssets model_assets; + model_assets.config = parse_config(resources); + model_assets.generation = parse_generation(resources); + model_assets.model_weights = resources.open_tensor_source("model_weights"); + model_assets.dac_weights = resources.open_tensor_source("dac_weights"); + model_assets.embedded_aligner = load_embedded_aligner(resources); + model_assets.resources = std::move(resources); + validate_anchors(model_assets); + return std::make_shared(std::move(model_assets)); +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/dac.cpp b/src/models/outetts/dac.cpp new file mode 100644 index 00000000..2f29880d --- /dev/null +++ b/src/models/outetts/dac.cpp @@ -0,0 +1,701 @@ +#include "engine/models/outetts/dac.h" + +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/resampling.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace engine::models::outetts { +namespace { + +struct GgmlContextDeleter { + void operator()(ggml_context *ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +struct SnakeWeights { + core::TensorValue alpha; +}; +struct ConvWeights { + modules::Conv1dWeights value; + int64_t in_channels = 0; + int64_t out_channels = 0; + int64_t kernel = 0; +}; +struct ConvTransposeWeights { + modules::ConvTranspose1dWeights value; + int64_t in_channels = 0; + int64_t out_channels = 0; + int64_t kernel = 0; +}; +struct ResidualWeights { + SnakeWeights snake1; + ConvWeights conv1; + SnakeWeights snake2; + ConvWeights conv2; +}; +struct DecoderBlockWeights { + SnakeWeights snake; + ConvTransposeWeights up; + std::vector residuals; + int stride = 1; +}; +struct EncoderBlockWeights { + std::vector residuals; + SnakeWeights snake; + ConvWeights down; + int stride = 1; +}; +struct HostQuantizerWeights { + std::vector codebook; + std::vector in_weight; + std::vector in_bias; + std::vector out_weight; + std::vector out_bias; +}; +struct QuantizerWeights { + core::TensorValue codebook; + ConvWeights out_proj; +}; +struct DacWeights { + std::shared_ptr store; + std::vector quantizers; + ConvWeights first; + std::vector blocks; + SnakeWeights final_snake; + ConvWeights final_conv; + ConvWeights encoder_first; + std::vector encoder_blocks; + SnakeWeights encoder_final_snake; + ConvWeights encoder_final_conv; + std::vector host_quantizers; +}; + +std::vector fold_weight_norm(const std::vector &v, + const std::vector &g, int64_t dim0, + int64_t dim1, int64_t kernel) { + std::vector out(v.size()); + for (int64_t i = 0; i < dim0; ++i) { + const size_t base = static_cast(i * dim1 * kernel); + double norm2 = 0.0; + for (int64_t j = 0; j < dim1 * kernel; ++j) { + const double x = v[base + static_cast(j)]; + norm2 += x * x; + } + const float scale = + g[static_cast(i)] / static_cast(std::sqrt(norm2)); + for (int64_t j = 0; j < dim1 * kernel; ++j) { + out[base + static_cast(j)] = + v[base + static_cast(j)] * scale; + } + } + return out; +} + +ConvWeights load_conv(core::BackendWeightStore &store, + const assets::TensorSource &source, + const std::string &prefix, int64_t out_channels, + int64_t in_channels, int64_t kernel, + assets::TensorStorageType storage_type) { + const auto v = source.require_f32(prefix + ".weight_v", + {out_channels, in_channels, kernel}); + const auto g = source.require_f32(prefix + ".weight_g", {out_channels, 1, 1}); + ConvWeights out; + out.in_channels = in_channels; + out.out_channels = out_channels; + out.kernel = kernel; + out.value.weight = store.make_from_f32( + core::TensorShape::from_dims({out_channels, in_channels, kernel}), + storage_type, fold_weight_norm(v, g, out_channels, in_channels, kernel)); + out.value.bias = + store.load_f32_tensor(source, prefix + ".bias", {out_channels}); + return out; +} + +ConvTransposeWeights load_conv_transpose( + core::BackendWeightStore &store, const assets::TensorSource &source, + const std::string &prefix, int64_t in_channels, int64_t out_channels, + int64_t kernel, assets::TensorStorageType storage_type) { + const auto v = source.require_f32(prefix + ".weight_v", + {in_channels, out_channels, kernel}); + const auto g = source.require_f32(prefix + ".weight_g", {in_channels, 1, 1}); + ConvTransposeWeights out; + out.in_channels = in_channels; + out.out_channels = out_channels; + out.kernel = kernel; + out.value.weight = store.make_from_f32( + core::TensorShape::from_dims({in_channels, out_channels, kernel}), + storage_type, fold_weight_norm(v, g, in_channels, out_channels, kernel)); + out.value.bias = + store.load_f32_tensor(source, prefix + ".bias", {out_channels}); + return out; +} + +SnakeWeights load_snake(core::BackendWeightStore &store, + const assets::TensorSource &source, + const std::string &name, int64_t channels) { + return {store.make_from_f32(core::TensorShape::from_dims({channels}), + assets::TensorStorageType::F32, + source.require_f32(name, {1, channels, 1}))}; +} + +ResidualWeights load_residual(core::BackendWeightStore &store, + const assets::TensorSource &source, + const std::string &prefix, int64_t channels, + assets::TensorStorageType storage_type) { + ResidualWeights out; + out.snake1 = load_snake(store, source, prefix + ".block.0.alpha", channels); + out.conv1 = load_conv(store, source, prefix + ".block.1", channels, channels, + 7, storage_type); + out.snake2 = load_snake(store, source, prefix + ".block.2.alpha", channels); + out.conv2 = load_conv(store, source, prefix + ".block.3", channels, channels, + 1, storage_type); + return out; +} + +DacWeights load_weights(const OuteTTSAssets &assets, + core::ExecutionContext &execution, size_t context_bytes, + assets::TensorStorageType storage_type) { + const auto &source = *assets.dac_weights; + DacWeights out; + out.store = std::make_shared( + execution.backend(), execution.backend_type(), "outetts.dac.weights", + context_bytes); + for (int i = 0; i < 2; ++i) { + const std::string p = "quantizer.quantizers." + std::to_string(i); + QuantizerWeights q; + q.codebook = + out.store->load_tensor(source, p + ".codebook.weight", + assets::TensorStorageType::F32, {1024, 8}); + q.out_proj = load_conv(*out.store, source, p + ".out_proj", 1024, 8, 1, + storage_type); + out.quantizers.push_back(std::move(q)); + } + out.first = load_conv(*out.store, source, "decoder.model.0", 1536, 1024, 7, + storage_type); + const int strides[] = {8, 5, 4, 2}; + int64_t in_channels = 1536; + for (int stage = 0; stage < 4; ++stage) { + const int64_t out_channels = in_channels / 2; + const std::string p = "decoder.model." + std::to_string(stage + 1); + DecoderBlockWeights block; + block.stride = strides[stage]; + block.snake = + load_snake(*out.store, source, p + ".block.0.alpha", in_channels); + block.up = + load_conv_transpose(*out.store, source, p + ".block.1", in_channels, + out_channels, 2 * strides[stage], storage_type); + for (int residual = 0; residual < 3; ++residual) { + block.residuals.push_back(load_residual( + *out.store, source, p + ".block." + std::to_string(residual + 2), + out_channels, storage_type)); + } + out.blocks.push_back(std::move(block)); + in_channels = out_channels; + } + out.final_snake = load_snake(*out.store, source, "decoder.model.5.alpha", 96); + out.final_conv = + load_conv(*out.store, source, "decoder.model.6", 1, 96, 7, storage_type); + + out.encoder_first = + load_conv(*out.store, source, "encoder.block.0", 64, 1, 7, storage_type); + const int encoder_strides[] = {2, 4, 5, 8}; + int64_t encoder_channels = 64; + for (int stage = 0; stage < 4; ++stage) { + const int64_t out_channels = encoder_channels * 2; + const std::string p = "encoder.block." + std::to_string(stage + 1); + EncoderBlockWeights block; + block.stride = encoder_strides[stage]; + for (int residual_index = 0; residual_index < 3; ++residual_index) { + block.residuals.push_back(load_residual( + *out.store, source, p + ".block." + std::to_string(residual_index), + encoder_channels, storage_type)); + } + block.snake = + load_snake(*out.store, source, p + ".block.3.alpha", encoder_channels); + block.down = + load_conv(*out.store, source, p + ".block.4", out_channels, + encoder_channels, 2 * encoder_strides[stage], storage_type); + out.encoder_blocks.push_back(std::move(block)); + encoder_channels = out_channels; + } + out.encoder_final_snake = + load_snake(*out.store, source, "encoder.block.5.alpha", 1024); + out.encoder_final_conv = load_conv(*out.store, source, "encoder.block.6", + 1024, 1024, 3, storage_type); + + for (int i = 0; i < 2; ++i) { + const std::string p = "quantizer.quantizers." + std::to_string(i); + HostQuantizerWeights q; + q.codebook = source.require_f32(p + ".codebook.weight", {1024, 8}); + q.in_weight = fold_weight_norm( + source.require_f32(p + ".in_proj.weight_v", {8, 1024, 1}), + source.require_f32(p + ".in_proj.weight_g", {8, 1, 1}), 8, 1024, 1); + q.in_bias = source.require_f32(p + ".in_proj.bias", {8}); + q.out_weight = fold_weight_norm( + source.require_f32(p + ".out_proj.weight_v", {1024, 8, 1}), + source.require_f32(p + ".out_proj.weight_g", {1024, 1, 1}), 1024, 8, 1); + q.out_bias = source.require_f32(p + ".out_proj.bias", {1024}); + out.host_quantizers.push_back(std::move(q)); + } + out.store->upload(); + return out; +} + +core::TensorValue snake(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const SnakeWeights &weights) { + const int64_t channels = input.shape.dims[1]; + auto alpha = core::reshape_tensor( + ctx, weights.alpha, core::TensorShape::from_dims({1, channels, 1})); + alpha = core::wrap_tensor(ggml_repeat(ctx.ggml, alpha.tensor, input.tensor), + input.shape, GGML_TYPE_F32); + auto ax = core::wrap_tensor(ggml_mul(ctx.ggml, input.tensor, alpha.tensor), + input.shape, GGML_TYPE_F32); + auto s = core::wrap_tensor(ggml_sin(ctx.ggml, ax.tensor), input.shape, + GGML_TYPE_F32); + auto s2 = core::wrap_tensor(ggml_mul(ctx.ggml, s.tensor, s.tensor), + input.shape, GGML_TYPE_F32); + auto denom = + core::wrap_tensor(ggml_scale_bias(ctx.ggml, alpha.tensor, 1.0F, 1.0e-9F), + input.shape, GGML_TYPE_F32); + auto frac = core::wrap_tensor(ggml_div(ctx.ggml, s2.tensor, denom.tensor), + input.shape, GGML_TYPE_F32); + return core::wrap_tensor(ggml_add(ctx.ggml, input.tensor, frac.tensor), + input.shape, GGML_TYPE_F32); +} + +core::TensorValue conv(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const ConvWeights &weights, int padding, + int dilation = 1) { + return modules::Conv1dModule({ + weights.in_channels, + weights.out_channels, + weights.kernel, + 1, + padding, + dilation, + true, + }) + .build(ctx, input, weights.value); +} + +core::TensorValue strided_conv(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const ConvWeights &weights, int stride, + int padding) { + return modules::Conv1dModule({ + weights.in_channels, + weights.out_channels, + weights.kernel, + stride, + padding, + 1, + true, + }) + .build(ctx, input, weights.value); +} + +core::TensorValue residual(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const ResidualWeights &weights, int dilation) { + auto x = snake(ctx, input, weights.snake1); + x = conv(ctx, x, weights.conv1, 3 * dilation, dilation); + x = snake(ctx, x, weights.snake2); + x = conv(ctx, x, weights.conv2, 0); + return modules::AddModule{}.build(ctx, input, x); +} + +std::vector load_reference_audio(const runtime::AudioBuffer &input) { + if (input.sample_rate <= 0 || input.channels <= 0 || input.samples.empty()) { + throw std::runtime_error( + "OuteTTS voice cloning requires non-empty reference audio"); + } + auto samples = audio::mixdown_interleaved_to_mono_average( + input.samples, input.channels, audio::MonoMixAccumulation::Float64); + if (input.sample_rate != 24000) { + audio::SoxrResampleOptions options; + options.output_length_policy = audio::SoxrOutputLengthPolicy::ActualOutput; + options.reject_empty_output = true; + options.warning_context = "OuteTTS reference audio"; + options.fallback_description = "linear reference-audio resampling"; + samples = audio::resample_mono_soxr_or_linear(samples, input.sample_rate, + 24000, options); + } + constexpr size_t max_samples = 20u * 24000u; + if (samples.size() > max_samples) { + throw std::runtime_error( + "OuteTTS reference audio is longer than the supported 20 seconds"); + } + return samples; +} + +std::vector prepare_reference_audio(const std::vector &input) { + auto samples = input; + double sum_sq = 0.0; + for (const float sample : samples) + sum_sq += static_cast(sample) * sample; + const float rms = static_cast( + std::sqrt(sum_sq / static_cast(samples.size()))); + if (rms > 1.0e-6F) { + const float scale = std::pow(10.0F, -18.0F / 20.0F) / rms; + for (float &sample : samples) + sample *= scale; + } + float peak = 0.0F; + for (const float sample : samples) + peak = std::max(peak, std::fabs(sample)); + const float peak_limit = std::pow(10.0F, -1.0F / 20.0F); + if (peak > peak_limit) { + const float scale = peak_limit / peak; + for (float &sample : samples) + sample *= scale; + } + samples.resize(((samples.size() + 319u) / 320u) * 320u, 0.0F); + return samples; +} + +std::pair, std::vector> +quantize_reference(const std::vector &frame_major, + const std::vector &quantizers) { + constexpr int hidden_size = 1024; + constexpr int codebook_size = 1024; + constexpr int codebook_dim = 8; + if (frame_major.empty() || frame_major.size() % hidden_size != 0 || + quantizers.size() != 2) { + throw std::runtime_error( + "OuteTTS DAC encoder produced invalid acoustic latents"); + } + const size_t frames = frame_major.size() / hidden_size; + std::vector residual_values = frame_major; + std::vector outputs[2] = { + std::vector(frames), + std::vector(frames), + }; + for (size_t q_index = 0; q_index < quantizers.size(); ++q_index) { + const auto &q = quantizers[q_index]; +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int64_t frame = 0; frame < static_cast(frames); ++frame) { + float projected[codebook_dim]; + float projected_norm_sq = 0.0F; + const size_t frame_offset = static_cast(frame) * hidden_size; + for (int d = 0; d < codebook_dim; ++d) { + float value = q.in_bias[static_cast(d)]; + const size_t weight_offset = static_cast(d) * hidden_size; + for (int h = 0; h < hidden_size; ++h) { + value += q.in_weight[weight_offset + static_cast(h)] * + residual_values[frame_offset + static_cast(h)]; + } + projected[d] = value; + projected_norm_sq += value * value; + } + const float projected_norm = std::sqrt(projected_norm_sq) + 1.0e-12F; + int best_code = 0; + float best_distance = std::numeric_limits::infinity(); + for (int code = 0; code < codebook_size; ++code) { + const size_t code_offset = static_cast(code) * codebook_dim; + float embedding_norm_sq = 0.0F; + for (int d = 0; d < codebook_dim; ++d) { + const float value = q.codebook[code_offset + static_cast(d)]; + embedding_norm_sq += value * value; + } + const float embedding_norm = std::sqrt(embedding_norm_sq) + 1.0e-12F; + float distance = 0.0F; + for (int d = 0; d < codebook_dim; ++d) { + const float difference = + projected[d] / projected_norm - + q.codebook[code_offset + static_cast(d)] / embedding_norm; + distance += difference * difference; + } + if (distance < best_distance) { + best_distance = distance; + best_code = code; + } + } + outputs[q_index][static_cast(frame)] = best_code; + } +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int64_t frame = 0; frame < static_cast(frames); ++frame) { + const int code = outputs[q_index][static_cast(frame)]; + const size_t code_offset = static_cast(code) * codebook_dim; + const size_t frame_offset = static_cast(frame) * hidden_size; + for (int out = 0; out < hidden_size; ++out) { + float value = q.out_bias[static_cast(out)]; + const size_t weight_offset = static_cast(out) * codebook_dim; + for (int d = 0; d < codebook_dim; ++d) { + value += q.out_weight[weight_offset + static_cast(d)] * + q.codebook[code_offset + static_cast(d)]; + } + residual_values[frame_offset + static_cast(out)] -= value; + } + } + } + return {std::move(outputs[0]), std::move(outputs[1])}; +} + +void normalize_decoded_audio(std::vector &audio) { + if (audio.empty()) { + return; + } + double sum_sq = 0.0; + for (const float sample : audio) { + sum_sq += static_cast(sample) * static_cast(sample); + } + const float rms = + static_cast(std::sqrt(sum_sq / static_cast(audio.size()))); + if (rms > 1.0e-6F) { + const float target_rms = std::pow(10.0F, -18.0F / 20.0F); + const float scale = target_rms / rms; + for (float &sample : audio) { + sample *= scale; + } + } + float peak = 0.0F; + for (const float sample : audio) { + peak = std::max(peak, std::fabs(sample)); + } + const float peak_limit = std::pow(10.0F, -1.0F / 20.0F); + if (peak > peak_limit && peak > 1.0e-6F) { + const float scale = peak_limit / peak; + for (float &sample : audio) { + sample *= scale; + } + } +} + +} // namespace + +struct OuteTTSDacDecoder::Impl { + Impl(std::shared_ptr assets_in, + core::ExecutionContext &execution_in, size_t weight_context_bytes, + size_t graph_context_bytes_in, assets::TensorStorageType storage_type) + : assets(std::move(assets_in)), execution(execution_in), + graph_context_bytes(graph_context_bytes_in), + weights(load_weights(*assets, execution, weight_context_bytes, + storage_type)) {} + + OuteTTSDacDecoder::EncodedReference + encode_reference(const runtime::AudioBuffer &input_audio) { + auto feature_samples = load_reference_audio(input_audio); + auto samples = prepare_reference_audio(feature_samples); + const int64_t sample_count = static_cast(samples.size()); + ggml_init_params params{graph_context_bytes, nullptr, true}; + std::unique_ptr ctx(ggml_init(params)); + if (!ctx) { + throw std::runtime_error( + "failed to create OuteTTS DAC encoder graph context"); + } + core::ModuleBuildContext build{ctx.get(), "outetts.dac.encode"}; + auto *pcm = + ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, sample_count, 1, 1); + auto x = core::wrap_tensor( + pcm, core::TensorShape::from_dims({1, 1, sample_count}), GGML_TYPE_F32); + x = conv(build, x, weights.encoder_first, 3); + for (const auto &block : weights.encoder_blocks) { + x = residual(build, x, block.residuals[0], 1); + x = residual(build, x, block.residuals[1], 3); + x = residual(build, x, block.residuals[2], 9); + x = snake(build, x, block.snake); + x = strided_conv(build, x, block.down, block.stride, + (block.stride + 1) / 2); + } + x = snake(build, x, weights.encoder_final_snake); + x = conv(build, x, weights.encoder_final_conv, 1); + x = core::ensure_backend_addressable_layout(build, x); + ggml_set_output(x.tensor); + auto *graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, x.tensor); + ggml_gallocr_t allocator = ggml_gallocr_new( + ggml_backend_get_default_buffer_type(execution.backend())); + if (allocator == nullptr || !ggml_gallocr_reserve(allocator, graph) || + !ggml_gallocr_alloc_graph(allocator, graph)) { + if (allocator != nullptr) + ggml_gallocr_free(allocator); + throw std::runtime_error("failed to allocate OuteTTS DAC encoder graph"); + } + ggml_backend_tensor_set(pcm, samples.data(), 0, + samples.size() * sizeof(float)); + core::set_backend_threads(execution.backend(), + std::max(1, execution.config().threads)); + const auto status = core::compute_backend_graph(execution.backend(), graph); + ggml_backend_synchronize(execution.backend()); + if (status != GGML_STATUS_SUCCESS) { + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + throw std::runtime_error("OuteTTS DAC encoder graph compute failed"); + } + const int64_t frames = x.shape.dims[2]; + if (x.shape.rank != 3 || x.shape.dims[0] != 1 || x.shape.dims[1] != 1024 || + frames <= 0) { + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + throw std::runtime_error( + "OuteTTS DAC encoder produced an unexpected shape"); + } + std::vector channel_major(static_cast(frames) * 1024u); + ggml_backend_tensor_get(x.tensor, channel_major.data(), 0, + channel_major.size() * sizeof(float)); + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + + std::vector frame_major(channel_major.size()); +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int64_t frame = 0; frame < frames; ++frame) { + for (int64_t channel = 0; channel < 1024; ++channel) { + frame_major[static_cast(frame * 1024 + channel)] = + channel_major[static_cast(channel * frames + frame)]; + } + } + auto codes = quantize_reference(frame_major, weights.host_quantizers); + OuteTTSDacDecoder::EncodedReference result; + // OuteTTS derives conditioning features from the resampled source audio, + // while the DAC encoder receives a separately loudness-normalized copy. + result.samples = std::move(feature_samples); + result.codebook1 = std::move(codes.first); + result.codebook2 = std::move(codes.second); + return result; + } + + runtime::AudioBuffer decode(const std::vector &c1, + const std::vector &c2) { + const int64_t frames = static_cast(std::min(c1.size(), c2.size())); + if (frames <= 0) { + throw std::runtime_error( + "OuteTTS DAC requires at least one complete code pair"); + } + ggml_init_params params{graph_context_bytes, nullptr, true}; + std::unique_ptr ctx(ggml_init(params)); + if (!ctx) { + throw std::runtime_error("failed to create OuteTTS DAC graph context"); + } + core::ModuleBuildContext build{ctx.get(), "outetts.dac.decode"}; + auto *ids1 = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_I32, frames, 1); + auto *ids2 = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_I32, frames, 1); + auto id_value1 = core::wrap_tensor( + ids1, core::TensorShape::from_dims({1, frames}), GGML_TYPE_I32); + auto id_value2 = core::wrap_tensor( + ids2, core::TensorShape::from_dims({1, frames}), GGML_TYPE_I32); + auto e1 = modules::EmbeddingModule({1024, 8}).build( + build, id_value1, weights.quantizers[0].codebook); + auto e2 = modules::EmbeddingModule({1024, 8}).build( + build, id_value2, weights.quantizers[1].codebook); + e1 = modules::TransposeModule({{0, 2, 1}, 3}).build(build, e1); + e2 = modules::TransposeModule({{0, 2, 1}, 3}).build(build, e2); + auto z1 = conv(build, core::ensure_backend_addressable_layout(build, e1), + weights.quantizers[0].out_proj, 0); + auto z2 = conv(build, core::ensure_backend_addressable_layout(build, e2), + weights.quantizers[1].out_proj, 0); + auto x = modules::AddModule{}.build(build, z1, z2); + x = conv(build, x, weights.first, 3); + for (const auto &block : weights.blocks) { + x = snake(build, x, block.snake); + auto upsampled = modules::ConvTranspose1dModule({ + block.up.in_channels, + block.up.out_channels, + block.up.kernel, + block.stride, + 0, + 1, + true, + }) + .build(build, x, block.up.value); + const int padding = static_cast(std::ceil(block.stride / 2.0)); + const int64_t target_frames = upsampled.shape.dims[2] - 2 * padding; + x = modules::SliceModule({2, padding, target_frames}) + .build(build, upsampled); + x = residual(build, x, block.residuals[0], 1); + x = residual(build, x, block.residuals[1], 3); + x = residual(build, x, block.residuals[2], 9); + } + x = snake(build, x, weights.final_snake); + x = conv(build, x, weights.final_conv, 3); + x = modules::TanhModule{}.build(build, x); + x = core::ensure_backend_addressable_layout(build, x); + ggml_set_output(x.tensor); + auto *graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, x.tensor); + ggml_gallocr_t allocator = ggml_gallocr_new( + ggml_backend_get_default_buffer_type(execution.backend())); + if (allocator == nullptr || !ggml_gallocr_reserve(allocator, graph) || + !ggml_gallocr_alloc_graph(allocator, graph)) { + if (allocator != nullptr) + ggml_gallocr_free(allocator); + throw std::runtime_error("failed to allocate OuteTTS DAC graph"); + } + ggml_backend_tensor_set(ids1, c1.data(), 0, + static_cast(frames) * sizeof(int32_t)); + ggml_backend_tensor_set(ids2, c2.data(), 0, + static_cast(frames) * sizeof(int32_t)); + core::set_backend_threads(execution.backend(), + std::max(1, execution.config().threads)); + const auto status = core::compute_backend_graph(execution.backend(), graph); + ggml_backend_synchronize(execution.backend()); + if (status != GGML_STATUS_SUCCESS) { + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + throw std::runtime_error("OuteTTS DAC graph compute failed"); + } + runtime::AudioBuffer audio; + audio.sample_rate = 24000; + audio.channels = 1; + audio.samples.resize(static_cast(x.shape.dims[2])); + ggml_backend_tensor_get(x.tensor, audio.samples.data(), 0, + audio.samples.size() * sizeof(float)); + normalize_decoded_audio(audio.samples); + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + return audio; + } + + std::shared_ptr assets; + core::ExecutionContext &execution; + size_t graph_context_bytes; + DacWeights weights; +}; + +OuteTTSDacDecoder::OuteTTSDacDecoder( + std::shared_ptr assets, + core::ExecutionContext &execution_context, size_t weight_context_bytes, + size_t graph_context_bytes, assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique(std::move(assets), execution_context, + weight_context_bytes, graph_context_bytes, + weight_storage_type)) {} + +OuteTTSDacDecoder::~OuteTTSDacDecoder() = default; + +runtime::AudioBuffer +OuteTTSDacDecoder::decode(const std::vector &codebook1, + const std::vector &codebook2) { + return impl_->decode(codebook1, codebook2); +} + +OuteTTSDacDecoder::EncodedReference +OuteTTSDacDecoder::encode_reference(const runtime::AudioBuffer &audio) { + return impl_->encode_reference(audio); +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/llama.cpp b/src/models/outetts/llama.cpp new file mode 100644 index 00000000..cd9c23ab --- /dev/null +++ b/src/models/outetts/llama.cpp @@ -0,0 +1,686 @@ +#include "engine/models/outetts/llama.h" + +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/debug/trace.h" +#include "engine/framework/modules/attention/qwen_causal_decoder.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/weight_binding.h" +#include "engine/framework/sampling/torch_random.h" +#include "../common/constant_tensor_cache.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::outetts { +namespace { + +namespace binding = modules::binding; + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +struct LayerWeights { + assets::TensorDataF32 input_norm; + modules::AttentionWeights attention; + assets::TensorDataF32 post_norm; + modules::LinearWeights gate; + modules::LinearWeights up; + modules::LinearWeights down; +}; + +struct ModelWeights { + std::shared_ptr store; + core::TensorValue embedding; + core::TensorValue lm_head; + core::TensorValue rope_factors; + std::vector layers; + assets::TensorDataF32 norm; +}; + +struct PrefillOutput { + std::vector logits; + runtime::TransformerKVState state; +}; + +std::vector llama3_rope_factors(const OuteTTSConfig & config) { + constexpr double pi = 3.14159265358979323846; + const auto & scaling = config.rope_scaling; + const double low_wavelength = static_cast(scaling.original_max_position_embeddings) / + static_cast(scaling.low_freq_factor); + const double high_wavelength = static_cast(scaling.original_max_position_embeddings) / + static_cast(scaling.high_freq_factor); + std::vector factors(static_cast(config.head_dim / 2), 1.0F); + for (int64_t i = 0; i < config.head_dim / 2; ++i) { + const double inv_freq = 1.0 / std::pow( + static_cast(config.rope_theta), + static_cast(2 * i) / static_cast(config.head_dim)); + const double wavelength = 2.0 * pi / inv_freq; + double scaled = inv_freq; + if (wavelength > low_wavelength) { + scaled = inv_freq / static_cast(scaling.factor); + } else if (wavelength >= high_wavelength) { + const double smooth = + (static_cast(scaling.original_max_position_embeddings) / wavelength - + static_cast(scaling.low_freq_factor)) / + (static_cast(scaling.high_freq_factor) - + static_cast(scaling.low_freq_factor)); + scaled = (1.0 - smooth) * inv_freq / static_cast(scaling.factor) + smooth * inv_freq; + } + // ggml divides its base inverse frequency by this tensor. This matches + // llama.cpp's rope_freqs.weight representation for Llama-3 scaling. + factors[static_cast(i)] = static_cast(inv_freq / scaled); + } + return factors; +} + +ModelWeights load_weights( + const OuteTTSAssets & assets, + ggml_backend_t backend, + core::BackendType backend_type, + size_t context_bytes, + assets::TensorStorageType storage_type) { + const auto & c = assets.config; + const auto & source = *assets.model_weights; + ModelWeights out; + out.store = std::make_shared( + backend, backend_type, "outetts.llama.weights", context_bytes); + out.embedding = out.store->load_tensor( + source, "model.embed_tokens.weight", storage_type, {c.vocab_size, c.hidden_size}); + out.lm_head = out.embedding; + out.rope_factors = out.store->make_from_f32( + core::TensorShape::from_dims({c.head_dim / 2}), + assets::TensorStorageType::F32, + llama3_rope_factors(c)); + out.layers.reserve(static_cast(c.num_hidden_layers)); + for (int64_t i = 0; i < c.num_hidden_layers; ++i) { + const std::string p = "model.layers." + std::to_string(i); + LayerWeights layer; + layer.input_norm = source.require_f32_tensor(p + ".input_layernorm.weight", {c.hidden_size}); + layer.attention.q_weight = out.store->load_tensor( + source, p + ".self_attn.q_proj.weight", storage_type, + {c.num_attention_heads * c.head_dim, c.hidden_size}); + layer.attention.k_weight = out.store->load_tensor( + source, p + ".self_attn.k_proj.weight", storage_type, + {c.num_key_value_heads * c.head_dim, c.hidden_size}); + layer.attention.v_weight = out.store->load_tensor( + source, p + ".self_attn.v_proj.weight", storage_type, + {c.num_key_value_heads * c.head_dim, c.hidden_size}); + layer.attention.out_weight = out.store->load_tensor( + source, p + ".self_attn.o_proj.weight", storage_type, + {c.hidden_size, c.num_attention_heads * c.head_dim}); + layer.post_norm = source.require_f32_tensor(p + ".post_attention_layernorm.weight", {c.hidden_size}); + layer.gate.weight = out.store->load_tensor( + source, p + ".mlp.gate_proj.weight", storage_type, {c.intermediate_size, c.hidden_size}); + layer.up.weight = out.store->load_tensor( + source, p + ".mlp.up_proj.weight", storage_type, {c.intermediate_size, c.hidden_size}); + layer.down.weight = out.store->load_tensor( + source, p + ".mlp.down_proj.weight", storage_type, {c.hidden_size, c.intermediate_size}); + out.layers.push_back(std::move(layer)); + } + out.norm = source.require_f32_tensor("model.norm.weight", {c.hidden_size}); + out.store->upload(); + return out; +} + +modules::QwenCausalDecoderConfig decoder_config(const OuteTTSConfig & c) { + modules::QwenCausalDecoderConfig out; + out.stack.hidden_size = c.hidden_size; + out.stack.intermediate_size = c.intermediate_size; + out.stack.num_attention_heads = c.num_attention_heads; + out.stack.num_key_value_heads = c.num_key_value_heads; + out.stack.head_dim = c.head_dim; + out.stack.layers = c.num_hidden_layers; + out.stack.rms_norm_eps = c.rms_norm_eps; + out.stack.rope_theta = c.rope_theta; + // Hugging Face Llama weights use split-half rotary pairs. llama.cpp's + // dedicated Llama converter permutes Q/K and then uses NORMAL RoPE, while + // audio.cpp preserves the source tensor layout in safetensors and GGUF. + out.stack.rope_type = GGML_ROPE_TYPE_NEOX; + out.stack.use_qk_norm = false; + out.stack.runtime.attention.prefill_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.attention.static_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.static_cache.update_mode = modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; + out.logits_size = c.vocab_size; + return out; +} + +modules::QwenCausalDecoderWeights graph_weights( + const ModelWeights & weights, + common::ConstantTensorCache & constants) { + modules::QwenCausalDecoderWeights out; + out.stack.layers.reserve(weights.layers.size()); + for (const auto & source : weights.layers) { + modules::QwenDecoderLayerWeights layer; + layer.input_norm = binding::norm_data(constants, source.input_norm); + layer.self_attention = source.attention; + layer.post_norm = binding::norm_data(constants, source.post_norm); + layer.mlp.gate_proj = source.gate; + layer.mlp.up_proj = source.up; + layer.mlp.down_proj = source.down; + layer.rope_frequency_factors = weights.rope_factors; + out.stack.layers.push_back(std::move(layer)); + } + out.final_norm = binding::norm_data(constants, weights.norm); + out.lm_head.weight = weights.lm_head; + return out; +} + +struct SamplingScratch { + std::vector repeated_ids; + std::vector order; + std::vector probabilities; +}; + +void apply_repetition_penalty( + std::vector & logits, + const std::vector & ids, + int64_t window, + float penalty, + std::vector & repeated_ids) { + if (penalty == 1.0F || window == 0) { + return; + } + const size_t begin = ids.size() > static_cast(window) ? ids.size() - static_cast(window) : 0; + repeated_ids.clear(); + for (auto it = ids.begin() + static_cast(begin); it != ids.end(); ++it) { + if (std::find(repeated_ids.begin(), repeated_ids.end(), *it) == repeated_ids.end()) { + repeated_ids.push_back(*it); + } + } + for (const int32_t id : repeated_ids) { + if (id < 0 || static_cast(id) >= logits.size()) { + continue; + } + float & value = logits[static_cast(id)]; + value = value <= 0.0F ? value * penalty : value / penalty; + } +} + +int32_t sample_token( + const std::vector & logits, + const OuteTTSGenerateOptions & o, + std::mt19937 & rng, + const sampling::TorchCudaSamplingPolicy & sampling_policy, + uint64_t call_index, + SamplingScratch & scratch) { + if (!(o.temperature > 0.0F) || !std::isfinite(o.temperature)) { + return static_cast(std::max_element(logits.begin(), logits.end()) - logits.begin()); + } + auto & order = scratch.order; + order.resize(logits.size()); + std::iota(order.begin(), order.end(), 0); + const size_t kept_by_top_k = + o.top_k > 0 && static_cast(o.top_k) < order.size() + ? static_cast(o.top_k) + : order.size(); + const auto by_logit_desc = [&](size_t a, size_t b) { + return logits[a] > logits[b]; + }; + if (kept_by_top_k < order.size()) { + std::partial_sort( + order.begin(), + order.begin() + static_cast(kept_by_top_k), + order.end(), + by_logit_desc); + order.resize(kept_by_top_k); + } else { + std::sort(order.begin(), order.end(), by_logit_desc); + } + const float max_logit = logits[order.front()]; + auto & probabilities = scratch.probabilities; + probabilities.assign(order.size(), 0.0F); + double sum = 0.0; + for (size_t i = 0; i < order.size(); ++i) { + probabilities[i] = std::exp((logits[order[i]] - max_logit) / o.temperature); + sum += probabilities[i]; + } + for (float & probability : probabilities) { + probability = static_cast(probability / sum); + } + const float max_probability = probabilities.front(); + float cumulative = 0.0F; + size_t kept = 0; + for (const float probability : probabilities) { + if (probability < max_probability * o.min_p && kept > 0) { + break; + } + cumulative += probability; + ++kept; + if (o.top_p < 1.0F && cumulative >= o.top_p) { + break; + } + } + order.resize(std::max(1, kept)); + probabilities.resize(order.size()); + if (sampling_policy.cuda_fast_path) { + double best_rank = -std::numeric_limits::infinity(); + int32_t best_token = -1; + for (size_t index = 0; index < order.size(); ++index) { + const float exponential = + sampling::torch_cuda_tensor_iterator_exponential_element( + o.seed, + static_cast(logits.size()), + static_cast(order[index]), + call_index, + sampling_policy.multiprocessor_count, + sampling_policy.max_threads_per_multiprocessor); + const double rank = + static_cast(probabilities[index]) / + static_cast(exponential); + if (rank > best_rank) { + best_rank = rank; + best_token = static_cast(order[index]); + } + } + if (best_token < 0) { + throw std::runtime_error( + "OuteTTS CUDA sampler failed to select a token"); + } + return best_token; + } + std::discrete_distribution distribution(probabilities.begin(), probabilities.end()); + return static_cast(order[static_cast(distribution(rng))]); +} + +class CachedStepGraph { +public: + CachedStepGraph( + const OuteTTSConfig & config, + const ModelWeights & weights, + common::ConstantTensorCache & constants, + ggml_backend_t backend, + int threads, + int64_t capacity) + : config_(config), weights_(&weights), constants_(&constants), backend_(backend), threads_(threads), capacity_(capacity) { + ggml_init_params params{1536ull * 1024ull * 1024ull, nullptr, true}; + ctx_.reset(ggml_init(params)); + if (!ctx_) throw std::runtime_error("failed to create OuteTTS cached-step context"); + core::ModuleBuildContext build{ctx_.get(), "outetts.llama.cached_step"}; + input_id_ = ggml_new_tensor_2d(ctx_.get(), GGML_TYPE_I32, 1, 1); + auto id = core::wrap_tensor(input_id_, core::TensorShape::from_dims({1, 1}), GGML_TYPE_I32); + auto x = modules::EmbeddingModule({config_.vocab_size, config_.hidden_size}).build(build, id, weights_->embedding); + positions_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, 1); + cache_slot_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, 1); + mask_ = ggml_new_tensor_4d(ctx_.get(), GGML_TYPE_F16, capacity_, 1, 1, 1); + auto position = core::wrap_tensor(positions_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + auto slot = core::wrap_tensor(cache_slot_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + auto mask = core::wrap_tensor(mask_, core::TensorShape::from_dims({1, 1, 1, capacity_}), GGML_TYPE_F16); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); + constants_->begin_graph(); + auto output = modules::QwenCausalDecoderModule(decoder_config(config_)).build_static_cache_tail( + build, graph_, x, position, graph_weights(*weights_, *constants_), capacity_, mask, slot); + cache_ = std::move(output.cache); + logits_ = output.logits.tensor; + ggml_set_output(logits_); + ggml_build_forward_expand(graph_, logits_); + constants_->finish_graph(); + constants_->ensure_uploaded(); + buffer_ = ggml_backend_alloc_ctx_tensors(ctx_.get(), backend_); + if (buffer_ == nullptr) throw std::runtime_error("failed to allocate OuteTTS cached-step graph"); + mask_values_.assign(static_cast(capacity_), ggml_fp32_to_fp16(-std::numeric_limits::infinity())); + } + + ~CachedStepGraph() { + core::release_backend_graph_resources(backend_, graph_); + if (buffer_ != nullptr) ggml_backend_buffer_free(buffer_); + } + + int64_t capacity() const noexcept { return capacity_; } + + void import_state(const runtime::TransformerKVState & state) { cache_.import_state(state); } + + void run(int32_t token, std::vector & logits) { + if (cache_.valid_steps() >= capacity_) throw std::runtime_error("OuteTTS cached-step capacity exceeded"); + ggml_backend_tensor_set(input_id_, &token, 0, sizeof(token)); + const int32_t position = static_cast(cache_.current_end()); + const int32_t slot = static_cast(cache_.valid_steps()); + ggml_backend_tensor_set(positions_, &position, 0, sizeof(position)); + ggml_backend_tensor_set(cache_slot_, &slot, 0, sizeof(slot)); + modules::write_qwen_cached_step_mask(mask_, mask_values_, capacity_, cache_.valid_steps(), cache_.valid_steps()); + core::set_backend_threads(backend_, threads_); + const auto status = core::compute_backend_graph(backend_, graph_); + ggml_backend_synchronize(backend_); + if (status != GGML_STATUS_SUCCESS) throw std::runtime_error("OuteTTS cached-step graph compute failed"); + if (logits.size() != static_cast(config_.vocab_size)) { + logits.resize(static_cast(config_.vocab_size)); + } + ggml_backend_tensor_get(logits_, logits.data(), 0, logits.size() * sizeof(float)); + cache_.advance_after_direct_append(1); + } + +private: + OuteTTSConfig config_; + const ModelWeights * weights_ = nullptr; + common::ConstantTensorCache * constants_ = nullptr; + ggml_backend_t backend_ = nullptr; + int threads_ = 1; + int64_t capacity_ = 0; + std::unique_ptr ctx_; + ggml_tensor * input_id_ = nullptr; + ggml_tensor * positions_ = nullptr; + ggml_tensor * cache_slot_ = nullptr; + ggml_tensor * mask_ = nullptr; + ggml_tensor * logits_ = nullptr; + runtime::TransformerKVCache cache_; + std::vector mask_values_; + ggml_cgraph * graph_ = nullptr; + ggml_backend_buffer_t buffer_ = nullptr; +}; + +} // namespace + +struct OuteTTSLlamaRuntime::Impl { + Impl( + std::shared_ptr assets_in, + core::BackendType backend_type, + int device, + int threads_in, + size_t weight_context_bytes, + size_t constant_context_bytes, + assets::TensorStorageType storage_type) + : assets(std::move(assets_in)), + threads(std::max(1, threads_in)), + sampling_policy( + backend_type == core::BackendType::Cuda + ? sampling::resolve_torch_cuda_sampling_policy( + backend_type, + device, + "outetts", + "OuteTTS", + sampling::TorchCudaSamplingPolicyFailureMode::StrictCuda) + : sampling::TorchCudaSamplingPolicy{}) { + if (assets == nullptr) { + throw std::runtime_error("OuteTTS Llama runtime requires assets"); + } + backend = core::init_backend({backend_type, device, threads}); + weights = load_weights(*assets, backend, backend_type, weight_context_bytes, storage_type); + constants = std::make_unique( + backend, threads, "outetts.llama.constants", constant_context_bytes); + } + + ~Impl() { + // CachedStepGraph owns backend buffers and asks the backend to release + // graph resources in its destructor. Destroy it before its constants, + // weights, and backend. The previous order left a live graph holding a + // freed backend and caused a Linux CUDA segfault during session teardown. + step_graph.reset(); + constants.reset(); + weights.store.reset(); + if (backend != nullptr) { + ggml_backend_free(backend); + } + } + + PrefillOutput prefill(const std::vector & ids) const { + using Clock = std::chrono::steady_clock; + const auto total_start = Clock::now(); + const auto & c = assets->config; + const int64_t steps = static_cast(ids.size()); + // This correctness-first graph uses full-sequence prefill. A cached-step + // graph can replace it without changing weights, sampling, or package layout. + const size_t arena = std::max( + 1024ull * 1024ull * 1024ull, + static_cast(steps) * static_cast(steps) * 256ull + 512ull * 1024ull * 1024ull); + ggml_init_params params{arena, nullptr, true}; + std::unique_ptr ctx(ggml_init(params)); + if (!ctx) { + throw std::runtime_error("failed to create OuteTTS Llama graph context"); + } + core::ModuleBuildContext build{ctx.get(), "outetts.llama.prefill"}; + auto * ids_tensor = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_I32, steps, 1); + auto ids_value = core::wrap_tensor(ids_tensor, core::TensorShape::from_dims({1, steps}), GGML_TYPE_I32); + auto x = modules::EmbeddingModule({c.vocab_size, c.hidden_size}) + .build(build, ids_value, weights.embedding); + auto * positions = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, steps); + auto position_value = core::wrap_tensor(positions, core::TensorShape::from_dims({steps}), GGML_TYPE_I32); + auto * mask = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F16, steps, steps, 1, 1); + auto mask_value = core::wrap_tensor( + mask, core::TensorShape::from_dims({1, 1, steps, steps}), GGML_TYPE_F16); + constants->begin_graph(); + auto output = modules::QwenCausalDecoderModule(decoder_config(c)).build( + build, x, position_value, graph_weights(weights, *constants), std::nullopt, mask_value); + std::vector keys; + std::vector values; + keys.reserve(output.state.layers.size()); + values.reserve(output.state.layers.size()); + for (const auto & layer : output.state.layers) { + if (!layer.key.has_value() || !layer.value.has_value()) { + throw std::runtime_error("OuteTTS prefill did not produce K/V state"); + } + auto * key = ggml_cpy(ctx.get(), layer.key->tensor, ggml_dup_tensor(ctx.get(), layer.key->tensor)); + auto * value = ggml_cpy(ctx.get(), layer.value->tensor, ggml_dup_tensor(ctx.get(), layer.value->tensor)); + ggml_set_output(key); + ggml_set_output(value); + keys.push_back(key); + values.push_back(value); + } + auto * logits = output.logits.tensor; + ggml_set_output(logits); + auto * graph = ggml_new_graph_custom(ctx.get(), 65536, false); + for (auto * key : keys) ggml_build_forward_expand(graph, key); + for (auto * value : values) ggml_build_forward_expand(graph, value); + ggml_build_forward_expand(graph, logits); + constants->finish_graph(); + constants->ensure_uploaded(); + ggml_gallocr_t allocator = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (allocator == nullptr || !ggml_gallocr_reserve(allocator, graph) || !ggml_gallocr_alloc_graph(allocator, graph)) { + if (allocator != nullptr) { + ggml_gallocr_free(allocator); + } + throw std::runtime_error("failed to allocate OuteTTS Llama graph"); + } + const auto build_end = Clock::now(); + const auto position_values = modules::qwen_position_ids(steps); + const auto mask_values = modules::qwen_causal_prefill_mask_values(1, steps); + ggml_backend_tensor_set(ids_tensor, ids.data(), 0, ids.size() * sizeof(int32_t)); + ggml_backend_tensor_set(positions, position_values.data(), 0, position_values.size() * sizeof(int32_t)); + ggml_backend_tensor_set(mask, mask_values.data(), 0, mask_values.size() * sizeof(ggml_fp16_t)); + core::set_backend_threads(backend, threads); + const auto status = core::compute_backend_graph(backend, graph); + ggml_backend_synchronize(backend); + const auto compute_end = Clock::now(); + if (status != GGML_STATUS_SUCCESS) { + core::release_backend_graph_resources(backend, graph); + ggml_gallocr_free(allocator); + throw std::runtime_error("OuteTTS Llama graph compute failed"); + } + PrefillOutput result; + result.logits.resize(static_cast(c.vocab_size)); + ggml_backend_tensor_get(logits, result.logits.data(), 0, result.logits.size() * sizeof(float)); + result.state.current_end = steps; + result.state.layers.resize(keys.size()); + const size_t layer_elements = static_cast(steps * c.num_key_value_heads * c.head_dim); + for (size_t layer = 0; layer < keys.size(); ++layer) { + auto & state = result.state.layers[layer]; + state.valid_steps = steps; + state.key.resize(layer_elements); + state.value.resize(layer_elements); + ggml_backend_tensor_get(keys[layer], state.key.data(), 0, layer_elements * sizeof(float)); + ggml_backend_tensor_get(values[layer], state.value.data(), 0, layer_elements * sizeof(float)); + } + const auto read_end = Clock::now(); + core::release_backend_graph_resources(backend, graph); + ggml_gallocr_free(allocator); + const auto release_end = Clock::now(); + debug::trace_log_scalar("outetts.llama.prefill.graph_rebuilt", true); + debug::trace_log_scalar("outetts.llama.prefill.graph_reused", false); + debug::trace_log_scalar("outetts.llama.prefill.tokens", steps); + debug::timing_log_scalar( + "outetts.llama.prefill.graph_build_ms", + debug::elapsed_ms(total_start, build_end)); + debug::timing_log_scalar( + "outetts.llama.prefill.compute_ms", + debug::elapsed_ms(build_end, compute_end)); + debug::timing_log_scalar( + "outetts.llama.prefill.read_ms", + debug::elapsed_ms(compute_end, read_end)); + debug::timing_log_scalar( + "outetts.llama.prefill.release_ms", + debug::elapsed_ms(read_end, release_end)); + return result; + } + + CachedStepGraph & ensure_step_graph(int64_t capacity) { + const auto build_start = std::chrono::steady_clock::now(); + const bool rebuilt = + step_graph == nullptr || step_graph->capacity() < capacity; + if (rebuilt) { + step_graph = std::make_unique( + assets->config, weights, *constants, backend, threads, capacity); + } + debug::trace_log_scalar("outetts.llama.step.graph_rebuilt", rebuilt); + debug::trace_log_scalar("outetts.llama.step.graph_reused", !rebuilt); + debug::trace_log_scalar( + "outetts.llama.step.capacity", step_graph->capacity()); + debug::timing_log_scalar( + "outetts.llama.step.graph_build_ms", + rebuilt ? debug::elapsed_ms(build_start) : 0.0); + return *step_graph; + } + + int64_t release_cached_step_graph() { + if (step_graph == nullptr) { + return 0; + } + const int64_t capacity = step_graph->capacity(); + step_graph.reset(); + return capacity; + } + + std::shared_ptr assets; + ggml_backend_t backend = nullptr; + int threads = 1; + sampling::TorchCudaSamplingPolicy sampling_policy; + ModelWeights weights; + std::unique_ptr constants; + std::unique_ptr step_graph; +}; + +OuteTTSLlamaRuntime::OuteTTSLlamaRuntime( + std::shared_ptr assets, + core::BackendType backend_type, + int device, + int threads, + size_t weight_context_bytes, + size_t constant_context_bytes, + assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique( + std::move(assets), backend_type, device, threads, + weight_context_bytes, constant_context_bytes, weight_storage_type)) {} + +OuteTTSLlamaRuntime::~OuteTTSLlamaRuntime() = default; + +std::string_view outetts_stop_reason_name(OuteTTSStopReason reason) noexcept { + switch (reason) { + case OuteTTSStopReason::Eos: + return "eos"; + case OuteTTSStopReason::AudioEnd: + return "audio_end"; + case OuteTTSStopReason::MaxTokens: + return "max_tokens"; + case OuteTTSStopReason::ContextLimit: + return "context_limit"; + } + return "unknown"; +} + +OuteTTSGenerateResult OuteTTSLlamaRuntime::generate( + const std::vector & prompt, + const OuteTTSGenerateOptions & options, + int32_t eos_id, + int32_t audio_end_id) const { + if (prompt.empty()) { + throw std::runtime_error("OuteTTS generation requires a prompt"); + } + if (options.max_new_tokens <= 0 || options.repetition_window < 0 || options.repetition_penalty <= 0.0F) { + throw std::runtime_error("invalid OuteTTS generation options"); + } + const auto total_start = std::chrono::steady_clock::now(); + std::vector all = prompt; + OuteTTSGenerateResult result; + result.tokens.reserve(static_cast(options.max_new_tokens)); + std::mt19937 rng(options.seed); + OuteTTSGenerateOptions sampling_options = options; + auto prefill = impl_->prefill(prompt); + const int64_t capacity = std::min( + impl_->assets->generation.max_length, + static_cast(prompt.size()) + options.max_new_tokens); + constexpr int64_t kStepGraphCapacityQuantum = 256; + const int64_t reusable_capacity = std::min( + impl_->assets->generation.max_length, + ((capacity + kStepGraphCapacityQuantum - 1) / + kStepGraphCapacityQuantum) * + kStepGraphCapacityQuantum); + auto & step = impl_->ensure_step_graph(reusable_capacity); + step.import_state(prefill.state); + std::vector logits = std::move(prefill.logits); + double sample_ms = 0.0; + double cached_step_compute_ms = 0.0; + SamplingScratch sampling_scratch; + for (int64_t i = 0; i < options.max_new_tokens; ++i) { + if (static_cast(all.size()) >= impl_->assets->generation.max_length) { + result.stop_reason = OuteTTSStopReason::ContextLimit; + break; + } + const auto sample_start = std::chrono::steady_clock::now(); + apply_repetition_penalty( + logits, all, options.repetition_window, options.repetition_penalty, + sampling_scratch.repeated_ids); + const int32_t token = sample_token( + logits, sampling_options, rng, + impl_->sampling_policy, static_cast(i), + sampling_scratch); + sample_ms += debug::elapsed_ms(sample_start); + result.tokens.push_back(token); + if (token == eos_id) { + result.stop_reason = OuteTTSStopReason::Eos; + break; + } + if (token == audio_end_id) { + result.stop_reason = OuteTTSStopReason::AudioEnd; + break; + } + // Do not execute one unused cached step after consuming the caller's + // final generation token. + if (i + 1 >= options.max_new_tokens) { + break; + } + all.push_back(token); + const auto step_start = std::chrono::steady_clock::now(); + step.run(token, logits); + cached_step_compute_ms += debug::elapsed_ms(step_start); + } + debug::trace_log_scalar( + "outetts.llama.generated_tokens", + static_cast(result.tokens.size())); + debug::trace_log_scalar("outetts.llama.stop_reason", + outetts_stop_reason_name(result.stop_reason)); + debug::timing_log_scalar("outetts.llama.sample_ms", sample_ms); + debug::timing_log_scalar("outetts.llama.cached_step_compute_ms", + cached_step_compute_ms); + debug::timing_log_scalar("outetts.llama.generate_total_ms", + debug::elapsed_ms(total_start)); + return result; +} + +int64_t OuteTTSLlamaRuntime::release_cached_step_graph() { + return impl_->release_cached_step_graph(); +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/loader.cpp b/src/models/outetts/loader.cpp new file mode 100644 index 00000000..11ec6ea9 --- /dev/null +++ b/src/models/outetts/loader.cpp @@ -0,0 +1,182 @@ +#include "engine/models/outetts/loader.h" + +#include "engine/framework/assets/model_package.h" +#include "engine/models/outetts/session.h" + +#include +#include + +namespace engine::models::outetts { +namespace { + +runtime::ModelMetadata metadata(const OuteTTSAssets &) { + runtime::ModelMetadata out; + out.family = "outetts"; + out.variant = "1.0-1B"; + out.description = + "OuteTTS 1.0 1B with native IBM DAC speech synthesis and voice cloning."; + return out; +} + +runtime::CapabilitySet capabilities(const OuteTTSAssets &) { + runtime::CapabilitySet out; + out.supported_tasks = { + {runtime::VoiceTaskKind::Tts, {runtime::RunMode::Offline}}, + {runtime::VoiceTaskKind::VoiceCloning, {runtime::RunMode::Offline}}, + }; + out.supports_speaker_reference = true; + out.languages = { + "Auto", "Arabic", "Belarusian", "Bengali", "Chinese", + "Dutch", "English", "French", "Georgian", "German", + "Hungarian", "Italian", "Japanese", "Korean", "Latvian", + "Lithuanian", "Persian", "Polish", "Portuguese", "Russian", + "Spanish", "Swahili", "Tamil", "Ukrainian", + }; + return out; +} + +runtime::ModelCliInterface cli(const OuteTTSAssets &) { + runtime::ModelCliInterface out; + out.request_options = { + {"max_tokens", "n", + "Maximum generated audio tokens per chunk. When omitted, OuteTTS " + "estimates a safe value from each chunk."}, + {"temperature", "float", + "Sampling temperature; official cloning default 0.4."}, + {"top_k", "n", "Top-k sampling; official default 40."}, + {"top_p", "float", "Nucleus sampling; official default 0.9."}, + {"min_p", "float", + "Minimum probability relative to the best token; official default " + "0.05."}, + {"repetition_penalty", "float", + "Windowed repetition penalty; official default 1.1."}, + {"repetition_window", "n", + "Recent-token penalty window; official value 64."}, + {"seed", "n", + "Sampling seed; cloning defaults to 4099 for native weights and " + "42 for quantized weights."}, + {"reference_text", "text", + "Transcript matching the --voice-ref audio for voice cloning."}, + {"reference_language", "code", + "Language code used to align the reference transcript; default en."}, + {"text_chunk_size", "n", + "Framework long-form text chunk size; default 256 characters. Chunks " + "are split further when required by max_tokens."}, + {"text_chunk_mode", "default|tag_aware|japanese|endline", + "Framework long-form text chunking mode."}, + }; + out.session_options = { + {"outetts.weight_type", "native|f32|f16|bf16|q8_0", + "Language-model weight storage type. Quantized CUDA voice cloning " + "is expanded to F32 in memory for generation correctness."}, + {"outetts.llama_weight_context_mb", "n", + "Language-model weight context size in MiB."}, + {"outetts.constant_context_mb", "n", + "Language-model constant tensor context size in MiB."}, + {"outetts.dac_weight_context_mb", "n", + "DAC decoder weight context size in MiB."}, + {"outetts.dac_graph_context_mb", "n", + "DAC decoder graph context size in MiB."}, + {"outetts.aligner_model_path", "path", + "Optional Qwen3 Forced Aligner override. Cloning automatically uses " + "the aligner embedded in a standalone OuteTTS GGUF when present."}, + {"outetts.reference_cache_slots", "n", + "Prepared reference-profile cache slots; default 1, set 0 to " + "disable."}, + {"outetts.mem_saver", "true|false", + "Release cached-step and aligner runtime state after use; default " + "false."}, + }; + return out; +} + +class OuteTTSLoader final : public runtime::IVoiceModelLoader { +public: + std::string family() const override { return "outetts"; } + + bool can_load(const runtime::ModelLoadRequest &request) const override { + try { + const auto package_spec = + assets::default_model_package_spec_path(family()); + (void)assets::load_resource_bundle_from_package_spec(request.model_path, + package_spec); + return !request.family_hint.has_value() || + *request.family_hint == family(); + } catch (...) { + return false; + } + } + + runtime::ModelInspection + inspect(const runtime::ModelLoadRequest &request) const override { + const auto model_assets = load_outetts_assets(request.model_path); + runtime::ModelInspection inspection; + inspection.model_root = model_assets->resources.model_root(); + inspection.metadata = metadata(*model_assets); + inspection.capabilities = capabilities(*model_assets); + inspection.cli = cli(*model_assets); + const auto package_spec = assets::default_model_package_spec_path(family()); + inspection.discovered_configs = + runtime::discover_named_assets_from_package_spec( + request.model_path, package_spec, + assets::ModelPackageResourceKind::Files); + inspection.discovered_weights = + runtime::discover_named_assets_from_package_spec( + request.model_path, package_spec, + assets::ModelPackageResourceKind::Tensors); + return inspection; + } + + std::unique_ptr + load(const runtime::ModelLoadRequest &request) const override { + return load_outetts_model(request.model_path); + } +}; + +} // namespace + +OuteTTSLoadedModel::OuteTTSLoadedModel( + runtime::ModelMetadata metadata, runtime::CapabilitySet capabilities, + std::shared_ptr assets) + : metadata_(std::move(metadata)), capabilities_(std::move(capabilities)), + assets_(std::move(assets)) { + if (assets_ == nullptr) { + throw std::runtime_error("OuteTTS loaded model requires assets"); + } +} + +const runtime::ModelMetadata &OuteTTSLoadedModel::metadata() const noexcept { + return metadata_; +} + +const runtime::CapabilitySet & +OuteTTSLoadedModel::capabilities() const noexcept { + return capabilities_; +} + +std::unique_ptr +OuteTTSLoadedModel::create_task_session( + const runtime::TaskSpec &task, + const runtime::SessionOptions &options) const { + if ((task.task != runtime::VoiceTaskKind::Tts && + task.task != runtime::VoiceTaskKind::VoiceCloning) || + task.mode != runtime::RunMode::Offline) { + throw std::runtime_error( + "OuteTTS supports offline TTS and voice cloning only"); + } + return std::make_unique(task, options, assets_); +} + +std::unique_ptr +load_outetts_model(const std::filesystem::path &model_path) { + auto model_assets = load_outetts_assets(model_path); + return std::make_unique(metadata(*model_assets), + capabilities(*model_assets), + std::move(model_assets)); +} + +std::shared_ptr make_outetts_loader() { + return std::make_shared(); +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/session.cpp b/src/models/outetts/session.cpp new file mode 100644 index 00000000..1afdb8bc --- /dev/null +++ b/src/models/outetts/session.cpp @@ -0,0 +1,926 @@ +#include "engine/models/outetts/session.h" + +#include "engine/framework/audio/fft.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/debug/trace.h" +#include "engine/framework/text/chunking.h" +#include "engine/framework/text/utf8.h" +#include "engine/models/qwen3_asr/assets.h" +#include "engine/models/qwen3_forced_aligner/session.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::outetts { +namespace { + +using Clock = std::chrono::steady_clock; + +constexpr int64_t kDefaultTextChunkSize = 256; +constexpr int64_t kAutomaticChunkTokenBudget = 4096; +constexpr size_t kDefaultReferenceCacheSlots = 1; + +int64_t generation_budget_with_headroom(int64_t estimated_tokens) { + // The upstream heuristic is a useful lower bound, but sampled generation can + // occasionally need a little more room before emitting audio_end. + return estimated_tokens + std::max(128, estimated_tokens / 4); +} + +uint64_t mix_cache_key(uint64_t key, uint64_t value) { + key ^= value; + key *= 1099511628211ull; + return key; +} + +uint64_t reference_audio_hash(const runtime::AudioBuffer &audio) { + uint64_t key = 1469598103934665603ull; + key = mix_cache_key(key, static_cast(audio.sample_rate)); + key = mix_cache_key(key, static_cast(audio.channels)); + key = mix_cache_key(key, static_cast(audio.samples.size())); + for (const float sample : audio.samples) { + uint32_t bits = 0; + std::memcpy(&bits, &sample, sizeof(bits)); + key = mix_cache_key(key, static_cast(bits)); + } + return key; +} + +size_t reference_cache_slots(const runtime::SessionOptions &options) { + const int64_t slots = runtime::parse_i64_option( + options.options, + {"outetts.reference_cache_slots", + "reference_cache_slots"}) + .value_or( + static_cast( + kDefaultReferenceCacheSlots)); + if (slots < 0 || + static_cast(slots) > + static_cast(std::numeric_limits::max())) { + throw std::runtime_error( + "outetts.reference_cache_slots must be a non-negative size"); + } + return static_cast(slots); +} + +bool mem_saver_from_options(const runtime::SessionOptions &options) { + if (const auto value = runtime::find_option( + options.options, {"outetts.mem_saver", "mem_saver"})) { + return runtime::parse_bool_option(*value, "outetts.mem_saver"); + } + return false; +} + +assets::TensorStorageType requested_weight_type( + const runtime::SessionOptions &options) { + const auto it = options.options.find("outetts.weight_type"); + return it == options.options.end() + ? assets::TensorStorageType::Native + : assets::parse_tensor_storage_type(it->second); +} + +bool has_quantized_clone_weights(const runtime::SessionOptions &options, + const OuteTTSAssets &model_assets) { + constexpr std::string_view probe = + "model.layers.0.self_attn.q_proj.weight"; + const auto source_type = assets::tensor_storage_type_for_dtype( + model_assets.model_weights->require_metadata(probe).dtype); + const auto requested_type = assets::resolve_tensor_storage_type( + *model_assets.model_weights, probe, requested_weight_type(options)); + return ggml_is_quantized( + assets::ggml_type_for_tensor_storage(source_type)) || + ggml_is_quantized( + assets::ggml_type_for_tensor_storage(requested_type)); +} + +assets::TensorStorageType clone_weight_type( + const runtime::SessionOptions &options, + const OuteTTSAssets &model_assets) { + if (options.backend.type == core::BackendType::Cuda && + has_quantized_clone_weights(options, model_assets)) { + // CUDA execution with quantized OuteTTS weights diverges over the long + // reference-codec prompt used for cloning. The GGUF stays quantized on + // disk; only the in-memory language-model tensors are expanded to F32. + // F16 still produces phonetic but unintelligible speech from Q8 source + // tensors on this route, while F32 matches the coherent CPU decode. + debug::trace_log_scalar("outetts.cuda_clone_quantized_f32_fallback", true); + return assets::TensorStorageType::F32; + } + return requested_weight_type(options); +} + +OuteTTSGenerateOptions +generation_options(const runtime::TaskRequest &request, + const OuteTTSGenerationConfig &defaults, + bool voice_cloning, + bool quantized_cloning, + int64_t automatic_max_new_tokens) { + OuteTTSGenerateOptions out; + out.max_new_tokens = automatic_max_new_tokens; + out.temperature = voice_cloning ? 0.4F : defaults.temperature; + out.repetition_penalty = defaults.repetition_penalty; + out.repetition_window = defaults.repetition_window; + out.top_k = voice_cloning ? 40 : defaults.top_k; + out.top_p = voice_cloning ? 0.9F : defaults.top_p; + out.min_p = voice_cloning ? 0.05F : defaults.min_p; + if (const auto v = runtime::parse_i64_option(request.options, {"max_tokens"})) + out.max_new_tokens = *v; + if (const auto v = + runtime::parse_finite_float_option(request.options, {"temperature"})) + out.temperature = *v; + if (const auto v = runtime::parse_finite_float_option(request.options, + {"repetition_penalty"})) + out.repetition_penalty = *v; + if (const auto v = + runtime::parse_i64_option(request.options, {"repetition_window"})) + out.repetition_window = *v; + if (const auto v = runtime::parse_i64_option(request.options, {"top_k"})) + out.top_k = *v; + if (const auto v = + runtime::parse_finite_float_option(request.options, {"top_p"})) + out.top_p = *v; + if (const auto v = + runtime::parse_finite_float_option(request.options, {"min_p"})) + out.min_p = *v; + out.seed = runtime::parse_u32_option(request.options, {"seed"}) + .value_or(voice_cloning + ? (quantized_cloning ? 42u : 4099u) + : runtime::random_u32_seed()); + if (out.max_new_tokens <= 0 || out.repetition_window < 0 || out.top_k < 0 || + out.temperature < 0.0F || out.repetition_penalty <= 0.0F || + out.top_p <= 0.0F || out.top_p > 1.0F || out.min_p < 0.0F || + out.min_p > 1.0F) { + throw std::runtime_error("invalid OuteTTS generation options"); + } + return out; +} + +std::vector chunk_text_request_to_token_budget( + const runtime::TaskRequest &request, int64_t requested_chunk_size, + engine::text::TextChunkMode mode, int64_t token_budget) { + auto initial = runtime::chunk_text_request(request, requested_chunk_size, + mode); + std::deque pending(initial.begin(), initial.end()); + std::vector chunks; + while (!pending.empty()) { + auto candidate = std::move(pending.front()); + pending.pop_front(); + if (!candidate.text_input.has_value()) { + chunks.push_back(std::move(candidate)); + continue; + } + const auto budget = + estimate_text_generation_budget(candidate.text_input->text); + // The upstream estimate has an intentional 384-token floor. Preserve + // explicitly smaller limits for short requests instead of inventing empty + // or one-character chunks; a cap is still reported as an error after + // generation if the model cannot stop within that caller-provided limit. + if (budget.recommended_max_new_tokens <= token_budget || + token_budget < 384) { + chunks.push_back(std::move(candidate)); + continue; + } + + const int64_t codepoints = static_cast( + engine::text::utf8_codepoint_count(candidate.text_input->text, + "OuteTTS text chunk")); + if (codepoints <= 1) { + throw std::runtime_error( + "OuteTTS max_tokens is too small for the requested text chunk"); + } + const int64_t smaller_chunk_size = std::max(1, codepoints / 2); + auto smaller = runtime::chunk_text_request(candidate, smaller_chunk_size, + mode); + if (smaller.size() <= 1) { + throw std::runtime_error( + "OuteTTS could not split text to fit the max_tokens budget"); + } + for (auto it = smaller.rbegin(); it != smaller.rend(); ++it) + pending.push_front(std::move(*it)); + } + return chunks; +} + +std::vector split_words(const std::string &text) { + std::istringstream input(text); + std::vector words; + std::string word; + while (input >> word) + words.push_back(word); + return words; +} + +size_t utf8_length(const std::string &text) { + return std::max( + 1, static_cast( + std::count_if(text.begin(), text.end(), [](unsigned char value) { + return (value & 0xc0) != 0x80; + }))); +} + +OuteTTSVoiceFeatures audio_features(const std::vector &samples, + size_t begin, size_t end) { + OuteTTSVoiceFeatures result; + if (begin >= end || begin >= samples.size()) + return result; + end = std::min(end, samples.size()); + double sum_sq = 0.0; + for (size_t i = begin; i < end; ++i) + sum_sq += static_cast(samples[i]) * samples[i]; + const double rms = std::sqrt(sum_sq / static_cast(end - begin)); + result.energy = + static_cast(std::clamp(std::lround(rms * 100.0), 0l, 100l)); + + const size_t count = end - begin; + std::vector> spectrum(count / 2u + 1u); + const auto fft = engine::audio::get_real_fft_plan(count); + fft->forward({count}, {static_cast(sizeof(float))}, + {static_cast(sizeof(std::complex))}, 0, + samples.data() + static_cast(begin), + spectrum.data()); + double magnitude_sum = 1.0e-10; + double weighted_frequency = 0.0; + for (size_t bin = 0; bin < spectrum.size(); ++bin) { + const double magnitude = std::abs(spectrum[bin]); + magnitude_sum += magnitude; + weighted_frequency += magnitude * static_cast(bin) * + 24000.0 / static_cast(count); + } + result.spectral_centroid = static_cast(std::clamp( + std::lround(weighted_frequency / magnitude_sum / 12000.0 * 100.0), + 0l, 100l)); + + if (count >= 400 && sum_sq >= 1.0e-8) { + constexpr int frame_length = 400; + constexpr int hop_length = 160; + constexpr int min_lag = 24000 / 600; + constexpr int max_lag_exclusive = 24000 / 75; + const size_t pad = + (frame_length - (count % hop_length)) % hop_length; + const size_t padded_count = count + pad; + const size_t frames = 1u + (padded_count - frame_length) / hop_length; + double pitch_sum = 0.0; + std::vector windowed(frame_length); + std::vector autocorrelation(frame_length); + for (size_t frame = 0; frame < frames; ++frame) { + const size_t offset = begin + frame * hop_length; + for (int i = 0; i < frame_length; ++i) { + const size_t source = offset + static_cast(i); + const double sample = source < end ? samples[source] : 0.0; + const double window = + 0.5 - 0.5 * std::cos(2.0 * 3.14159265358979323846 * i / + frame_length); + windowed[static_cast(i)] = sample * window; + } + for (int lag = 0; lag < frame_length; ++lag) { + double value = 0.0; + for (int i = 0; i + lag < frame_length; ++i) + value += windowed[static_cast(i)] * + windowed[static_cast(i + lag)]; + autocorrelation[static_cast(lag)] = value; + } + int best_lag = min_lag; + for (int lag = min_lag + 1; lag < max_lag_exclusive; ++lag) { + if (autocorrelation[static_cast(lag)] > + autocorrelation[static_cast(best_lag)]) + best_lag = lag; + } + double frequency = 75.0; + const double beta = autocorrelation[static_cast(best_lag)]; + if (autocorrelation[0] > 1.0e-10 && + beta / autocorrelation[0] > 0.3) { + const double alpha = + autocorrelation[static_cast(best_lag - 1)]; + const double gamma = + autocorrelation[static_cast(best_lag + 1)]; + const double delta = 0.5 * (alpha - gamma) / + (alpha - 2.0 * beta + gamma + 1.0e-8); + frequency = std::clamp(24000.0 / (best_lag + delta), 75.0, 600.0); + } + pitch_sum += frequency; + } + const double average_pitch = pitch_sum / static_cast(frames); + result.pitch = static_cast(std::clamp( + std::lround((average_pitch - 75.0) / 525.0 * 100.0), 0l, 100l)); + } + return result; +} + +struct ReferenceAlignment { + std::vector words; + int sample_rate = 16000; +}; + +OuteTTSVoiceProfile +make_voice_profile(OuteTTSDacDecoder::EncodedReference encoded, + std::string reference_text, + const ReferenceAlignment *alignment) { + auto words = split_words(reference_text); + if (words.empty()) + throw std::runtime_error("OuteTTS reference_text must not be empty"); + const size_t frame_count = + std::min(encoded.codebook1.size(), encoded.codebook2.size()); + if (frame_count == 0) + throw std::runtime_error( + "OuteTTS DAC encoder produced no reference codec frames"); + if (alignment != nullptr) { + words.clear(); + for (const auto &word : alignment->words) + words.push_back(word.word); + } + if (words.size() > frame_count) + words.resize(frame_count); + std::vector weights(words.size()); + size_t total_weight = 0; + for (size_t i = 0; i < words.size(); ++i) { + weights[i] = utf8_length(words[i]); + total_weight += weights[i]; + } + + OuteTTSVoiceProfile profile; + profile.text = reference_text; + profile.global_features = + audio_features(encoded.samples, 0, encoded.samples.size()); + debug::trace_log_scalar("outetts.reference.global.energy", + profile.global_features.energy); + debug::trace_log_scalar("outetts.reference.global.spectral_centroid", + profile.global_features.spectral_centroid); + debug::trace_log_scalar("outetts.reference.global.pitch", + profile.global_features.pitch); + size_t start = 0; + size_t cumulative_weight = 0; + for (size_t word_index = 0; word_index < words.size(); ++word_index) { + size_t feature_begin = 0; + size_t feature_end = 0; + size_t end = 0; + if (alignment != nullptr && word_index < alignment->words.size()) { + const auto &span = alignment->words[word_index].span; + const double begin_seconds = + static_cast(span.start_sample) / alignment->sample_rate; + const double end_seconds = + static_cast(span.end_sample) / alignment->sample_rate; + if (word_index == 0) { + const int64_t aligned_start = + static_cast(begin_seconds * 75.0) - 20; + start = static_cast(std::clamp( + aligned_start, 0, static_cast(frame_count - 1))); + } + int64_t aligned_end = static_cast(end_seconds * 75.0); + if (word_index + 1 == words.size()) + aligned_end += 20; + end = static_cast(std::clamp( + aligned_end, static_cast(start + 1), + static_cast(frame_count))); + feature_begin = static_cast(std::clamp( + static_cast(begin_seconds * 24000.0), 0, + static_cast(encoded.samples.size()))); + feature_end = static_cast(std::clamp( + static_cast(end_seconds * 24000.0), + static_cast(feature_begin), + static_cast(encoded.samples.size()))); + } else { + cumulative_weight += weights[word_index]; + end = word_index + 1 == words.size() + ? frame_count + : (frame_count * cumulative_weight + total_weight / 2) / + total_weight; + feature_begin = start * 320u; + feature_end = std::min(encoded.samples.size(), end * 320u); + } + end = std::max(end, std::min(frame_count, start + 1)); + OuteTTSVoiceWord word; + word.text = words[word_index]; + word.duration = + std::round(static_cast(end - start) / 75.0 * 100.0) / 100.0; + word.features = audio_features(encoded.samples, feature_begin, feature_end); + word.codebook1.assign( + encoded.codebook1.begin() + static_cast(start), + encoded.codebook1.begin() + static_cast(end)); + word.codebook2.assign( + encoded.codebook2.begin() + static_cast(start), + encoded.codebook2.begin() + static_cast(end)); + const std::string trace_prefix = + "outetts.reference.word." + std::to_string(word_index); + debug::trace_log_scalar(trace_prefix + ".text", word.text); + debug::trace_log_scalar(trace_prefix + ".duration", word.duration); + debug::trace_log_scalar(trace_prefix + ".energy", word.features.energy); + debug::trace_log_scalar(trace_prefix + ".spectral_centroid", + word.features.spectral_centroid); + debug::trace_log_scalar(trace_prefix + ".pitch", word.features.pitch); + profile.words.push_back(std::move(word)); + start = end; + } + return profile; +} + +runtime::SessionOptions aligner_session_options( + const runtime::SessionOptions &options) { + runtime::SessionOptions out; + out.backend = options.backend; + for (const auto &[key, value] : options.options) { + if (key.rfind("qwen3_forced_aligner.", 0) == 0) + out.options.emplace(key, value); + } + return out; +} + +std::shared_ptr +resolve_aligner_assets(const runtime::SessionOptions &options, + const OuteTTSAssets &model_assets) { + const auto model_path = runtime::find_option( + options.options, + {"outetts.aligner_model_path", "outetts.forced_aligner_model_path"}); + if (model_path.has_value()) { + return engine::models::qwen3_asr::load_qwen3_asr_assets( + std::filesystem::path(*model_path), "qwen3_forced_aligner"); + } + return model_assets.embedded_aligner; +} + +ReferenceAlignment align_reference( + engine::models::qwen3_forced_aligner::Qwen3ForcedAlignerSession &session, + const engine::models::qwen3_asr::Qwen3ASRAssets &aligner_assets, + const runtime::AudioBuffer &audio, + const std::string &text, + const std::string &language) { + runtime::TaskRequest request; + request.audio_input = audio; + request.text_input = runtime::Transcript{text, language}; + request.options["audio_chunk_mode"] = "none"; + session.prepare(runtime::build_preparation_request(request)); + auto result = session.run(request); + if (result.word_timestamps.empty()) + throw std::runtime_error("OuteTTS reference aligner returned no words"); + return ReferenceAlignment{std::move(result.word_timestamps), + aligner_assets.config.sample_rate}; +} + +const runtime::AudioBuffer * +reference_audio(const runtime::TaskRequest &request) { + if (request.voice.has_value() && request.voice->speaker.has_value() && + request.voice->speaker->audio.has_value()) { + return &*request.voice->speaker->audio; + } + return request.audio_input.has_value() ? &*request.audio_input : nullptr; +} + +} // namespace + +OuteTTSSession::OuteTTSSession(runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets) + : RuntimeSessionBase(options), task_(task), assets_(std::move(assets)), + tokenizer_(assets_), + dac_(assets_, execution_context(), + runtime::parse_size_mb_option(options.options, + {"outetts.dac_weight_context_mb"}, + 1024ull * 1024ull * 1024ull), + runtime::parse_size_mb_option(options.options, + {"outetts.dac_graph_context_mb"}, + 1536ull * 1024ull * 1024ull), + assets::TensorStorageType::F32), + mem_saver_(mem_saver_from_options(options)), + reference_profile_cache_(reference_cache_slots(options)) { + if (assets_ == nullptr) + throw std::runtime_error("OuteTTS session requires assets"); + if ((task_.task != runtime::VoiceTaskKind::Tts && + task_.task != runtime::VoiceTaskKind::VoiceCloning) || + task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error( + "OuteTTS supports offline TTS and voice cloning only"); + } +} + +OuteTTSSession::~OuteTTSSession() = default; + +OuteTTSLlamaRuntime &OuteTTSSession::llama(bool voice_cloning) { + const auto ensure_start = Clock::now(); + const auto storage_type = + voice_cloning ? clone_weight_type(options(), *assets_) + : requested_weight_type(options()); + const bool rebuilt = + llama_ == nullptr || !llama_storage_type_.has_value() || + *llama_storage_type_ != storage_type; + if (rebuilt) { + // Keep only one language-model runtime resident. CUDA cloning may require + // an F32 runtime for quantized source weights, so switching routes replaces + // the previous runtime instead of retaining duplicate weights and graphs. + llama_.reset(); + llama_ = std::make_unique( + assets_, options().backend.type, options().backend.device, + std::max(1, options().backend.threads), + runtime::parse_size_mb_option(options().options, + {"outetts.llama_weight_context_mb"}, + 4096ull * 1024ull * 1024ull), + runtime::parse_size_mb_option(options().options, + {"outetts.constant_context_mb"}, + 256ull * 1024ull * 1024ull), + storage_type); + llama_storage_type_ = storage_type; + } + debug::trace_log_scalar("outetts.llama.runtime_rebuilt", rebuilt); + debug::trace_log_scalar("outetts.llama.runtime_reused", !rebuilt); + debug::trace_log_scalar("outetts.llama.clone_route", voice_cloning); + debug::timing_log_scalar( + "outetts.llama.ensure_runtime_ms", + rebuilt ? debug::elapsed_ms(ensure_start) : 0.0); + return *llama_; +} + +bool OuteTTSSession::ReferenceProfileCacheKeyEqual::operator()( + const ReferenceProfileCacheKey &lhs, + const ReferenceProfileCacheKey &rhs) const { + return lhs.audio_hash == rhs.audio_hash && + lhs.sample_rate == rhs.sample_rate && + lhs.channels == rhs.channels && + lhs.sample_count == rhs.sample_count && lhs.text == rhs.text && + lhs.language == rhs.language; +} + +OuteTTSVoiceProfile OuteTTSSession::prepare_voice_profile( + const runtime::AudioBuffer &audio, const std::string &reference_text, + const std::string &language, bool &cache_hit) { + const auto total_start = Clock::now(); + ReferenceProfileCacheKey key{ + reference_audio_hash(audio), audio.sample_rate, audio.channels, + audio.samples.size(), reference_text, language}; + if (const auto *cached = reference_profile_cache_.find(key)) { + cache_hit = true; + debug::trace_log_scalar("outetts.reference_cache.hit", true); + debug::trace_log_scalar( + "outetts.reference_cache.slots", + static_cast(reference_profile_cache_.capacity())); + debug::trace_log_scalar( + "outetts.reference_cache.entries", + static_cast(reference_profile_cache_.size())); + debug::trace_log_scalar("outetts.reference_cache.evicted", false); + debug::timing_log_scalar("outetts.reference.total_ms", + debug::elapsed_ms(total_start)); + return *cached; + } + + cache_hit = false; + const bool will_evict = + reference_profile_cache_.capacity() > 0 && + reference_profile_cache_.size() >= reference_profile_cache_.capacity(); + const auto ensure_aligner_start = Clock::now(); + const bool aligner_built = aligner_session_ == nullptr; + if (aligner_built) { + if (aligner_assets_ == nullptr) { + aligner_assets_ = resolve_aligner_assets(options(), *assets_); + } + if (aligner_assets_ == nullptr) { + throw std::runtime_error( + "OuteTTS voice cloning requires a GGUF with an embedded Qwen3 " + "Forced Aligner or --session-option " + "outetts.aligner_model_path="); + } + aligner_session_ = std::make_unique< + engine::models::qwen3_forced_aligner::Qwen3ForcedAlignerSession>( + runtime::TaskSpec{runtime::VoiceTaskKind::Alignment, + runtime::RunMode::Offline}, + aligner_session_options(options()), aligner_assets_); + } + debug::trace_log_scalar("outetts.aligner.runtime_rebuilt", aligner_built); + debug::trace_log_scalar("outetts.aligner.runtime_reused", !aligner_built); + debug::timing_log_scalar( + "outetts.aligner.ensure_runtime_ms", + aligner_built ? debug::elapsed_ms(ensure_aligner_start) : 0.0); + + const auto align_start = Clock::now(); + const auto alignment = align_reference(*aligner_session_, *aligner_assets_, + audio, reference_text, language); + debug::timing_log_scalar("outetts.reference.align_ms", + debug::elapsed_ms(align_start)); + + const auto encode_start = Clock::now(); + auto encoded = dac_.encode_reference(audio); + debug::timing_log_scalar("outetts.reference.dac_encode_ms", + debug::elapsed_ms(encode_start)); + + const auto profile_start = Clock::now(); + auto profile = make_voice_profile(std::move(encoded), reference_text, + &alignment); + debug::timing_log_scalar("outetts.reference.profile_ms", + debug::elapsed_ms(profile_start)); + reference_profile_cache_.put(std::move(key), profile); + + debug::trace_log_scalar("outetts.reference_cache.hit", false); + debug::trace_log_scalar( + "outetts.reference_cache.slots", + static_cast(reference_profile_cache_.capacity())); + debug::trace_log_scalar( + "outetts.reference_cache.entries", + static_cast(reference_profile_cache_.size())); + debug::trace_log_scalar("outetts.reference_cache.evicted", will_evict); + if (mem_saver_) { + const auto release_start = Clock::now(); + aligner_session_.reset(); + debug::trace_log_scalar("outetts.aligner.runtime_released", true); + debug::timing_log_scalar("outetts.aligner.release_ms", + debug::elapsed_ms(release_start)); + } else { + debug::trace_log_scalar("outetts.aligner.runtime_released", false); + debug::timing_log_scalar("outetts.aligner.release_ms", 0.0); + } + debug::timing_log_scalar("outetts.reference.total_ms", + debug::elapsed_ms(total_start)); + return profile; +} + +std::string OuteTTSSession::family() const { return "outetts"; } +runtime::VoiceTaskKind OuteTTSSession::task_kind() const { return task_.task; } +runtime::RunMode OuteTTSSession::run_mode() const { return task_.mode; } +void OuteTTSSession::prepare( + const runtime::SessionPreparationRequest &request) { + voice_profile_.reset(); + if (request.voice.has_value() && request.voice->speaker.has_value() && + request.voice->speaker->audio.has_value()) { + const auto reference_text = + runtime::find_option(request.options, {"reference_text"}).value_or(""); + if (reference_text.empty()) { + throw std::runtime_error( + "OuteTTS voice cloning requires --reference-text"); + } + const auto &audio = *request.voice->speaker->audio; + const auto language = runtime::find_option( + request.options, {"reference_language"}) + .value_or(request.text.has_value() && + !request.text->language.empty() + ? request.text->language + : "en"); + bool cache_hit = false; + voice_profile_ = + prepare_voice_profile(audio, reference_text, language, cache_hit); + debug::trace_log_scalar("outetts.prepare.reference_cache_hit", cache_hit); + } + mark_prepared(); +} + +runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { + const auto wall_start = Clock::now(); + require_prepared("OuteTTS run"); + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("OuteTTS requires text input"); + } + const auto *voice_audio = reference_audio(request); + std::optional request_profile; + bool reference_cache_hit = false; + if (voice_audio != nullptr) { + const auto reference_text = + runtime::find_option(request.options, {"reference_text"}).value_or(""); + if (reference_text.empty()) + throw std::runtime_error( + "OuteTTS voice cloning requires --reference-text"); + const auto language = runtime::find_option( + request.options, {"reference_language"}) + .value_or(request.text_input.has_value() && + !request.text_input->language.empty() + ? request.text_input->language + : "en"); + request_profile = prepare_voice_profile( + *voice_audio, reference_text, language, reference_cache_hit); + } + const OuteTTSVoiceProfile *profile = + request_profile.has_value() + ? &*request_profile + : (voice_profile_.has_value() ? &*voice_profile_ : nullptr); + if (task_.task == runtime::VoiceTaskKind::VoiceCloning && + profile == nullptr) { + throw std::runtime_error( + "OuteTTS voice cloning requires --voice-ref and --reference-text"); + } + + const int64_t text_chunk_size = + engine::text::parse_text_chunk_size_override(request.options) + .value_or(kDefaultTextChunkSize); + const auto text_chunk_mode = + engine::text::parse_text_chunk_mode_override(request.options) + .value_or(engine::text::TextChunkMode::Default); + const auto explicit_max_tokens = + runtime::parse_i64_option(request.options, {"max_tokens"}); + if (explicit_max_tokens.has_value() && *explicit_max_tokens <= 0) { + throw std::runtime_error("OuteTTS max_tokens must be positive"); + } + const int64_t chunk_token_budget = std::min( + explicit_max_tokens.value_or(kAutomaticChunkTokenBudget), + kAutomaticChunkTokenBudget); + runtime::TaskRequest chunk_source = request; + // Reference conditioning is already represented by profile. Avoid copying + // the full reference waveform into every long-form chunk and retry. + if (profile != nullptr) { + chunk_source.voice.reset(); + chunk_source.audio_input.reset(); + } + const auto initial_chunk_requests = chunk_text_request_to_token_budget( + chunk_source, text_chunk_size, text_chunk_mode, chunk_token_budget); + if (initial_chunk_requests.empty()) { + throw std::runtime_error("OuteTTS text chunking produced no requests"); + } + std::deque pending_chunks( + initial_chunk_requests.begin(), initial_chunk_requests.end()); + debug::trace_log_scalar("outetts.text_chunk_size", text_chunk_size); + debug::trace_log_scalar("outetts.text_chunk_mode", + engine::text::text_chunk_mode_name(text_chunk_mode)); + debug::trace_log_scalar("outetts.text_chunk_token_budget", + chunk_token_budget); + debug::trace_log_scalar( + "outetts.text_chunk_count", + static_cast(initial_chunk_requests.size())); + debug::trace_log_scalar("outetts.reference.cache_hit", + reference_cache_hit); + + const bool quantized_cloning = + profile != nullptr && has_quantized_clone_weights(options(), *assets_); + + runtime::AudioBuffer merged_audio; + double prompt_ms = 0.0; + double generate_ms = 0.0; + double decode_ms = 0.0; + double release_ms = 0.0; + int64_t generated_tokens = 0; + int64_t retry_generated_tokens = 0; + int64_t released_cache_capacity = 0; + int64_t retry_count = 0; + size_t chunk_index = 0; + while (!pending_chunks.empty()) { + auto chunk_request = std::move(pending_chunks.front()); + pending_chunks.pop_front(); + const auto prompt_start = Clock::now(); + const auto prompt = + profile != nullptr + ? tokenizer_.build_clone_prompt(chunk_request.text_input->text, + *profile) + : tokenizer_.build_prompt(chunk_request.text_input->text); + prompt_ms += debug::elapsed_ms(prompt_start); + + const auto text_budget = estimate_text_generation_budget( + chunk_request.text_input->text); + const int64_t automatic_max_new_tokens = generation_budget_with_headroom( + text_budget.recommended_max_new_tokens); + const int64_t expected_generation_tokens = std::min( + automatic_max_new_tokens, + explicit_max_tokens.value_or(automatic_max_new_tokens)); + const int64_t available_context = + assets_->generation.max_length - static_cast(prompt.size()); + if (available_context <= 0) { + throw std::runtime_error( + "OuteTTS chunk " + std::to_string(chunk_index + 1) + + " prompt exhausts the generation context; use a shorter cloning " + "reference"); + } + if (expected_generation_tokens > available_context) { + const int64_t codepoints = static_cast( + engine::text::utf8_codepoint_count(chunk_request.text_input->text, + "OuteTTS context retry")); + auto smaller = runtime::chunk_text_request( + chunk_request, std::max(1, codepoints / 2), + text_chunk_mode); + if (codepoints <= 1 || smaller.size() <= 1) { + throw std::runtime_error( + "OuteTTS chunk does not fit the generation context; reduce " + "text_chunk_size or use a shorter cloning reference"); + } + debug::trace_log_scalar( + "outetts.retry." + std::to_string(retry_count) + ".reason", + std::string_view("context_budget")); + debug::trace_log_scalar( + "outetts.retry." + std::to_string(retry_count) + ".split_count", + static_cast(smaller.size())); + ++retry_count; + for (auto it = smaller.rbegin(); it != smaller.rend(); ++it) + pending_chunks.push_front(std::move(*it)); + continue; + } + + auto generate_options = + generation_options(chunk_request, assets_->generation, + profile != nullptr, quantized_cloning, + automatic_max_new_tokens); + generate_options.max_new_tokens = + std::min(generate_options.max_new_tokens, available_context); + const auto generate_start = Clock::now(); + const auto generation = llama(profile != nullptr).generate( + prompt, generate_options, tokenizer_.eos_id(), + tokenizer_.audio_end_id()); + generate_ms += debug::elapsed_ms(generate_start); + const auto &generated = generation.tokens; + + if (generation.stop_reason == OuteTTSStopReason::MaxTokens || + generation.stop_reason == OuteTTSStopReason::ContextLimit) { + retry_generated_tokens += static_cast(generated.size()); + const int64_t codepoints = static_cast( + engine::text::utf8_codepoint_count(chunk_request.text_input->text, + "OuteTTS generation retry")); + auto smaller = runtime::chunk_text_request( + chunk_request, std::max(1, codepoints / 2), + text_chunk_mode); + if (codepoints <= 1 || smaller.size() <= 1) { + throw std::runtime_error( + "OuteTTS reached " + + std::string(outetts_stop_reason_name(generation.stop_reason)) + + " before an audio end token and cannot split the remaining text " + "further; increase max_tokens"); + } + const std::string retry_prefix = + "outetts.retry." + std::to_string(retry_count); + debug::trace_log_scalar(retry_prefix + ".reason", + outetts_stop_reason_name(generation.stop_reason)); + debug::trace_log_scalar(retry_prefix + ".generated_tokens", + static_cast(generated.size())); + debug::trace_log_scalar(retry_prefix + ".text_codepoints", + text_budget.non_whitespace_codepoints); + debug::trace_log_scalar(retry_prefix + ".split_count", + static_cast(smaller.size())); + ++retry_count; + for (auto it = smaller.rbegin(); it != smaller.rend(); ++it) + pending_chunks.push_front(std::move(*it)); + continue; + } + generated_tokens += static_cast(generated.size()); + + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".text_codepoints", + text_budget.non_whitespace_codepoints); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".text_words", + text_budget.words); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + + ".recommended_max_new_tokens", + text_budget.recommended_max_new_tokens); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".max_new_tokens", + generate_options.max_new_tokens); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".stop_reason", + outetts_stop_reason_name(generation.stop_reason)); + + std::vector c1; + std::vector c2; + for (const int32_t token : generated) + tokenizer_.append_audio_code(token, c1, c2); + const size_t pairs = std::min(c1.size(), c2.size()); + c1.resize(pairs); + c2.resize(pairs); + if (pairs == 0) { + std::string detail; + for (size_t i = 0; i < std::min(generated.size(), 12); ++i) { + detail += (i == 0 ? "" : ",") + std::to_string(generated[i]); + } + throw std::runtime_error( + "OuteTTS generated no complete DAC code pairs (tokens=" + detail + + ")"); + } + + const auto decode_start = Clock::now(); + runtime::append_audio_buffer(merged_audio, dac_.decode(c1, c2)); + decode_ms += debug::elapsed_ms(decode_start); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".prompt_tokens", + static_cast(prompt.size())); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".generated_tokens", + static_cast(generated.size())); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".codec_frames", + static_cast(pairs)); + + if (mem_saver_) { + const auto release_start = Clock::now(); + released_cache_capacity += llama_->release_cached_step_graph(); + release_ms += debug::elapsed_ms(release_start); + } + ++chunk_index; + } + + runtime::TaskResult result; + result.audio_output = std::move(merged_audio); + debug::trace_log_scalar("outetts.mem_saver", mem_saver_); + debug::trace_log_scalar("outetts.generated_tokens", generated_tokens); + debug::trace_log_scalar("outetts.retry_generated_tokens", + retry_generated_tokens); + debug::trace_log_scalar("outetts.text_chunk_count_final", + static_cast(chunk_index)); + debug::trace_log_scalar("outetts.retry_count", retry_count); + debug::trace_log_scalar("outetts.llama.step.released_cache_capacity", + released_cache_capacity); + debug::timing_log_scalar("outetts.prompt_ms", prompt_ms); + debug::timing_log_scalar("outetts.generate_ms", generate_ms); + debug::timing_log_scalar("outetts.dac_decode_ms", decode_ms); + debug::timing_log_scalar("outetts.llama.step.release_ms", release_ms); + debug::timing_log_scalar("session.wall_ms", + debug::elapsed_ms(wall_start)); + return result; +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/tokenizer.cpp b/src/models/outetts/tokenizer.cpp new file mode 100644 index 00000000..74d4ccaa --- /dev/null +++ b/src/models/outetts/tokenizer.cpp @@ -0,0 +1,237 @@ +#include "engine/models/outetts/tokenizer.h" + +#include "engine/framework/text/utf8.h" +#include "engine/framework/tokenizers/llama_bpe.h" + +#include +#include +#include +#include +#include + +namespace engine::models::outetts { +namespace { + +int32_t require_id(const engine::tokenizers::LlamaBpeTokenizer &tokenizer, + const std::string &token) { + const auto id = tokenizer.find_token_id(token); + if (!id.has_value()) { + throw std::runtime_error("OuteTTS tokenizer is missing token: " + token); + } + return *id; +} + +bool is_punctuation(char value) { + return value == ',' || value == '.' || value == '?' || value == '!' || + value == ':' || value == ';'; +} + +std::string trim_text(const std::string &input) { + const auto first = std::find_if_not( + input.begin(), input.end(), + [](unsigned char value) { return std::isspace(value); }); + const auto last = std::find_if_not( + input.rbegin(), input.rend(), + [](unsigned char value) { return std::isspace(value); }) + .base(); + return first < last ? std::string(first, last) : std::string{}; +} + +std::string normalize_text(const std::string &input) { + std::string text; + text.reserve(input.size()); + bool pending_space = false; + for (const unsigned char value : input) { + if (value < 0x20 || value == 0x7f || value == '"') + continue; + if (std::isspace(value)) { + pending_space = !text.empty(); + continue; + } + const char character = static_cast(value); + if (is_punctuation(character)) { + while (!text.empty() && text.back() == ' ') + text.pop_back(); + if ((character == '?' || character == '!') && !text.empty() && + text.back() == character) { + pending_space = true; + continue; + } + if (character == '.' && text.size() >= 3 && + text[text.size() - 1] == '.' && text[text.size() - 2] == '.' && + text[text.size() - 3] == '.') { + pending_space = true; + continue; + } + text.push_back(character); + pending_space = true; + continue; + } + if (pending_space && !text.empty()) + text.push_back(' '); + text.push_back(character); + pending_space = false; + } + while (!text.empty() && text.back() == ' ') + text.pop_back(); + return text; +} + +std::string speaker_separator(const std::string &text) { + if (text.empty()) + return {}; + const char last = text.back(); + return last == '.' || last == '?' || last == '!' ? " " : ". "; +} + +std::string profile_codes(const OuteTTSVoiceProfile &profile, + const std::string &separator) { + std::ostringstream out; + for (size_t word_index = 0; word_index < profile.words.size(); ++word_index) { + const auto &word = profile.words[word_index]; + std::string word_text = trim_text(word.text); + if (word_index + 1 == profile.words.size()) + word_text += trim_text(separator); + out << "<|word_start|>" << word_text << "<|features|><|t_" << std::fixed + << std::setprecision(2) << word.duration << "|>" + << "<|energy_" << word.features.energy << "|>" + << "<|spectral_centroid_" << word.features.spectral_centroid << "|>" + << "<|pitch_" << word.features.pitch << "|>" + << "<|code|>"; + const size_t frames = + std::min(word.codebook1.size(), word.codebook2.size()); + for (size_t frame = 0; frame < frames; ++frame) { + out << "<|c1_" << word.codebook1[frame] << "|>" + << "<|c2_" << word.codebook2[frame] << "|>"; + } + out << "<|word_end|>"; + if (word_index + 1 != profile.words.size()) + out << '\n'; + } + return out.str(); +} + +} // namespace + +OuteTTSTextGenerationBudget +estimate_text_generation_budget(std::string_view text) { + OuteTTSTextGenerationBudget out; + size_t position = 0; + while (position < text.size()) { + while (position < text.size() && + std::isspace(static_cast(text[position])) != 0) { + ++position; + } + if (position >= text.size()) + break; + const size_t begin = position; + while (position < text.size() && + std::isspace(static_cast(text[position])) == 0) { + ++position; + } + ++out.words; + out.non_whitespace_codepoints += static_cast( + engine::text::utf8_codepoint_count(text.substr(begin, position - begin), + "OuteTTS text")); + } + const int64_t by_words = out.words * 72; + const int64_t by_characters = out.non_whitespace_codepoints * 12; + out.recommended_max_new_tokens = + std::max({256, by_words, by_characters}) + 128; + return out; +} + +struct OuteTTSTokenizer::Impl { + explicit Impl(const OuteTTSAssets &assets) + : tokenizer({ + {}, + {}, + assets.resources.require_file("tokenizer_config"), + assets.resources.require_file("tokenizer"), + engine::tokenizers::LlamaBpePreTokenizer::Llama3, + }), + eos(require_id(tokenizer, "<|im_end|>")), + audio_end(require_id(tokenizer, "<|audio_end|>")), + word_end(require_id(tokenizer, "<|word_end|>")) { + for (int32_t code = 0; code <= 1024; ++code) { + c1.emplace(require_id(tokenizer, "<|c1_" + std::to_string(code) + "|>"), + code); + c2.emplace(require_id(tokenizer, "<|c2_" + std::to_string(code) + "|>"), + code); + } + } + + engine::tokenizers::LlamaBpeTokenizer tokenizer; + int32_t eos = 0; + int32_t audio_end = 0; + int32_t word_end = 0; + std::unordered_map c1; + std::unordered_map c2; +}; + +OuteTTSTokenizer::OuteTTSTokenizer( + std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("OuteTTS tokenizer requires assets"); + } + impl_ = std::make_shared(*assets); +} + +std::vector +OuteTTSTokenizer::build_prompt(const std::string &text) const { + if (text.empty()) { + throw std::runtime_error("OuteTTS requires non-empty text"); + } + const std::string prompt_text = normalize_text(text); + const std::string prompt = "<|im_start|>\n<|text_start|>" + prompt_text + + "<|text_end|>\n<|audio_start|>\n"; + return impl_->tokenizer.encode(prompt, true); +} + +std::vector +OuteTTSTokenizer::build_clone_prompt(const std::string &text, + const OuteTTSVoiceProfile &profile) const { + const std::string prompt_text = normalize_text(text); + const std::string reference_text = trim_text(profile.text); + if (prompt_text.empty() || reference_text.empty() || profile.words.empty()) { + throw std::runtime_error("OuteTTS voice cloning requires text, " + "reference_text, and reference codec frames"); + } + const std::string separator = speaker_separator(reference_text); + const std::string merged = reference_text + separator + prompt_text; + const std::string prompt = "<|im_start|>\n<|text_start|>" + merged + + "<|text_end|>\n<|audio_start|>\n" + + profile_codes(profile, separator) + + "\n<|word_start|>"; + return impl_->tokenizer.encode(prompt, true); +} + +bool OuteTTSTokenizer::is_stop_token(int32_t token) const noexcept { + return token == impl_->eos || token == impl_->audio_end; +} + +int32_t OuteTTSTokenizer::eos_id() const noexcept { return impl_->eos; } + +int32_t OuteTTSTokenizer::audio_end_id() const noexcept { + return impl_->audio_end; +} + +bool OuteTTSTokenizer::append_audio_code( + int32_t token, std::vector &codebook1, + std::vector &codebook2) const { + if (const auto it = impl_->c1.find(token); it != impl_->c1.end()) { + if (it->second < 1024) { + codebook1.push_back(it->second); + } + return true; + } + if (const auto it = impl_->c2.find(token); it != impl_->c2.end()) { + if (it->second < 1024) { + codebook2.push_back(it->second); + } + return true; + } + return false; +} + +} // namespace engine::models::outetts diff --git a/src/models/qwen3_asr/assets.cpp b/src/models/qwen3_asr/assets.cpp index ef576f5b..3205dcfe 100644 --- a/src/models/qwen3_asr/assets.cpp +++ b/src/models/qwen3_asr/assets.cpp @@ -165,6 +165,26 @@ assets::ResourceBundle make_resource_bundle( return resources; } +std::shared_ptr make_assets( + assets::ResourceBundle resources) { + if (!resources.has_file("preprocessor_config") && + !resources.has_file("processor_config")) { + throw std::runtime_error( + "Qwen3 ASR requires preprocessor_config.json or processor_config.json"); + } + const bool has_legacy_tokenizer = + resources.has_file("vocab") && resources.has_file("merges"); + if (!has_legacy_tokenizer && !resources.has_file("tokenizer_json")) { + throw std::runtime_error( + "Qwen3 ASR requires vocab.json plus merges.txt, or tokenizer.json"); + } + Qwen3ASRAssets assets; + assets.resources = std::move(resources); + assets.config = parse_config(assets.resources); + assets.model_weights = assets.resources.open_tensor_source("weights"); + return std::make_shared(std::move(assets)); +} + } // namespace std::shared_ptr load_qwen3_asr_assets(const std::filesystem::path & model_path) { @@ -174,12 +194,12 @@ std::shared_ptr load_qwen3_asr_assets(const std::filesyste std::shared_ptr load_qwen3_asr_assets( const std::filesystem::path & model_path, std::string_view package_family) { - auto resources = make_resource_bundle(model_path, package_family); - Qwen3ASRAssets assets; - assets.resources = std::move(resources); - assets.config = parse_config(assets.resources); - assets.model_weights = assets.resources.open_tensor_source("weights"); - return std::make_shared(std::move(assets)); + return make_assets(make_resource_bundle(model_path, package_family)); +} + +std::shared_ptr load_qwen3_asr_assets( + assets::ResourceBundle resources) { + return make_assets(std::move(resources)); } } // namespace engine::models::qwen3_asr diff --git a/tests/outetts/outetts_warm_bench.cpp b/tests/outetts/outetts_warm_bench.cpp new file mode 100644 index 00000000..c1dbf662 --- /dev/null +++ b/tests/outetts/outetts_warm_bench.cpp @@ -0,0 +1,266 @@ +#include "engine/framework/audio/wav_reader.h" +#include "engine/framework/audio/wav_writer.h" +#include "engine/framework/debug/trace.h" +#include "engine/framework/io/json.h" +#include "engine/framework/runtime/registry.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct RequestCase { + std::string name; + std::string text; + std::string language; + std::filesystem::path voice_ref; + std::string reference_text; + std::unordered_map options; +}; + +std::string arg_value(int argc, char **argv, const std::string &name, + const std::string &fallback) { + for (int i = 1; i + 1 < argc; ++i) { + if (argv[i] == name) + return argv[i + 1]; + } + return fallback; +} + +std::vector arg_values(int argc, char **argv, + const std::string &name) { + std::vector out; + for (int i = 1; i + 1 < argc; ++i) { + if (argv[i] == name) + out.emplace_back(argv[i + 1]); + } + return out; +} + +int int_arg(int argc, char **argv, const std::string &name, int fallback) { + return std::stoi(arg_value(argc, argv, name, std::to_string(fallback))); +} + +engine::core::BackendType parse_backend(const std::string &value) { + if (value == "cpu") + return engine::core::BackendType::Cpu; + if (value == "cuda") + return engine::core::BackendType::Cuda; + if (value == "vulkan") + return engine::core::BackendType::Vulkan; + if (value == "best") + return engine::core::BackendType::BestAvailable; + throw std::runtime_error("unsupported backend: " + value); +} + +std::string scalar_option(const engine::io::json::Value &value) { + if (value.is_string()) + return value.as_string(); + if (value.is_bool()) + return value.as_bool() ? "true" : "false"; + if (value.is_number()) + return engine::io::json::stringify_number(value.as_number()); + throw std::runtime_error( + "OuteTTS warm-bench options must be strings, numbers, or booleans"); +} + +void copy_option_if_present( + std::unordered_map &options, + const engine::io::json::Value &item, const std::string &name) { + if (const auto *value = item.find(name); + value != nullptr && !value->is_null()) { + options[name] = scalar_option(*value); + } +} + +std::vector +load_requests(const std::filesystem::path &path, + const std::unordered_map &defaults) { + const auto root = engine::io::json::parse_file(path); + const auto &items = root.require("requests").as_array(); + if (items.empty()) + throw std::runtime_error("OuteTTS request file has no requests"); + std::vector out; + out.reserve(items.size()); + for (size_t index = 0; index < items.size(); ++index) { + const auto &item = items[index]; + RequestCase request; + request.name = engine::io::json::optional_string( + item, "name", "request_" + std::to_string(index)); + request.text = engine::io::json::require_string(item, "text"); + request.language = + engine::io::json::optional_string(item, "language", "en"); + request.voice_ref = + engine::io::json::optional_string(item, "voice_ref", ""); + request.reference_text = + engine::io::json::optional_string(item, "reference_text", ""); + request.options = defaults; + for (const char *name : + {"max_tokens", "seed", "temperature", "top_k", "top_p", "min_p", + "repetition_penalty", "repetition_window", "text_chunk_size", + "text_chunk_mode", "reference_language"}) { + copy_option_if_present(request.options, item, name); + } + if (!request.reference_text.empty()) + request.options["reference_text"] = request.reference_text; + out.push_back(std::move(request)); + } + return out; +} + +engine::runtime::AudioBuffer read_audio(const std::filesystem::path &path) { + const auto wav = engine::audio::read_wav_f32(path); + return {wav.sample_rate, wav.channels, wav.samples}; +} + +double audio_seconds(const engine::runtime::AudioBuffer &audio) { + if (audio.sample_rate <= 0 || audio.channels <= 0) + return 0.0; + return static_cast(audio.samples.size()) / + static_cast(audio.sample_rate * audio.channels); +} + +engine::runtime::TaskRequest make_request( + const RequestCase &request, + std::unordered_map + &audio_cache) { + engine::runtime::TaskRequest out; + out.text_input = + engine::runtime::Transcript{request.text, request.language}; + out.options = request.options; + if (!request.voice_ref.empty()) { + const std::string key = request.voice_ref.string(); + auto found = audio_cache.find(key); + if (found == audio_cache.end()) + found = audio_cache.emplace(key, read_audio(request.voice_ref)).first; + out.voice = engine::runtime::VoiceCondition{}; + out.voice->speaker = engine::runtime::VoiceReference{}; + out.voice->speaker->audio = found->second; + } + return out; +} + +} // namespace + +int main(int argc, char **argv) try { + const std::filesystem::path model_path = + arg_value(argc, argv, "--model", + "models/Llama-OuteTTS-1.0-1B-Q8_0/model.gguf"); + const std::filesystem::path request_file = + arg_value(argc, argv, "--request-file", ""); + if (request_file.empty()) + throw std::runtime_error("OuteTTS warm bench requires --request-file"); + const std::filesystem::path output_dir = + arg_value(argc, argv, "--audio-out-dir", + "build/logs/warmbench/outetts_audio"); + const std::filesystem::path log_file = + arg_value(argc, argv, "--log-file", + arg_value(argc, argv, "--trace-file", + "build/logs/warmbench/outetts.log")); + const std::string backend_name = + arg_value(argc, argv, "--backend", "cpu"); + const int device = int_arg(argc, argv, "--device", 0); + const int threads = int_arg(argc, argv, "--threads", 8); + const int iterations = int_arg(argc, argv, "--iterations", 1); + const int hold_seconds = int_arg(argc, argv, "--hold-seconds", 0); + if (iterations <= 0) + throw std::runtime_error("--iterations must be positive"); + if (hold_seconds < 0) + throw std::runtime_error("--hold-seconds must be non-negative"); + + std::unordered_map defaults; + for (const auto &option : arg_values(argc, argv, "--request-option")) { + const size_t equals = option.find('='); + if (equals == std::string::npos || equals == 0) + throw std::runtime_error("invalid --request-option: " + option); + defaults[option.substr(0, equals)] = option.substr(equals + 1); + } + const auto requests = load_requests(request_file, defaults); + + std::filesystem::create_directories(output_dir); + if (!log_file.parent_path().empty()) + std::filesystem::create_directories(log_file.parent_path()); + engine::debug::configure_logging( + engine::debug::LoggingConfig{true, log_file.string()}); + + auto registry = engine::runtime::make_default_registry(); + engine::runtime::ModelLoadRequest load_request; + load_request.model_path = model_path; + load_request.family_hint = "outetts"; + auto model = registry.load(load_request); + + engine::runtime::SessionOptions session_options; + session_options.backend.type = parse_backend(backend_name); + session_options.backend.device = device; + session_options.backend.threads = threads; + for (const auto &option : arg_values(argc, argv, "--session-option")) { + const size_t equals = option.find('='); + if (equals == std::string::npos || equals == 0) + throw std::runtime_error("invalid --session-option: " + option); + session_options.options[option.substr(0, equals)] = + option.substr(equals + 1); + } + + auto session_base = model->create_task_session( + {engine::runtime::VoiceTaskKind::Tts, + engine::runtime::RunMode::Offline}, + session_options); + auto *session = + dynamic_cast( + session_base.get()); + if (session == nullptr) + throw std::runtime_error( + "OuteTTS did not create an offline TTS session"); + session->prepare(engine::runtime::SessionPreparationRequest{}); + + std::unordered_map audio_cache; + for (size_t request_index = 0; request_index < requests.size(); + ++request_index) { + for (int iteration = 0; iteration < iterations; ++iteration) { + const auto started = std::chrono::steady_clock::now(); + auto result = session->run( + make_request(requests[request_index], audio_cache)); + const double wall_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - started) + .count(); + if (!result.audio_output.has_value()) + throw std::runtime_error("OuteTTS produced no audio"); + const double seconds = audio_seconds(*result.audio_output); + const auto output_path = + output_dir / + (requests[request_index].name + "_" + + std::to_string(iteration + 1) + ".wav"); + engine::audio::write_pcm16_wav( + output_path, result.audio_output->sample_rate, + result.audio_output->channels, result.audio_output->samples); + std::cout << "request=" << requests[request_index].name << "\n"; + std::cout << "iteration=" << iteration + 1 << "\n"; + std::cout << "wall_ms=" << wall_ms << "\n"; + std::cout << "audio_seconds=" << seconds << "\n"; + std::cout << "rtf=" + << (seconds > 0.0 ? wall_ms / 1000.0 / seconds : 0.0) + << "\n"; + std::cout << "audio_out=" << output_path.string() << "\n"; + } + } + std::cout << "log_out=" << log_file.string() << "\n"; + if (hold_seconds > 0) { + std::cout << "holding_session_seconds=" << hold_seconds << "\n"; + std::cout.flush(); + std::this_thread::sleep_for(std::chrono::seconds(hold_seconds)); + } + return 0; +} catch (const std::exception &error) { + std::cerr << "outetts_warm_bench failed: " << error.what() << "\n"; + return 1; +} diff --git a/tests/outetts/warm_bench_requests.json b/tests/outetts/warm_bench_requests.json new file mode 100644 index 00000000..57d45a70 --- /dev/null +++ b/tests/outetts/warm_bench_requests.json @@ -0,0 +1,46 @@ +{ + "requests": [ + { + "name": "tts_cold", + "text": "This is the first OuteTTS request in a long-lived session.", + "language": "en", + "max_tokens": 1024, + "seed": 1234 + }, + { + "name": "tts_repeat", + "text": "This is the first OuteTTS request in a long-lived session.", + "language": "en", + "max_tokens": 1024, + "seed": 1234 + }, + { + "name": "tts_longform", + "text": "Long-form synthesis is split by the shared framework text chunker. Each sentence becomes a bounded request while model weights and the cached generation graph stay inside the same long-lived session. The generated audio chunks are appended in order to form one output waveform.", + "language": "en", + "seed": 1234, + "text_chunk_size": 100, + "text_chunk_mode": "default" + }, + { + "name": "clone_cold", + "text": "This request builds and caches the reference voice profile.", + "language": "en", + "voice_ref": "assets/resources/b.wav", + "reference_text": "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you.", + "reference_language": "en", + "max_tokens": 1024, + "seed": 42 + }, + { + "name": "clone_repeat", + "text": "This request builds and caches the reference voice profile.", + "language": "en", + "voice_ref": "assets/resources/b.wav", + "reference_text": "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you.", + "reference_language": "en", + "max_tokens": 1024, + "seed": 42 + } + ] +} diff --git a/tests/unittests/test_outetts_generation_budget.cpp b/tests/unittests/test_outetts_generation_budget.cpp new file mode 100644 index 00000000..5b7b985a --- /dev/null +++ b/tests/unittests/test_outetts_generation_budget.cpp @@ -0,0 +1,47 @@ +#include "engine/models/outetts/tokenizer.h" + +#include "test_assert.h" + +#include + +int main() try { + using engine::models::outetts::estimate_text_generation_budget; + + const auto empty = estimate_text_generation_budget(""); + engine::test::require_eq(empty.words, int64_t{0}, "empty words"); + engine::test::require_eq(empty.non_whitespace_codepoints, int64_t{0}, + "empty codepoints"); + engine::test::require_eq(empty.recommended_max_new_tokens, int64_t{384}, + "empty minimum budget"); + + const auto words = + estimate_text_generation_budget("one two three four five six"); + engine::test::require_eq(words.words, int64_t{6}, "word count"); + engine::test::require_eq(words.non_whitespace_codepoints, int64_t{22}, + "non-whitespace codepoints"); + engine::test::require_eq(words.recommended_max_new_tokens, int64_t{560}, + "word-dominated budget"); + + const auto characters = estimate_text_generation_budget( + "supercalifragilisticexpialidocious"); + engine::test::require_eq(characters.words, int64_t{1}, + "long word count"); + engine::test::require_eq(characters.non_whitespace_codepoints, int64_t{34}, + "long word codepoints"); + engine::test::require_eq(characters.recommended_max_new_tokens, int64_t{536}, + "character-dominated budget"); + + const auto unicode = estimate_text_generation_budget(u8"Zażółć gęślą"); + engine::test::require_eq(unicode.words, int64_t{2}, "UTF-8 word count"); + engine::test::require_eq(unicode.non_whitespace_codepoints, int64_t{11}, + "UTF-8 codepoints"); + engine::test::require_eq(unicode.recommended_max_new_tokens, int64_t{384}, + "UTF-8 minimum budget"); + + std::cout << "outetts_generation_budget_test passed\n"; + return 0; +} catch (const std::exception &error) { + std::cerr << "outetts_generation_budget_test failed: " << error.what() + << "\n"; + return 1; +} diff --git a/tools/audiocpp_cli/outetts_reference.py b/tools/audiocpp_cli/outetts_reference.py new file mode 100644 index 00000000..41a3e119 --- /dev/null +++ b/tools/audiocpp_cli/outetts_reference.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Run the official OuteTTS 1.0 HF reference for audio.cpp parity. + +The script intentionally keeps the official sampling configuration. It resets +the PyTorch CPU and CUDA generators immediately before each model.generate() +call, after prompt/profile construction, so no unrelated torch operation can +consume the request's sampling stream. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import re +import time +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_REQUESTS = REPO_ROOT / "tests" / "outetts" / "warm_bench_requests.json" + + +def framework_chunks(text: str, budget: int | None) -> list[str]: + text = text.strip() + if budget is None or len(text) <= budget: + return [text] + words = list(re.finditer(r"\S+", text)) + chunks: list[str] = [] + start = 0 + while start < len(words): + hard_end = start + while hard_end < len(words): + width = words[hard_end].end() - words[start].start() + if width > budget: + break + hard_end += 1 + if hard_end == start: + hard_end += 1 + end = hard_end + if hard_end < len(words) and hard_end > start + 1: + for index in range(hard_end, start + 1, -1): + if words[index - 1].group()[-1:] in ".!?": + end = index + break + if end == hard_end: + for index in range(hard_end, start + 1, -1): + if words[index - 1].group()[-1:] in ",;:": + end = index + break + chunks.append(text[words[start].start() : words[end - 1].end()].strip()) + start = end + return chunks + + +def resolve_path(value: str, base: Path) -> Path: + path = Path(value) + return path if path.is_absolute() else (base / path).resolve() + + +def reset_seed(torch_mod: Any, seed: int, device: str) -> None: + torch_mod.manual_seed(seed) + if device == "cuda": + torch_mod.cuda.manual_seed_all(seed) + + +def memory_snapshot(torch_mod: Any, device: str) -> dict[str, float]: + result: dict[str, float] = {} + try: + import psutil + + result["rss_mb"] = psutil.Process().memory_info().rss / (1024.0 * 1024.0) + except Exception: # psutil is optional for reference generation. + pass + if device == "cuda": + result["cuda_alloc_mb"] = torch_mod.cuda.memory_allocated() / (1024.0 * 1024.0) + result["cuda_peak_mb"] = torch_mod.cuda.max_memory_allocated() / (1024.0 * 1024.0) + result["cuda_reserved_mb"] = torch_mod.cuda.memory_reserved() / (1024.0 * 1024.0) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--dac", type=Path, required=True) + parser.add_argument("--alignment-json", type=Path, required=True) + parser.add_argument("--request-file", type=Path, default=DEFAULT_REQUESTS) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cuda") + parser.add_argument("--dtype", choices=("fp32", "bf16"), default="fp32") + parser.add_argument( + "--out-dir", type=Path, default=REPO_ROOT / "build" / "logs" / "outetts_python_fp32" + ) + args = parser.parse_args() + + import torch + import torchaudio + import outetts + + model_path = args.model.resolve() + dac_path = args.dac.resolve() + request_path = args.request_file.resolve() + request_base = request_path.parent.parent.parent if request_path.parent.name == "outetts" else REPO_ROOT + output_dir = args.out_dir.resolve() + wav_dir = output_dir / "outputs" + wav_dir.mkdir(parents=True, exist_ok=True) + + dtype = torch.float32 if args.dtype == "fp32" else torch.bfloat16 + config = outetts.ModelConfig( + model_path=str(model_path), + tokenizer_path=str(model_path), + interface_version=outetts.InterfaceVersion.V3, + backend=outetts.Backend.HF, + device=args.device, + dtype=dtype, + audio_codec_path=str(dac_path), + max_seq_length=8192, + ) + interface = outetts.Interface(config) + alignment = json.loads(args.alignment_json.read_text(encoding="utf-8")) + requests = json.loads(request_path.read_text(encoding="utf-8"))["requests"] + + speakers: dict[tuple[str, str], dict[str, Any]] = {} + boundaries: dict[str, list[dict[str, Any]]] = {} + memory: list[dict[str, Any]] = [] + stdout_lines = [ + "family=outetts", + "reference=official_hf", + f"dtype={args.dtype}", + f"device={args.device}", + ] + + for request in requests: + name = str(request["name"]) + seed = int(request["seed"]) + max_new_tokens = int(request["max_tokens"]) + profile_ms = 0.0 + speaker = None + + started = time.perf_counter() + voice_ref = request.get("voice_ref") + if voice_ref: + reference_text = str(request["reference_text"]) + reference_path = resolve_path(str(voice_ref), request_base) + key = (str(reference_path), reference_text) + if key not in speakers: + profile_started = time.perf_counter() + speakers[key] = interface.audio_processor.create_speaker_from_dict( + { + "audio": {"bytes": reference_path.read_bytes()}, + "text": reference_text, + "words": [ + { + "word": item["word"], + "start": item["start_sample"] / 16000.0, + "end": item["end_sample"] / 16000.0, + } + for item in alignment + ], + } + ) + speakers[key]["interface_version"] = outetts.InterfaceVersion.V3.value + profile_ms = (time.perf_counter() - profile_started) * 1000.0 + speaker = speakers[key] + + chunks = framework_chunks(request["text"], request.get("text_chunk_size")) + audio_chunks = [] + request_boundaries = [] + if args.device == "cuda": + torch.cuda.reset_peak_memory_stats() + + for chunk in chunks: + current_speaker = copy.deepcopy(speaker) + prompt_text = interface.prompt_processor.get_completion_prompt(chunk, current_speaker) + prompt = interface._prepare_prompt(prompt_text) + prompt_tokens = int(prompt.shape[-1]) + generation = outetts.GenerationConfig( + text=chunk, + speaker=current_speaker, + generation_type=outetts.GenerationType.REGULAR, + max_length=prompt_tokens + max_new_tokens, + sampler_config=outetts.SamplerConfig( + temperature=0.4, + repetition_penalty=1.1, + repetition_range=64, + top_k=40, + top_p=0.9, + min_p=0.05, + ), + ) + + # This reset is deliberately adjacent to generate(). Prompt/profile + # work above cannot consume the model-sampling RNG stream. + reset_seed(torch, seed, args.device) + full_tokens = interface.model.generate(prompt, generation) + generated = full_tokens[prompt_tokens:] + codebooks = interface.prompt_processor.extract_audio_from_tokens(generated) + audio = interface.get_audio(generated).detach().cpu().flatten() + audio_chunks.append(audio) + request_boundaries.append( + { + "prompt_text": prompt_text, + "prompt_ids": prompt.detach().cpu().flatten().tolist(), + "generated_ids": generated, + "c1": codebooks[0], + "c2": codebooks[1], + "audio_frames_24k": int(audio.numel()), + } + ) + + if args.device == "cuda": + torch.cuda.synchronize() + wall_ms = (time.perf_counter() - started) * 1000.0 + audio = torch.cat(audio_chunks) + wav_path = wav_dir / f"{name}_1.wav" + torchaudio.save(str(wav_path), audio.unsqueeze(0), 24000, encoding="PCM_S", bits_per_sample=16) + audio_seconds = audio.numel() / 24000.0 + rtf = wall_ms / 1000.0 / audio_seconds + + snapshot = memory_snapshot(torch, args.device) + snapshot.update( + { + "request_id": name, + "wall_ms": wall_ms, + "profile_ms": profile_ms, + "audio_seconds": audio_seconds, + "rtf": rtf, + } + ) + memory.append(snapshot) + boundaries[name] = request_boundaries + stdout_lines.extend( + [ + f"request_id={name}", + f"[TIMING] request.{name}.wall_ms {wall_ms}", + f"[TIMING] request.{name}.profile_ms {profile_ms}", + f"audio_seconds={audio_seconds}", + f"rtf={rtf}", + f"audio_out={wav_path}", + ] + ) + print(f"[REF] {name}: {wall_ms:.2f} ms, {audio_seconds:.5f}s, RTF={rtf:.3f}") + + (output_dir / "command.json").write_text( + json.dumps( + { + "reference": "official OuteTTS HF", + "model": str(model_path), + "dac": str(dac_path), + "requests": str(request_path), + "dtype": args.dtype, + "device": args.device, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + (output_dir / "stdout.log").write_text("\n".join(stdout_lines) + "\n", encoding="utf-8") + (output_dir / "memory.json").write_text( + json.dumps({"requests": memory}, indent=2) + "\n", encoding="utf-8" + ) + (output_dir / "boundaries.json").write_text( + json.dumps(boundaries, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/convert_outetts_dac.py b/tools/convert_outetts_dac.py new file mode 100644 index 00000000..097a4723 --- /dev/null +++ b/tools/convert_outetts_dac.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Convert OuteTTS 1.0's official IBM DAC checkpoint to safetensors. + +The official OuteTTS runtime loads a trusted PyTorch checkpoint. audio.cpp uses +the resulting plain tensor source for both native safetensors loading and GGUF +packing; no Python or pickle loader is needed at inference time. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import dac +from safetensors.torch import save_file + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("input", type=Path, help="weights_24khz_1.5kbps_v1.0.pth") + parser.add_argument("output", type=Path, help="output model.safetensors") + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + if args.output.exists() and not args.overwrite: + raise SystemExit(f"output exists (pass --overwrite): {args.output}") + + model = dac.DAC.load(str(args.input)).cpu().eval() + if model.sample_rate != 24000 or model.hop_length != 320: + raise RuntimeError( + f"expected OuteTTS DAC at 24 kHz with hop 320, got " + f"{model.sample_rate} Hz / hop {model.hop_length}" + ) + if model.n_codebooks != 2 or model.codebook_size != 1024: + raise RuntimeError( + f"expected two 1024-entry codebooks, got " + f"{model.n_codebooks} x {model.codebook_size}" + ) + + tensors = {name: tensor.detach().contiguous() for name, tensor in model.state_dict().items()} + args.output.parent.mkdir(parents=True, exist_ok=True) + save_file( + tensors, + str(args.output), + metadata={ + "format": "pt", + "source": "ibm-research/DAC.speech.v1.0", + "checkpoint": args.input.name, + }, + ) + print(f"wrote {args.output} ({len(tensors)} tensors)") + + +if __name__ == "__main__": + main() diff --git a/tools/model_manager.py b/tools/model_manager.py index e299bb0d..5c2a10b8 100644 --- a/tools/model_manager.py +++ b/tools/model_manager.py @@ -934,6 +934,62 @@ def package_usage_examples(package: ModelPackage) -> list[str]: ), description="Installs Irodori-TTS VoiceDesign plus the sibling llm-jp tokenizer and DACVAE codec dependencies required by the framework runtime.", ), + ModelPackage( + id="outetts_1_0_1b", + display_name="OuteTTS 1.0 1B", + target_directory="Llama-OuteTTS-1.0-1B", + source=CompositeSnapshotSource( + placements=( + SnapshotPlacement( + source=SnapshotSource(repo_id="OuteAI/Llama-OuteTTS-1.0-1B"), + required_files=( + "config.json", + "generation_config.json", + "model.safetensors", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + ), + ), + SnapshotPlacement( + source=SnapshotSource(repo_id="ibm-research/DAC.speech.v1.0"), + target_subdir="../DAC.speech.v1.0", + required_files=("config.json", "weights_24khz_1.5kbps_v1.0.pth"), + ), + SnapshotPlacement( + source=SnapshotSource(repo_id="Qwen/Qwen3-ForcedAligner-0.6B"), + target_subdir="../Qwen3-ForcedAligner-0.6B", + required_files=( + "config.json", + "generation_config.json", + "model.safetensors", + "preprocessor_config.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt", + ), + ), + ), + ), + required_files=( + "config.json", + "generation_config.json", + "model.safetensors", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "../DAC.speech.v1.0/config.json", + "../DAC.speech.v1.0/model.safetensors", + "../Qwen3-ForcedAligner-0.6B/config.json", + "../Qwen3-ForcedAligner-0.6B/generation_config.json", + "../Qwen3-ForcedAligner-0.6B/model.safetensors", + "../Qwen3-ForcedAligner-0.6B/preprocessor_config.json", + "../Qwen3-ForcedAligner-0.6B/tokenizer_config.json", + "../Qwen3-ForcedAligner-0.6B/vocab.json", + "../Qwen3-ForcedAligner-0.6B/merges.txt", + ), + description="Installs OuteTTS, its IBM DAC 1.5 kbps codec, and Qwen3 Forced Aligner for reliable voice cloning.", + ), ModelPackage( id="stable_audio_3_small_music", display_name="Stable Audio 3 Small Music", @@ -1696,6 +1752,24 @@ def convert_irodori_dacvae_weights(root: Path) -> None: write_checked_safetensors(tensors, output_path, input_path, overwrite=True) +def convert_outetts_dac_weights(root: Path) -> None: + input_path = root / "weights_24khz_1.5kbps_v1.0.pth" + output_path = root / "model.safetensors" + payload = torch.load(input_path, map_location="cpu", weights_only=True) + state = checkpoint_state_dict(payload) + tensors = tensor_state_dict(state) + expected = { + "quantizer.quantizers.0.codebook.weight": (1024, 8), + "quantizer.quantizers.1.codebook.weight": (1024, 8), + "decoder.model.0.weight_v": (1536, 1024, 7), + "decoder.model.6.weight_v": (1, 96, 7), + } + for name, shape in expected.items(): + if name not in tensors or tuple(tensors[name].shape) != shape: + raise RuntimeError(f"unexpected OuteTTS DAC tensor {name}: {getattr(tensors.get(name), 'shape', None)}") + write_checked_safetensors(tensors, output_path, input_path, overwrite=True) + + def write_irodori_model_config(root: Path) -> None: input_path = root / "model.safetensors" output_path = root / "model_config.json" @@ -1792,6 +1866,10 @@ def install_composite_snapshot( dacvae_root = staged_package_root.parent / "Semantic-DACVAE-Japanese-32dim" if dacvae_root.exists(): convert_irodori_dacvae_weights(dacvae_root) + elif package.id == "outetts_1_0_1b": + dac_root = staged_package_root.parent / "DAC.speech.v1.0" + if dac_root.exists(): + convert_outetts_dac_weights(dac_root) elif package.id == "vibevoice_asr": copy_bundled_model_manager_assets( "vibevoice_1_5b",