diff --git a/CMakeLists.txt b/CMakeLists.txt index 6dece6f2..3ae11599 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -367,11 +367,9 @@ audiocpp_configure_runtime_object(engine_model_marblenet_vad) audiocpp_add_model(roformer SOURCES src/models/roformer/assets.cpp - src/models/roformer/loader.cpp src/models/roformer/runtime.cpp src/models/roformer/session.cpp INCLUDES - engine/models/roformer/loader.h engine/models/roformer/session.h LOADERS engine::models::roformer::make_mel_band_roformer_loader @@ -387,10 +385,9 @@ audiocpp_add_model(demucs src/models/demucs/frontend.cpp src/models/demucs/pipeline.cpp src/models/demucs/postprocess.cpp - src/models/demucs/loader.cpp src/models/demucs/session.cpp INCLUDES - engine/models/demucs/loader.h + engine/models/demucs/session.h LOADERS engine::models::demucs::make_htdemucs_loader ALIASES @@ -721,12 +718,11 @@ audiocpp_add_model(heartmula src/models/heartmula/assets.cpp src/models/heartmula/codec.cpp src/models/heartmula/generator.cpp - src/models/heartmula/loader.cpp src/models/heartmula/mula.cpp src/models/heartmula/session.cpp src/models/heartmula/tokenizer_text.cpp INCLUDES - engine/models/heartmula/loader.h + engine/models/heartmula/session.h LOADERS engine::models::heartmula::make_heartmula_loader ) @@ -772,10 +768,9 @@ audiocpp_add_model(hviske_asr src/models/hviske_asr/encoder.cpp src/models/hviske_asr/decoder.cpp src/models/hviske_asr/session.cpp - src/models/hviske_asr/loader.cpp src/models/hviske_asr/weights.cpp INCLUDES - engine/models/hviske_asr/loader.h + engine/models/hviske_asr/session.h LOADERS engine::models::hviske_asr::make_hviske_asr_loader ) @@ -992,9 +987,8 @@ audiocpp_add_model(seed_vc src/models/seed_vc/v1_cfm.cpp src/models/seed_vc/v2_cfm.cpp src/models/seed_vc/session.cpp - src/models/seed_vc/loader.cpp INCLUDES - engine/models/seed_vc/loader.h + engine/models/seed_vc/session.h LOADERS engine::models::seed_vc::make_seed_vc_loader ) diff --git a/docs/asr.md b/docs/asr.md index 2df1bee7..52caee2e 100644 --- a/docs/asr.md +++ b/docs/asr.md @@ -203,9 +203,17 @@ completed GGUF can therefore be moved, renamed, and passed directly to `--model` | `--top-p` | float | model default | Nucleus sampling limit. | | `--seed` | integer | random if omitted | Sampling seed. | | `--audio-chunk-mode` | `auto`, `fixed`, `none` | `auto` | Long-audio chunking mode. `auto` uses the model clip limit and speech-energy boundaries when chunking is needed. | -| `--audio-chunk-seconds` | float seconds | model config | Fixed audio chunk duration. | +| `--request-option audio_chunk_duration_sec=` | float seconds | model config | Fixed audio chunk duration. | | `--text-out` | TXT path | not set | Transcript output. The transcript is also printed to stdout. | +Compatibility aliases for existing requests: + +| Legacy option | Current option | +|---|---| +| `audio_chunk_seconds` | `audio_chunk_duration_sec` | +| `audio_chunk_duration_seconds` | `audio_chunk_duration_sec` | +| `audio_chunk_duration` | `audio_chunk_duration_sec` | + ## Nemotron ASR Nemotron ASR is an NVIDIA Nemotron 3.5 ASR RNNT model with offline and streaming sessions. It supports language prompts and optional token timestamp output. diff --git a/docs/audio_tools.md b/docs/audio_tools.md index 2ac0cfbe..347d065a 100644 --- a/docs/audio_tools.md +++ b/docs/audio_tools.md @@ -147,6 +147,13 @@ audiocpp_cli --task sep --family htdemucs --model models/htdemucs --backend cuda | `--audio` | 44.1 kHz WAV path | required | Input music mixture. | | `--out-dir` | directory | required | Directory for separated stems. | | `--backend` | `cpu`, `cuda`, `vulkan`, `metal`, `best` | `cpu` | Compute backend. | +| `--session-option htdemucs.weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | backend-dependent | Weight storage type. Defaults to `f32` for host graph planning, `f16` on CUDA, and `native` otherwise. | + +Schema-v1 option compatibility: + +| Legacy/session input | Schema-v1 option | Notes | +|---|---|---| +| `weight_type` | `htdemucs.weight_type` | Accepted as a compatibility alias for direct session-option callers. Prefer the family-prefixed form. | ## BS-RoFormer @@ -218,7 +225,7 @@ Mel-Band RoFormer is wired as a vocal/source-separation model. The CLI uses the | Modes | `offline` | | Input | 44.1 kHz music mixture WAV through `--audio` | | Output | Named separated artifacts under `--out-dir` | -| Notes | Chunking/overlap behavior is internal to the integration; no user chunk option is exposed here | +| Notes | Uses the package overlap count by default; `mel_band_roformer.num_overlap` can lower the overlap for faster inference with a quality tradeoff | ```bash audiocpp_cli --task sep --family mel_band_roformer --model models/mel-roformer-mlx --backend cuda --audio song_44k.wav --out-dir stems @@ -229,5 +236,14 @@ audiocpp_cli --task sep --family mel_band_roformer --model models/mel-roformer-m | `--audio` | 44.1 kHz WAV path | required | Input music mixture. | | `--out-dir` | directory | required | Directory for separated outputs. | | `--backend` | `cpu`, `cuda`, `vulkan`, `metal`, `best` | `cpu` | Compute backend. | +| `--session-option mel_band_roformer.weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | backend-dependent | Weight storage type. Defaults to `f32` when the backend requires a host graph plan, otherwise `native`. | +| `--session-option mel_band_roformer.num_overlap=` | integer `>= 1` | package config | Number of overlapping inference windows. Lower values improve throughput but can reduce boundary quality. | + +Schema-v1 option compatibility: + +| Legacy/session input | Schema-v1 option | Notes | +|---|---|---| +| `weight_type` | `mel_band_roformer.weight_type` | Accepted as a compatibility alias for direct session-option callers. Prefer the family-prefixed form. | +| `num_overlap` | `mel_band_roformer.num_overlap` | Accepted as a compatibility alias for direct session-option callers. Prefer the family-prefixed form. | For backend weight-type controls, use `audiocpp_cli --inspect --model --family `. diff --git a/docs/models/seed_vc.md b/docs/models/seed_vc.md index dfdcfbe3..ada7e68c 100644 --- a/docs/models/seed_vc.md +++ b/docs/models/seed_vc.md @@ -31,7 +31,7 @@ Use the V2 voice-conversion path for speech voice conversion. This route uses se | Source audio | Required | | Target voice reference | Required | | Pitch controls | Not used | -| Style conversion | `convert_style=true` is parsed but not implemented | +| Style conversion | Not exposed in the current C++ route | ```bash audiocpp_cli --task vc --family seed_vc --model models/SeedVC-MLX --backend cuda --task-route v2_vc --audio source.wav --voice-ref target.wav --out converted.wav @@ -41,15 +41,11 @@ audiocpp_cli --task vc --family seed_vc --model models/SeedVC-MLX --backend cuda |---|---|---:|---| | `--num-inference-steps` | integer | `30` | V2 CFM denoising steps. | | `--request-option length_adjust=` | float | `1.0` | Stretch or compress generated duration. | -| `--request-option intelligibility_cfg_rate=` | float | `0.7` | CFG strength for source-content intelligibility. | -| `--request-option similarity_cfg_rate=` | float | `0.7` | CFG strength for target-speaker similarity. | -| `--temperature` | float | `1.0` | V2 AR sampling temperature. | -| `--top-p` | `0..1` | `0.9` | V2 AR nucleus sampling. | -| `--repetition-penalty` | positive float | `1.0` | V2 AR repetition penalty. | +| `--request-option intelligibility_guidance_scale=` | float | `0.7` | CFG strength for source-content intelligibility. | +| `--request-option similarity_guidance_scale=` | float | `0.7` | CFG strength for target-speaker similarity. | | `--seed` | integer | random if omitted | Seed for V2 random sampling/noise. | -| `--request-option anonymization_only=true|false` | bool | `false` | Use randomized voice conditioning instead of target-style conditioning. | -| `--request-option convert_style=true|false` | bool | `false` | Parsed option; `true` is not implemented in the current C++ route. | -| `--request-option noise_file=` | path | empty | Optional deterministic noise input for validation. | +| `--request-option voice_anonymization=true|false` | bool | `false` | Use randomized voice conditioning instead of target-speaker conditioning. | +| `--request-option noise_path=` | path | empty | Optional deterministic noise input for validation. | ## V1 Whisper + BigVGAN Voice Conversion @@ -72,7 +68,7 @@ audiocpp_cli --task vc --family seed_vc --model models/SeedVC-MLX --backend cuda |---|---|---:|---| | `--num-inference-steps` | integer | `30` | V1 CFM denoising steps. | | `--request-option length_adjust=` | float | `1.0` | Stretch or compress generated duration. | -| `--request-option inference_cfg_rate=` | float | `0.7` | V1 CFM guidance strength. | +| `--request-option inference_guidance_scale=` | float | `0.7` | V1 CFM guidance strength. | | `--seed` | integer | random if omitted | Seed for V1 random noise. | ## V1 XLSR + HiFT Voice Conversion @@ -96,7 +92,7 @@ audiocpp_cli --task vc --family seed_vc --model models/SeedVC-MLX --backend cuda |---|---|---:|---| | `--num-inference-steps` | integer | `30` | V1 CFM denoising steps. | | `--request-option length_adjust=` | float | `1.0` | Stretch or compress generated duration. | -| `--request-option inference_cfg_rate=` | float | `0.7` | V1 CFM guidance strength. | +| `--request-option inference_guidance_scale=` | float | `0.7` | V1 CFM guidance strength. | | `--seed` | integer | random if omitted | Seed for V1 random noise. | ## V1 Singing Voice Conversion @@ -110,7 +106,7 @@ Use this route for singing voice conversion. It supports the V1 F0 and pitch con | Source singing | Required | | Target singer reference | Required | | F0 extraction | Optional through `f0_condition` | -| Pitch controls | `semi_tone_shift`, `auto_f0_adjust` | +| Pitch controls | `semitone_shift`, `auto_f0_adjust` | ```bash audiocpp_cli --task svc --family seed_vc --model models/SeedVC-MLX --backend cuda --task-route v1_svc --audio singing.wav --voice-ref target.wav --out svc.wav @@ -120,10 +116,10 @@ audiocpp_cli --task svc --family seed_vc --model models/SeedVC-MLX --backend cud |---|---|---:|---| | `--num-inference-steps` | integer | `30` | V1 CFM denoising steps. | | `--request-option length_adjust=` | float | `1.0` | Stretch or compress generated duration. | -| `--request-option inference_cfg_rate=` | float | `0.7` | V1 CFM guidance strength. | +| `--request-option inference_guidance_scale=` | float | `0.7` | V1 CFM guidance strength. | | `--request-option f0_condition=true|false` | bool | `false` | Enable F0-conditioned conversion. | | `--request-option auto_f0_adjust=true|false` | bool | `false` | Automatically adjust F0 when F0 conditioning is enabled. | -| `--request-option semi_tone_shift=` | integer | `0` | Shift pitch by semitones when F0 conditioning is enabled. | +| `--request-option semitone_shift=` | integer | `0` | Shift pitch by semitones when F0 conditioning is enabled. | | `--seed` | integer | random if omitted | Seed for V1 random noise. | ## Shared Controls @@ -134,3 +130,16 @@ audiocpp_cli --task svc --family seed_vc --model models/SeedVC-MLX --backend cud | `--audio` | WAV path | required | Source speech or singing audio. | | `--voice-ref` | WAV path | required | Target voice or singer reference. | | `--session-option seed_vc.weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Weight storage type. | + +## Compatibility + +The schema-v1 option names are the preferred names. Existing local requests using the old names are still accepted through this compatibility map: + +| Old request option | New request option | +|---|---| +| `inference_cfg_rate` | `inference_guidance_scale` | +| `intelligibility_cfg_rate` | `intelligibility_guidance_scale` | +| `similarity_cfg_rate` | `similarity_guidance_scale` | +| `anonymization_only` | `voice_anonymization` | +| `noise_file` | `noise_path` | +| `semi_tone_shift` | `semitone_shift` | diff --git a/docs/music_generation.md b/docs/music_generation.md index 85188685..c49a35a7 100644 --- a/docs/music_generation.md +++ b/docs/music_generation.md @@ -110,17 +110,39 @@ audiocpp_cli --task gen --family heartmula --model models/HeartMuLa --backend cu | `--text` | text | required | Music prompt or short description. | | `--lyrics` | text | empty string | Lyrics for generated music. | | `--request-option tags=` | comma-separated text | required | Music tags; the model path wraps them as tag tokens internally. | -| `--duration-seconds` | seconds | `120` | Maximum generated duration. | +| `--request-option duration_sec=` | seconds | `120` | Maximum generated duration. | | `--temperature` | float | `1.0` | Music-token sampling temperature. | | `--top-k` | integer | `50` | Music-token top-k sampling limit. | | `--guidance-scale` | float | `1.5` | MuLa classifier-free guidance scale. | | `--num-inference-steps` | integer | `10` | Codec flow solver steps. | -| `--request-option codec_duration=` | seconds | `29.76` | Codec detokenization chunk duration. | +| `--request-option codec_duration_sec=` | seconds | `29.76` | Codec detokenization chunk duration. | | `--request-option codec_guidance_scale=` | float | `1.25` | Codec classifier-free guidance scale. | | `--request-option infinite_mode=true|false` | bool | `false` | Generate long outputs by splitting lyrics into bounded HeartMuLa requests. | | `--text-chunk-size` | chars | `4096` | Text chunk size for infinite mode. | -| `--request-option infinite_chunk_audio_length_ms=` | milliseconds | `240000` | Per-chunk audio cap for infinite mode. | +| `--request-option infinite_chunk_audio_duration_ms=` | milliseconds | `240000` | Per-chunk audio cap for infinite mode. | | `--seed` | integer | `1234` | Generation seed. | +| `--session-option heartmula.weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | MuLa and codec weight storage type. | +| `--session-option heartmula.generator_weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `heartmula.weight_type` or `native` | MuLa music-token generator weight storage type. | +| `--session-option heartmula.codec_weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `heartmula.weight_type` or `native` | Codec weight storage type. | | `--session-option heartmula.mem_saver=true|false` | bool | `false` | Release staged graph/cache state after AR/codec phases and infinite-mode chunks to reduce resident VRAM. Later requests may rebuild released graphs. | -For backend weight-type controls, use `audiocpp_cli --inspect --model --family `. +Compatibility mapping: + +| Legacy option | Schema-v1 option | +|---|---| +| `duration_seconds` | `duration_sec` | +| `codec_duration` | `codec_duration_sec` | +| `infinite_chunk_audio_length_ms` | `infinite_chunk_audio_duration_ms` | +| `heartmula.mula_weight_type` | `heartmula.generator_weight_type` | +| `heartmula.mula_weight_context_mb` | `heartmula.generator_weight_context_mb` | +| `heartmula.mula_constant_context_mb` | `heartmula.generator_constant_context_mb` | +| `heartmula.mula_backbone_prefill_graph_arena_mb` | `heartmula.backbone_prefill_graph_arena_mb` | +| `heartmula.mula_backbone_step_graph_arena_mb` | `heartmula.backbone_step_graph_arena_mb` | +| `heartmula.mula_decoder_prefill_graph_arena_mb` | `heartmula.decoder_prefill_graph_arena_mb` | +| `heartmula.mula_decoder_step_graph_arena_mb` | `heartmula.decoder_step_graph_arena_mb` | +| `heartmula.mula_frame_embedding_graph_arena_mb` | `heartmula.frame_embedding_graph_arena_mb` | +| `heartmula.codec_flow_estimator_graph_arena_mb` | `heartmula.flow_estimator_graph_arena_mb` | +| `heartmula.codec_conditioning_graph_arena_mb` | `heartmula.conditioning_graph_arena_mb` | +| `heartmula.codec_scalar_decoder_graph_arena_mb` | `heartmula.scalar_decoder_graph_arena_mb` | + +For the full backend memory-arena controls, use `audiocpp_cli --help --model --family heartmula`. diff --git a/include/engine/framework/runtime/options.h b/include/engine/framework/runtime/options.h index 13ca346c..404dd13c 100644 --- a/include/engine/framework/runtime/options.h +++ b/include/engine/framework/runtime/options.h @@ -41,6 +41,10 @@ std::optional parse_finite_float_option( const std::unordered_map & options, std::initializer_list keys); +std::optional parse_positive_finite_float_option( + const std::unordered_map & options, + std::initializer_list keys); + std::optional parse_u32_option( const std::unordered_map & options, std::initializer_list keys); @@ -69,6 +73,13 @@ assets::TensorStorageType parse_tensor_storage_option( assets::TensorStorageType fallback, std::initializer_list allowed); +assets::TensorStorageType parse_tensor_storage_option( + const std::unordered_map & options, + std::string_view key, + std::string_view fallback_key, + assets::TensorStorageType fallback, + std::initializer_list allowed); + uint32_t random_u32_seed(); uint64_t random_u64_seed(); diff --git a/include/engine/framework/runtime/spec_backed_model.h b/include/engine/framework/runtime/spec_backed_model.h index 285a5c20..59921d9c 100644 --- a/include/engine/framework/runtime/spec_backed_model.h +++ b/include/engine/framework/runtime/spec_backed_model.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,11 @@ namespace engine::runtime { +struct OptionV1CompatibilityAlias { + std::string legacy_key; + std::string current_key; +}; + template struct SpecBackedVoiceModelConfig { std::string family; @@ -59,6 +65,42 @@ inline void validate_spec_backed_request_options( } } +inline std::unordered_map apply_option_v1_compatibility( + std::unordered_map options, + std::initializer_list aliases, + std::string_view model_name, + std::string_view scope_name) { + for (const auto & alias : aliases) { + auto legacy = options.find(alias.legacy_key); + if (legacy == options.end()) { + continue; + } + auto current = options.find(alias.current_key); + if (current != options.end()) { + throw std::runtime_error( + std::string(model_name) + " " + std::string(scope_name) + + " options contain both " + alias.legacy_key + " and " + + alias.current_key + "; use " + alias.current_key); + } + std::string value = std::move(legacy->second); + options.erase(legacy); + options.emplace(alias.current_key, std::move(value)); + } + return options; +} + +inline SessionOptions apply_option_v1_compatibility( + SessionOptions options, + std::initializer_list aliases, + std::string_view model_name) { + options.options = apply_option_v1_compatibility( + std::move(options.options), + aliases, + model_name, + "session"); + return options; +} + template class SpecBackedLoadedVoiceModel final : public ILoadedVoiceModel { public: diff --git a/include/engine/models/demucs/assets.h b/include/engine/models/demucs/assets.h index d3066d7a..e13b5356 100644 --- a/include/engine/models/demucs/assets.h +++ b/include/engine/models/demucs/assets.h @@ -2,8 +2,8 @@ #include "engine/framework/assets/resource_bundle.h" #include "engine/framework/assets/tensor_source.h" -#include "engine/framework/runtime/model.h" +#include #include #include #include @@ -83,6 +83,6 @@ struct HTDemucsAssets { }; void validate_demucs_weight_storage_type(assets::TensorStorageType storage_type); -std::shared_ptr load_htdemucs_assets(const runtime::ModelLoadRequest & request); +std::shared_ptr load_htdemucs_assets(const std::filesystem::path & model_path); } // namespace engine::models::demucs diff --git a/include/engine/models/demucs/loader.h b/include/engine/models/demucs/loader.h deleted file mode 100644 index d68b5ad6..00000000 --- a/include/engine/models/demucs/loader.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "engine/framework/runtime/model.h" -#include "engine/models/demucs/assets.h" - -#include - -namespace engine::models::demucs { - -class HTDemucsLoadedModel final : public runtime::ILoadedVoiceModel { -public: - HTDemucsLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets); - - const runtime::ModelMetadata & metadata() const noexcept override; - const runtime::CapabilitySet & capabilities() const noexcept override; - std::unique_ptr create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const override; - -private: - runtime::ModelMetadata metadata_; - runtime::CapabilitySet capabilities_; - std::shared_ptr assets_; -}; - -std::unique_ptr load_htdemucs_model( - const runtime::ModelLoadRequest & request); -std::shared_ptr make_htdemucs_loader(); - -} // namespace engine::models::demucs diff --git a/include/engine/models/demucs/session.h b/include/engine/models/demucs/session.h index 96066380..74a9d7c8 100644 --- a/include/engine/models/demucs/session.h +++ b/include/engine/models/demucs/session.h @@ -1,5 +1,6 @@ #pragma once +#include "engine/framework/model_spec/metadata.h" #include "engine/framework/runtime/session_base.h" #include "engine/models/demucs/assets.h" #include "engine/models/demucs/pipeline.h" @@ -9,14 +10,17 @@ namespace engine::models::demucs { +std::shared_ptr make_htdemucs_loader(); + class HTDemucsSession final : public runtime::RuntimeSessionBase , public runtime::IOfflineVoiceTaskSession { public: HTDemucsSession( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options, - std::shared_ptr assets); + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); ~HTDemucsSession() override; std::string family() const override; @@ -28,6 +32,7 @@ class HTDemucsSession final private: runtime::TaskSpec task_; std::shared_ptr assets_; + std::shared_ptr contract_; assets::TensorStorageType weight_storage_type_ = assets::TensorStorageType::Native; std::unique_ptr pipeline_; int64_t chunk_size_ = 0; diff --git a/include/engine/models/heartmula/codec.h b/include/engine/models/heartmula/codec.h index 41490fe1..9348195a 100644 --- a/include/engine/models/heartmula/codec.h +++ b/include/engine/models/heartmula/codec.h @@ -129,7 +129,7 @@ class HeartCodecWeightsRuntime final { HeartCodecWeightsRuntime( std::shared_ptr assets, core::ExecutionContext & execution_context, - size_t weight_context_bytes = 512ull * 1024ull * 1024ull, + size_t weight_context_bytes = 32ull * 1024ull * 1024ull, size_t flow_estimator_graph_arena_bytes = 2048ull * 1024ull * 1024ull, size_t conditioning_graph_arena_bytes = 512ull * 1024ull * 1024ull, size_t scalar_decoder_graph_arena_bytes = 1536ull * 1024ull * 1024ull, diff --git a/include/engine/models/heartmula/loader.h b/include/engine/models/heartmula/loader.h deleted file mode 100644 index f0ff760d..00000000 --- a/include/engine/models/heartmula/loader.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "engine/framework/runtime/model.h" -#include "engine/models/heartmula/assets.h" - -#include -#include - -namespace engine::models::heartmula { - -class HeartMuLaLoadedModel final : public runtime::ILoadedVoiceModel { -public: - HeartMuLaLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets); - - const runtime::ModelMetadata & metadata() const noexcept override; - const runtime::CapabilitySet & capabilities() const noexcept override; - std::unique_ptr create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const override; - -private: - runtime::ModelMetadata metadata_; - runtime::CapabilitySet capabilities_; - std::shared_ptr assets_; -}; - -std::unique_ptr load_heartmula_model(const std::filesystem::path & model_path); -std::shared_ptr make_heartmula_loader(); - -} // namespace engine::models::heartmula diff --git a/include/engine/models/heartmula/mula.h b/include/engine/models/heartmula/mula.h index dc1fbffb..77e43b6c 100644 --- a/include/engine/models/heartmula/mula.h +++ b/include/engine/models/heartmula/mula.h @@ -170,7 +170,7 @@ class HeartMuLaWeightsRuntime final { core::BackendType backend_type, int device, int threads, - size_t weight_context_bytes = 512ull * 1024ull * 1024ull, + size_t weight_context_bytes = 32ull * 1024ull * 1024ull, size_t constant_context_bytes = 256ull * 1024ull * 1024ull, size_t backbone_prefill_graph_arena_bytes = 1536ull * 1024ull * 1024ull, size_t backbone_step_graph_arena_bytes = 1536ull * 1024ull * 1024ull, diff --git a/include/engine/models/heartmula/session.h b/include/engine/models/heartmula/session.h index 84c36513..fb7e37a6 100644 --- a/include/engine/models/heartmula/session.h +++ b/include/engine/models/heartmula/session.h @@ -1,5 +1,7 @@ #pragma once +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/model.h" #include "engine/framework/runtime/session_base.h" #include "engine/models/heartmula/assets.h" #include "engine/models/heartmula/codec.h" @@ -20,7 +22,8 @@ class HeartMuLaSession final HeartMuLaSession( runtime::TaskSpec task, runtime::SessionOptions options, - std::shared_ptr assets); + std::shared_ptr assets, + std::shared_ptr contract); std::string family() const override; runtime::VoiceTaskKind task_kind() const override; @@ -33,14 +36,15 @@ class HeartMuLaSession final runtime::TaskSpec task_; std::shared_ptr assets_; - size_t mula_weight_context_bytes_ = 512ull * 1024ull * 1024ull; - size_t mula_constant_context_bytes_ = 256ull * 1024ull * 1024ull; - size_t mula_backbone_prefill_graph_arena_bytes_ = 1536ull * 1024ull * 1024ull; - size_t mula_backbone_step_graph_arena_bytes_ = 1536ull * 1024ull * 1024ull; - size_t mula_decoder_prefill_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; - size_t mula_decoder_step_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; - size_t mula_frame_embedding_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; - size_t codec_weight_context_bytes_ = 512ull * 1024ull * 1024ull; + std::shared_ptr contract_; + size_t generator_weight_context_bytes_ = 32ull * 1024ull * 1024ull; + size_t generator_constant_context_bytes_ = 256ull * 1024ull * 1024ull; + size_t generator_backbone_prefill_graph_arena_bytes_ = 1536ull * 1024ull * 1024ull; + size_t generator_backbone_step_graph_arena_bytes_ = 1536ull * 1024ull * 1024ull; + size_t generator_decoder_prefill_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; + size_t generator_decoder_step_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; + size_t generator_frame_embedding_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; + size_t codec_weight_context_bytes_ = 32ull * 1024ull * 1024ull; size_t codec_flow_estimator_graph_arena_bytes_ = 2048ull * 1024ull * 1024ull; size_t codec_conditioning_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; size_t codec_scalar_decoder_graph_arena_bytes_ = 1536ull * 1024ull * 1024ull; @@ -52,4 +56,6 @@ class HeartMuLaSession final HeartCodecWeightsRuntime codec_; }; +std::shared_ptr make_heartmula_loader(); + } // namespace engine::models::heartmula diff --git a/include/engine/models/hviske_asr/loader.h b/include/engine/models/hviske_asr/loader.h deleted file mode 100644 index 1e620891..00000000 --- a/include/engine/models/hviske_asr/loader.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "engine/framework/runtime/model.h" -#include "engine/models/hviske_asr/assets.h" - -#include -#include - -namespace engine::models::hviske_asr { - -class HviskeASRLoadedModel final : public runtime::ILoadedVoiceModel { -public: - HviskeASRLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets); - - const runtime::ModelMetadata & metadata() const noexcept override; - const runtime::CapabilitySet & capabilities() const noexcept override; - std::unique_ptr create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const override; - -private: - runtime::ModelMetadata metadata_; - runtime::CapabilitySet capabilities_; - std::shared_ptr assets_; -}; - -std::unique_ptr load_hviske_asr_model(const std::filesystem::path & model_path); -std::shared_ptr make_hviske_asr_loader(); - -} // namespace engine::models::hviske_asr diff --git a/include/engine/models/hviske_asr/session.h b/include/engine/models/hviske_asr/session.h index c8ad616d..a3b797c9 100644 --- a/include/engine/models/hviske_asr/session.h +++ b/include/engine/models/hviske_asr/session.h @@ -1,6 +1,7 @@ #pragma once -#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/model.h" #include "engine/framework/runtime/session_base.h" #include "engine/models/hviske_asr/assets.h" #include "engine/models/hviske_asr/decoder.h" @@ -16,6 +17,8 @@ namespace engine::models::hviske_asr { +std::shared_ptr make_hviske_asr_loader(); + class HviskeASRSession final : public runtime::RuntimeSessionBase , public runtime::IOfflineVoiceTaskSession { @@ -23,7 +26,8 @@ class HviskeASRSession final HviskeASRSession( runtime::TaskSpec task, runtime::SessionOptions options, - std::shared_ptr assets); + std::shared_ptr assets, + std::shared_ptr contract); ~HviskeASRSession() override; std::string family() const override; @@ -40,14 +44,18 @@ class HviskeASRSession final std::vector prepare_segments( const runtime::AudioBuffer & audio, const std::unordered_map & options) const; - std::string language_for_request(const runtime::TaskRequest & request) const; - bool punctuation_for_request(const runtime::TaskRequest & request) const; - HviskeDecodingOptions decoding_options_for_request(const runtime::TaskRequest & request) const; + std::string language_for_request( + const runtime::TaskRequest & request, + const std::unordered_map & options) const; + bool punctuation_for_request(const std::unordered_map & options) const; + HviskeDecodingOptions decoding_options_for_request( + const std::unordered_map & options) const; runtime::TaskSpec task_; std::shared_ptr assets_; + std::shared_ptr contract_; std::shared_ptr weights_; - size_t weight_context_bytes_ = 512ull * 1024ull * 1024ull; + size_t weight_context_bytes_ = 32ull * 1024ull * 1024ull; size_t encoder_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; size_t decoder_prefill_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; size_t decoder_decode_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; diff --git a/include/engine/models/roformer/loader.h b/include/engine/models/roformer/loader.h deleted file mode 100644 index 3cdee7e1..00000000 --- a/include/engine/models/roformer/loader.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "engine/framework/runtime/model.h" -#include "engine/models/roformer/assets.h" - -#include - -namespace engine::models::roformer { - -class RoformerLoadedModel final : public runtime::ILoadedVoiceModel { -public: - RoformerLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets); - - const runtime::ModelMetadata & metadata() const noexcept override; - const runtime::CapabilitySet & capabilities() const noexcept override; - std::unique_ptr create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const override; - -private: - runtime::ModelMetadata metadata_; - runtime::CapabilitySet capabilities_; - std::shared_ptr assets_; -}; - -std::unique_ptr load_roformer_model( - const runtime::ModelLoadRequest & request, - std::string_view family); -std::shared_ptr make_mel_band_roformer_loader(); - -} // namespace engine::models::roformer diff --git a/include/engine/models/roformer/session.h b/include/engine/models/roformer/session.h index 62b1ec43..005c1a91 100644 --- a/include/engine/models/roformer/session.h +++ b/include/engine/models/roformer/session.h @@ -10,6 +10,7 @@ namespace engine::models::roformer { +std::shared_ptr make_mel_band_roformer_loader(); std::shared_ptr make_bs_roformer_loader(); class RoformerSession final @@ -18,11 +19,7 @@ class RoformerSession final public: RoformerSession( const runtime::TaskSpec & task, - const runtime::SessionOptions & options, - std::shared_ptr assets); - RoformerSession( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options, + runtime::SessionOptions options, std::shared_ptr assets, std::shared_ptr contract); ~RoformerSession() override; diff --git a/include/engine/models/seed_vc/loader.h b/include/engine/models/seed_vc/loader.h deleted file mode 100644 index 8e301d2b..00000000 --- a/include/engine/models/seed_vc/loader.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "engine/framework/runtime/model.h" - -#include - -namespace engine::models::seed_vc { - -std::shared_ptr make_seed_vc_loader(); - -} // namespace engine::models::seed_vc diff --git a/include/engine/models/seed_vc/session.h b/include/engine/models/seed_vc/session.h index 9146df79..ed7182d0 100644 --- a/include/engine/models/seed_vc/session.h +++ b/include/engine/models/seed_vc/session.h @@ -1,5 +1,7 @@ #pragma once +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/model.h" #include "engine/framework/runtime/session_base.h" #include "engine/models/seed_vc/assets.h" @@ -9,6 +11,8 @@ namespace engine::models::seed_vc { +std::shared_ptr make_seed_vc_loader(); + struct SeedVcRouteRuntime; class SeedVcSession final @@ -18,7 +22,8 @@ class SeedVcSession final SeedVcSession( runtime::TaskSpec task, runtime::SessionOptions options, - std::shared_ptr assets); + std::shared_ptr assets, + std::shared_ptr contract); std::string family() const override; runtime::VoiceTaskKind task_kind() const override; @@ -29,6 +34,7 @@ class SeedVcSession final private: runtime::TaskSpec task_; std::shared_ptr assets_; + std::shared_ptr contract_; std::shared_ptr route_runtime_; std::optional weight_storage_type_; }; diff --git a/model_specs/heartmula.json b/model_specs/heartmula.json index 2d81756c..0392f868 100644 --- a/model_specs/heartmula.json +++ b/model_specs/heartmula.json @@ -1,4 +1,5 @@ { + "schema_version": 1, "family": "heartmula", "display_name": "HeartMuLa", "description": "Open-source music foundation model for generating complete songs from text, lyric prompts, and style tags, with controllable song structure and broad multilingual lyric conditioning.", @@ -15,9 +16,233 @@ ], "capabilities": { "music": [ - "lyrics" + "lyrics", + "style_control" ] }, + "options": { + "request": [ + { + "name": "lyrics", + "type": "string", + "description": "Lyrics text.", + "required": false + }, + { + "name": "tags", + "type": "string", + "description": "Comma-separated music tags.", + "required": false + }, + { + "name": "duration_sec", + "type": "float", + "description": "Maximum generated audio duration in seconds; must be positive when set; default 120.", + "required": false, + "min": 0.0, + "default": 120.0 + }, + { + "name": "temperature", + "type": "float", + "description": "Audio-token sampling temperature; must be positive when set; default 1.0.", + "required": false, + "min": 0.0, + "default": 1.0 + }, + { + "name": "top_k", + "type": "int", + "description": "Audio-token top-k sampling limit; default 50.", + "required": false, + "min": 1, + "default": 50 + }, + { + "name": "guidance_scale", + "type": "float", + "description": "MuLa classifier-free guidance scale; must be positive when set; default 1.5.", + "required": false, + "min": 0.0, + "default": 1.5 + }, + { + "name": "codec_duration_sec", + "type": "float", + "description": "Codec detokenization chunk duration in seconds; must be positive when set; default 29.76.", + "required": false, + "min": 0.0, + "default": 29.76 + }, + { + "name": "num_inference_steps", + "type": "int", + "description": "Codec flow solver steps; default 10.", + "required": false, + "min": 1, + "default": 10 + }, + { + "name": "codec_guidance_scale", + "type": "float", + "description": "Codec classifier-free guidance scale; must be positive when set; default 1.25.", + "required": false, + "min": 0.0, + "default": 1.25 + }, + { + "name": "infinite_mode", + "type": "bool", + "description": "Generate long outputs by splitting lyrics into bounded HeartMuLa requests.", + "required": false, + "default": false + }, + { + "name": "text_chunk_size", + "type": "int", + "description": "Maximum characters per text chunk in infinite mode; default 4096.", + "required": false, + "min": 1, + "default": 4096 + }, + { + "name": "infinite_chunk_audio_duration_ms", + "type": "int", + "description": "Maximum generated audio duration per infinite-mode chunk in milliseconds; default 240000.", + "required": false, + "min": 1, + "default": 240000 + }, + { + "name": "seed", + "type": "int", + "description": "Generation seed for reproducible output; default 1234.", + "required": false, + "min": 0, + "default": 1234 + } + ], + "session": [ + { + "name": "weight_type", + "type": "enum", + "description": "MuLa and codec weight storage type; default native.", + "preset": "weight_type_full", + "required": false, + "default": "native" + }, + { + "name": "generator_weight_type", + "type": "enum", + "description": "MuLa music-token generator weight storage type; defaults to weight_type when set, otherwise native.", + "preset": "weight_type_full", + "required": false + }, + { + "name": "codec_weight_type", + "type": "enum", + "description": "Codec weight storage type; defaults to weight_type when set, otherwise native.", + "preset": "weight_type_full", + "required": false + }, + { + "name": "generator_weight_context_mb", + "type": "int", + "description": "MuLa music-token generator weight descriptor context size in MiB; default 32.", + "required": false, + "min": 1, + "default": 32 + }, + { + "name": "codec_weight_context_mb", + "type": "int", + "description": "Codec weight descriptor context size in MiB; default 32.", + "required": false, + "min": 1, + "default": 32 + }, + { + "name": "generator_constant_context_mb", + "type": "int", + "description": "MuLa music-token generator reusable constant context size in MiB; default 256.", + "required": false, + "min": 1, + "default": 256 + }, + { + "name": "backbone_prefill_graph_arena_mb", + "type": "int", + "description": "MuLa music-token generator backbone prefill graph arena size in MiB; default 1536.", + "required": false, + "min": 1, + "default": 1536 + }, + { + "name": "backbone_step_graph_arena_mb", + "type": "int", + "description": "MuLa music-token generator backbone cached-step graph arena size in MiB; default 1536.", + "required": false, + "min": 1, + "default": 1536 + }, + { + "name": "decoder_prefill_graph_arena_mb", + "type": "int", + "description": "MuLa music-token generator decoder prefill graph arena size in MiB; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "decoder_step_graph_arena_mb", + "type": "int", + "description": "MuLa music-token generator decoder cached-step graph arena size in MiB; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "frame_embedding_graph_arena_mb", + "type": "int", + "description": "MuLa music-token generator frame-embedding graph arena size in MiB; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "flow_estimator_graph_arena_mb", + "type": "int", + "description": "Codec flow-estimator graph arena size in MiB; default 2048.", + "required": false, + "min": 1, + "default": 2048 + }, + { + "name": "conditioning_graph_arena_mb", + "type": "int", + "description": "Codec conditioning graph arena size in MiB; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "scalar_decoder_graph_arena_mb", + "type": "int", + "description": "Codec scalar-decoder graph arena size in MiB; default 1536.", + "required": false, + "min": 1, + "default": 1536 + }, + { + "name": "mem_saver", + "type": "bool", + "description": "Release staged runtime graphs after each request; default false.", + "required": false, + "default": false + } + ], + "load": [] + }, "runtime": { "tags": [ "gguf" @@ -30,7 +255,7 @@ "GGUF" ], "docs": [ - "docs/tts.md", + "docs/music_generation.md", "docs/gguf.md" ] }, @@ -67,6 +292,7 @@ "strip_prefix": "HeartMuLa-GGUF" } ], + "dependencies": [], "sources": [ { "format": "gguf", diff --git a/model_specs/htdemucs.json b/model_specs/htdemucs.json index 1ea7ecba..d19c38c6 100644 --- a/model_specs/htdemucs.json +++ b/model_specs/htdemucs.json @@ -1,4 +1,5 @@ { + "schema_version": 1, "family": "htdemucs", "display_name": "HTDemucs", "description": "Hybrid Transformer Demucs music source-separation model that combines waveform and spectrogram processing with Transformer layers to split mixtures into vocals, drums, bass, and accompaniment.", @@ -18,6 +19,19 @@ "stems" ] }, + "options": { + "request": [], + "session": [ + { + "name": "weight_type", + "type": "enum", + "description": "HTDemucs weight storage type; default is f32 for host graph planning, f16 on CUDA, otherwise native.", + "preset": "weight_type_full", + "required": false + } + ], + "load": [] + }, "runtime": { "tags": [ "gguf" @@ -67,6 +81,7 @@ "strip_prefix": "HTDemucs-GGUF" } ], + "dependencies": [], "sources": [ { "format": "gguf", diff --git a/model_specs/hviske_asr.json b/model_specs/hviske_asr.json index 647913be..2cc69730 100644 --- a/model_specs/hviske_asr.json +++ b/model_specs/hviske_asr.json @@ -1,5 +1,6 @@ { "family": "hviske_asr", + "schema_version": 1, "display_name": "Hviske ASR", "description": "Danish-optimized Conformer encoder-decoder ASR model fine-tuned from the Hviske v5 family, with selectable Cohere ASR language prompts.", "category": "asr", @@ -28,6 +29,157 @@ "zh" ], "capabilities": {}, + "dependencies": [], + "options": { + "request": [ + { + "name": "language", + "type": "string", + "description": "ASR language code; default da.", + "required": false, + "default": "da" + }, + { + "name": "punctuation", + "type": "bool", + "description": "Enable or disable punctuation tokens in the decoder prompt; default true.", + "required": false, + "default": true + }, + { + "name": "max_tokens", + "type": "int", + "description": "Maximum generated transcript tokens; defaults to the model config.", + "required": false, + "min": 1 + }, + { + "name": "num_beams", + "type": "int", + "description": "Beam-search beam count; default 1 uses greedy or sampling decode.", + "required": false, + "min": 1, + "default": 1 + }, + { + "name": "length_penalty", + "type": "float", + "description": "Beam-search length penalty; must be positive when set; default 1.0.", + "required": false, + "min": 0.000001, + "default": 1.0 + }, + { + "name": "do_sample", + "type": "bool", + "description": "Enable sampling instead of greedy decode when num_beams is 1; default false.", + "required": false, + "default": false + }, + { + "name": "temperature", + "type": "float", + "description": "Decoder sampling temperature; must be positive when set; default 1.0.", + "required": false, + "min": 0.000001, + "default": 1.0 + }, + { + "name": "top_k", + "type": "int", + "description": "Top-k sampling limit; default 50, 0 disables top-k.", + "required": false, + "min": 0, + "default": 50 + }, + { + "name": "top_p", + "type": "float", + "description": "Nucleus sampling limit in (0, 1]; default 1.0.", + "required": false, + "min": 0.000001, + "max": 1.0, + "default": 1.0 + }, + { + "name": "seed", + "type": "int", + "description": "Decoder sampling seed; random if omitted.", + "required": false, + "min": 0 + }, + { + "name": "audio_chunk_mode", + "type": "enum", + "description": "Audio chunking mode; default auto uses quiet-energy splitting only when audio exceeds the model clip window.", + "values": [ + "auto", + "fixed", + "quiet_energy", + "none" + ], + "required": false, + "default": "auto" + }, + { + "name": "audio_chunk_duration_sec", + "type": "float", + "description": "Maximum audio chunk duration in seconds; defaults to the model clip window.", + "required": false, + "min": 0.000001 + } + ], + "session": [ + { + "name": "weight_type", + "type": "enum", + "description": "Matmul weight storage type; default native.", + "preset": "weight_type_full", + "required": false, + "default": "native" + }, + { + "name": "conv_weight_type", + "type": "enum", + "description": "Convolution weight storage type; defaults to weight_type when set, otherwise native.", + "preset": "weight_type_conv", + "required": false + }, + { + "name": "weight_context_mb", + "type": "int", + "description": "Weight descriptor context size in MiB; default 32.", + "required": false, + "min": 1, + "default": 32 + }, + { + "name": "encoder_graph_arena_mb", + "type": "int", + "description": "Encoder graph arena size in MiB; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "decoder_prefill_graph_arena_mb", + "type": "int", + "description": "Decoder prefill graph arena size in MiB; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "decoder_decode_graph_arena_mb", + "type": "int", + "description": "Decoder cached-step graph arena size in MiB; default 512.", + "required": false, + "min": 1, + "default": 512 + } + ], + "load": [] + }, "runtime": { "tags": [ "gguf" diff --git a/model_specs/mel_band_roformer.json b/model_specs/mel_band_roformer.json index 664cb56c..b866db3b 100644 --- a/model_specs/mel_band_roformer.json +++ b/model_specs/mel_band_roformer.json @@ -1,4 +1,5 @@ { + "schema_version": 1, "family": "mel_band_roformer", "display_name": "Mel-Band RoFormer", "description": "Music source-separation model that uses mel-scaled overlapping frequency bands and RoFormer attention to estimate stems such as vocals, drums, and accompaniment.", @@ -18,6 +19,26 @@ "stems" ] }, + "options": { + "request": [], + "session": [ + { + "name": "weight_type", + "type": "enum", + "description": "RoFormer weight storage type. Defaults to f32 when the backend requires a host graph plan, otherwise native.", + "preset": "weight_type_full", + "required": false + }, + { + "name": "num_overlap", + "type": "int", + "description": "Number of overlapping inference windows; defaults to the package configuration. Lower values improve throughput but can reduce boundary quality.", + "required": false, + "min": 1 + } + ], + "load": [] + }, "runtime": { "tags": [ "gguf" @@ -82,6 +103,7 @@ } } ], + "dependencies": [], "sources": [ { "format": "gguf", diff --git a/model_specs/seed_vc.json b/model_specs/seed_vc.json index 2a7b9076..0a0bebf1 100644 --- a/model_specs/seed_vc.json +++ b/model_specs/seed_vc.json @@ -1,5 +1,6 @@ { "family": "seed_vc", + "schema_version": 1, "display_name": "Seed-VC", "description": "Zero-shot voice conversion and singing voice conversion model for transferring timbre and style from reference audio, with low-latency realtime conversion and optional lightweight fine-tuning.", "category": "voice_conversion", @@ -23,6 +24,7 @@ "singing" ] }, + "dependencies": [], "runtime": { "tags": [ "gguf" @@ -40,6 +42,114 @@ "docs/gguf.md" ] }, + "options": { + "request": [ + { + "name": "route", + "type": "enum", + "description": "Select the Seed-VC conversion route. Defaults to v2_vc for VC and v1_svc for SVC.", + "values": [ + "v2_vc", + "v1_svc", + "v1_whisper_bigvgan_vc", + "v1_xlsr_hift_vc" + ], + "required": false + }, + { + "name": "length_adjust", + "type": "float", + "description": "Output duration multiplier; must be positive, default 1.0.", + "required": false, + "min": 0.0, + "default": 1.0 + }, + { + "name": "num_inference_steps", + "type": "int", + "description": "Diffusion steps; default 30.", + "required": false, + "min": 1, + "default": 30 + }, + { + "name": "inference_guidance_scale", + "type": "float", + "description": "V1 classifier-free guidance scale; default 0.7.", + "required": false, + "min": 0.0, + "default": 0.7 + }, + { + "name": "intelligibility_guidance_scale", + "type": "float", + "description": "V2 classifier-free guidance scale for source-content intelligibility; default 0.7.", + "required": false, + "min": 0.0, + "default": 0.7 + }, + { + "name": "similarity_guidance_scale", + "type": "float", + "description": "V2 classifier-free guidance scale for target-speaker similarity; default 0.7.", + "required": false, + "min": 0.0, + "default": 0.7 + }, + { + "name": "voice_anonymization", + "type": "bool", + "description": "Use randomized average-voice conditioning instead of target-speaker conditioning for V2 anonymization; default false.", + "required": false, + "default": false + }, + { + "name": "seed", + "type": "int", + "description": "Seed for V1/V2 diffusion noise and HiFT stochastic source excitation; omitted requests choose a random seed.", + "required": false, + "min": 0 + }, + { + "name": "noise_path", + "type": "path", + "description": "Optional raw f32 noise file for deterministic V1/V2 diffusion noise and XLSR/HiFT source excitation.", + "required": false + }, + { + "name": "f0_condition", + "type": "bool", + "description": "Enable V1 F0 conditioning for singing voice conversion; default false.", + "required": false, + "default": false + }, + { + "name": "auto_f0_adjust", + "type": "bool", + "description": "Automatically adjust V1 source pitch toward the target pitch level; default false.", + "required": false, + "default": false + }, + { + "name": "semitone_shift", + "type": "int", + "description": "V1 pitch shift in semitones for singing voice conversion; default 0.", + "required": false, + "default": 0 + } + ], + "session": [ + { + "name": "weight_type", + "type": "enum", + "description": "Shared Seed-VC component weight storage type; default native, except RMVPE uses f32 unless overridden.", + "preset": "weight_type_full", + "required": false, + "default": "native" + } + ], + "load": [] + }, "package_defaults": { "download": { "kind": "huggingface_snapshot", diff --git a/src/framework/audio/chunking.cpp b/src/framework/audio/chunking.cpp index 38005356..c2f16e44 100644 --- a/src/framework/audio/chunking.cpp +++ b/src/framework/audio/chunking.cpp @@ -185,7 +185,7 @@ std::optional parse_audio_chunk_seconds_override( const std::unordered_map & options) { return runtime::parse_float_option( options, - {"audio_chunk_seconds", "audio_chunk_duration_seconds", "audio_chunk_duration"}); + {"audio_chunk_duration_sec", "audio_chunk_seconds", "audio_chunk_duration_seconds", "audio_chunk_duration"}); } std::vector plan_vad_audio_chunks( diff --git a/src/framework/model_spec/package.cpp b/src/framework/model_spec/package.cpp index 1e550d79..1627f51d 100644 --- a/src/framework/model_spec/package.cpp +++ b/src/framework/model_spec/package.cpp @@ -7,11 +7,13 @@ #include #include +#include #include #include #include #include #include +#include #include namespace engine::model_spec { @@ -21,6 +23,7 @@ thread_local std::optional active_model_spec_override; thread_local std::optional active_model_path; thread_local bool active_embedded_spec_checked = false; thread_local std::optional active_embedded_spec; +thread_local std::unordered_set legacy_embedded_contract_warnings; const std::unordered_map & builtin_model_specs() { static const std::unordered_map specs = { @@ -116,6 +119,27 @@ const std::optional & embedded_model_spec() { return active_embedded_spec; } +bool embedded_model_spec_has_v1_contract( + const assets::GgufEmbeddedModelSpec & embedded, + std::string_view family) { + if (embedded.family != family) { + throw std::runtime_error("GGUF embeds model spec for family '" + embedded.family + "', not '" + + std::string(family) + "'"); + } + const auto root = engine::io::json::parse(embedded.json); + return root.find("schema_version") != nullptr; +} + +void warn_legacy_embedded_contract(std::string_view family) { + if (!legacy_embedded_contract_warnings.emplace(family).second) { + return; + } + std::cerr << "[warning][model_spec] GGUF for family '" << family + << "' embeds a legacy model spec. Using the current schema-v1 model contract " + "from the installed audio.cpp runtime for option validation. Regenerate the " + "GGUF to make the package fully standalone.\n"; +} + bool external_spec_matches_family(const std::filesystem::path & path, std::string_view family) { if (!engine::io::is_existing_file(path)) return false; @@ -465,22 +489,37 @@ std::filesystem::path default_contract_spec_path(std::string_view family) { } return std::filesystem::weakly_canonical(path); } + bool active_gguf_has_legacy_spec = false; + if (const auto gguf = active_gguf_path()) { + const auto & embedded = embedded_model_spec(); + if (!embedded.has_value()) { + throw std::runtime_error("GGUF for family '" + std::string(family) + + "' does not embed an audio.cpp model spec: " + gguf->string() + + ". Published GGUF packages must embed a model spec; regenerate the GGUF " + "or pass --model-spec-override."); + } + if (embedded_model_spec_has_v1_contract(*embedded, family)) { + return std::filesystem::path("@gguf") / (std::string(family) + ".json"); + } + active_gguf_has_legacy_spec = true; + warn_legacy_embedded_contract(family); + } if (const auto external = discover_workspace_model_spec(family)) { return *external; } if (builtin_model_specs().find(std::string(family)) != builtin_model_specs().end()) { return std::filesystem::path("@builtin") / (std::string(family) + ".json"); } - if (const auto & embedded = embedded_model_spec(); embedded.has_value()) { - if (embedded->family != family) { - throw std::runtime_error("GGUF embeds model spec for family '" + embedded->family + "', not '" + - std::string(family) + "'"); - } - return std::filesystem::path("@gguf") / (std::string(family) + ".json"); - } if (const auto hint = directory_gguf_hint(family); !hint.empty()) { throw std::runtime_error(hint); } + if (active_gguf_has_legacy_spec) { + throw std::runtime_error("GGUF for family '" + std::string(family) + + "' embeds a legacy model spec, but no current schema-v1 model contract was found. " + "Install model_specs/" + std::string(family) + + ".json, enable AUDIOCPP_DEPLOYMENT_BUILD, regenerate the GGUF, " + "or pass --model-spec-override."); + } throw std::runtime_error("model contract spec not found for family '" + std::string(family) + "' (provide --model-spec-override, install model_specs/" + std::string(family) + ".json, enable AUDIOCPP_DEPLOYMENT_BUILD, or embed it in the GGUF)"); diff --git a/src/framework/model_spec/schema.cpp b/src/framework/model_spec/schema.cpp index d1f58dbb..291da22b 100644 --- a/src/framework/model_spec/schema.cpp +++ b/src/framework/model_spec/schema.cpp @@ -158,7 +158,7 @@ const std::unordered_set & capabilities_for_task(const std::string {"vad", {"speech_segments", "chunk_planning"}}, {"diar", {"speaker_turns"}}, {"sep", {"stems"}}, - {"music", {"lyrics", "instrumental", "continuation"}}, + {"music", {"lyrics", "instrumental", "continuation", "style_control"}}, {"sfx", {"prompt_generation"}}, {"edit", {"prompt_editing", "inpaint"}}, {"design", {"voice_design"}}, diff --git a/src/framework/runtime/options.cpp b/src/framework/runtime/options.cpp index b866701a..8bba2494 100644 --- a/src/framework/runtime/options.cpp +++ b/src/framework/runtime/options.cpp @@ -208,6 +208,22 @@ std::optional parse_finite_float_option( return std::nullopt; } +std::optional parse_positive_finite_float_option( + const std::unordered_map & options, + std::initializer_list keys) { + if (const auto match = find_option_match(options, keys)) { + const float value = parse_float_value(match->value, match->key); + if (!std::isfinite(value)) { + throw std::runtime_error(match->key + " must be a finite float"); + } + if (value <= 0.0F) { + throw std::runtime_error(match->key + " must be positive"); + } + return value; + } + return std::nullopt; +} + std::optional parse_u32_option( const std::unordered_map & options, std::initializer_list keys) { @@ -282,6 +298,18 @@ assets::TensorStorageType parse_tensor_storage_option( throw std::runtime_error(match->key + " supports only " + join_tensor_storage_types(allowed)); } +assets::TensorStorageType parse_tensor_storage_option( + const std::unordered_map & options, + std::string_view key, + std::string_view fallback_key, + assets::TensorStorageType fallback, + std::initializer_list allowed) { + if (find_option_match(options, {key}).has_value()) { + return parse_tensor_storage_option(options, key, fallback, allowed); + } + return parse_tensor_storage_option(options, fallback_key, fallback, allowed); +} + uint64_t random_u64_seed() { std::random_device random_device; const uint64_t hi = static_cast(random_device()) << 32U; diff --git a/src/models/demucs/assets.cpp b/src/models/demucs/assets.cpp index 20cfe811..50e00ee3 100644 --- a/src/models/demucs/assets.cpp +++ b/src/models/demucs/assets.cpp @@ -140,11 +140,9 @@ void validate_demucs_weight_storage_type(assets::TensorStorageType storage_type) } } -std::shared_ptr load_htdemucs_assets(const runtime::ModelLoadRequest & request) { +std::shared_ptr load_htdemucs_assets(const std::filesystem::path & model_path) { auto out = std::make_shared(); - out->resources = engine::model_spec::load_resource_bundle( - request.model_path, - engine::model_spec::default_spec_path("htdemucs")); + out->resources = engine::model_spec::load_resource_bundle_for_family(model_path, "htdemucs"); out->manifest = parse_package_manifest(out->resources); out->submodels.push_back(load_submodel(out->resources)); return out; diff --git a/src/models/demucs/loader.cpp b/src/models/demucs/loader.cpp deleted file mode 100644 index 4076e334..00000000 --- a/src/models/demucs/loader.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include "engine/models/demucs/loader.h" - -#include "engine/framework/model_spec/package.h" -#include "engine/models/demucs/session.h" - -#include -#include - -namespace engine::models::demucs { -namespace { - -runtime::ModelCliInterface cli() { - runtime::ModelCliInterface out; - out.session_options = { - {"htdemucs.weight_type", "native|f32|f16|bf16|q8_0", "HTDemucs weight storage type."}, - }; - return out; -} - -runtime::ModelMetadata metadata(const HTDemucsAssets & assets) { - runtime::ModelMetadata out; - out.family = "htdemucs"; - out.variant = assets.manifest.name; - out.description = "HTDemucs source separation model converted from a Demucs reference checkpoint."; - return out; -} - -runtime::CapabilitySet capabilities(const HTDemucsAssets &) { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::SourceSeparation, {runtime::RunMode::Offline}}, - }; - out.languages = {"N/A"}; - return out; -} - -class HTDemucsLoader final : public runtime::IVoiceModelLoader { -public: - std::string family() const override { - return "htdemucs"; - } - - runtime::CapabilitySet advertised_capabilities() const override { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::SourceSeparation, {runtime::RunMode::Offline}}, - }; - return out; - } - - bool can_load(const runtime::ModelLoadRequest & request) const override { - try { - const auto package_spec = engine::model_spec::default_spec_path(family()); - (void) engine::model_spec::load_resource_bundle(request.model_path, package_spec); - return !request.family_hint.has_value() || *request.family_hint == family(); - } catch (...) { - return false; - } - } - - runtime::ModelInspection inspect(const runtime::ModelLoadRequest & request) const override { - const auto model_assets = load_htdemucs_assets(request); - const auto package_spec = engine::model_spec::default_spec_path(family()); - runtime::ModelInspection inspection; - inspection.model_root = model_assets->resources.model_root(); - inspection.metadata = metadata(*model_assets); - inspection.capabilities = capabilities(*model_assets); - inspection.cli = cli(); - inspection.discovered_configs = runtime::discover_named_assets_from_package_spec( - request.model_path, - package_spec, - engine::model_spec::ResourceKind::Files); - inspection.discovered_weights = runtime::discover_named_assets_from_package_spec( - request.model_path, - package_spec, - engine::model_spec::ResourceKind::Tensors); - return inspection; - } - - std::unique_ptr load(const runtime::ModelLoadRequest & request) const override { - return load_htdemucs_model(request); - } -}; - -} // namespace - -HTDemucsLoadedModel::HTDemucsLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets) - : metadata_(std::move(metadata)), - capabilities_(std::move(capabilities)), - assets_(std::move(assets)) { - if (assets_ == nullptr) { - throw std::runtime_error("HTDemucs loaded model requires assets"); - } -} - -const runtime::ModelMetadata & HTDemucsLoadedModel::metadata() const noexcept { - return metadata_; -} - -const runtime::CapabilitySet & HTDemucsLoadedModel::capabilities() const noexcept { - return capabilities_; -} - -std::unique_ptr HTDemucsLoadedModel::create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const { - return std::make_unique(task, options, assets_); -} - -std::unique_ptr load_htdemucs_model(const runtime::ModelLoadRequest & request) { - auto assets = load_htdemucs_assets(request); - return std::make_unique( - metadata(*assets), - capabilities(*assets), - std::move(assets)); -} - -std::shared_ptr make_htdemucs_loader() { - return std::make_shared(); -} - -} // namespace engine::models::demucs diff --git a/src/models/demucs/session.cpp b/src/models/demucs/session.cpp index 62acea22..70fbe644 100644 --- a/src/models/demucs/session.cpp +++ b/src/models/demucs/session.cpp @@ -4,6 +4,7 @@ #include "engine/framework/audio/conversion.h" #include "engine/framework/core/backend.h" #include "engine/framework/debug/profiler.h" +#include "engine/framework/runtime/spec_backed_model.h" #include #include @@ -16,6 +17,23 @@ namespace engine::models::demucs { namespace { +constexpr const char * kFamily = "htdemucs"; + +std::shared_ptr require_assets(std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("HTDemucs session requires assets"); + } + return assets; +} + +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("HTDemucs session requires a model contract"); + } + return contract; +} + assets::TensorStorageType option_weight_type( const runtime::SessionOptions & options, const std::string & key, @@ -70,15 +88,18 @@ std::pair normalize_separator_audio(runtime::AudioBuffer & audio) } // namespace HTDemucsSession::HTDemucsSession( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options, - std::shared_ptr assets) - : RuntimeSessionBase(options), - task_(task), - assets_(std::move(assets)) { - if (assets_ == nullptr) { - throw std::runtime_error("HTDemucs session requires assets"); - } + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(runtime::apply_option_v1_compatibility( + std::move(options), + {{"weight_type", "htdemucs.weight_type"}}, + "HTDemucs")), + task_(std::move(task)), + assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))) { + runtime::validate_spec_backed_session_options(RuntimeSessionBase::options(), *contract_, kFamily, "HTDemucs"); if (task_.task != runtime::VoiceTaskKind::SourceSeparation) { throw std::runtime_error("HTDemucs models only support --task sep"); } @@ -112,7 +133,7 @@ HTDemucsSession::HTDemucsSession( HTDemucsSession::~HTDemucsSession() = default; std::string HTDemucsSession::family() const { - return "htdemucs"; + return kFamily; } runtime::VoiceTaskKind HTDemucsSession::task_kind() const { @@ -146,6 +167,7 @@ void HTDemucsSession::prepare(const runtime::SessionPreparationRequest & request runtime::TaskResult HTDemucsSession::run(const runtime::TaskRequest & request) { require_prepared("HTDemucs run()"); + runtime::validate_spec_backed_request_options(request.options, *contract_, "HTDemucs"); if (!request.audio_input.has_value()) { throw std::runtime_error("HTDemucs run() requires audio_input"); } @@ -264,4 +286,23 @@ runtime::TaskResult HTDemucsSession::run(const runtime::TaskRequest & request) { return result; } +// Loading adapter: HTDemucs uses the schema-v1 spec-backed loader, so the +// loader wiring stays beside the session it constructs. +std::shared_ptr make_htdemucs_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_htdemucs_assets; + config.create_session = [](const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, + options, + std::move(assets), + std::move(contract)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + } // namespace engine::models::demucs diff --git a/src/models/heartmula/assets.cpp b/src/models/heartmula/assets.cpp index cd501a00..e6b05410 100644 --- a/src/models/heartmula/assets.cpp +++ b/src/models/heartmula/assets.cpp @@ -319,9 +319,7 @@ void validate_codec_weight_anchors(const HeartMuLaAssets & assets) { } // namespace std::shared_ptr load_heartmula_assets(const std::filesystem::path & model_path) { - auto resources = engine::model_spec::load_resource_bundle( - model_path, - engine::model_spec::default_spec_path("heartmula")); + auto resources = engine::model_spec::load_resource_bundle_for_family(model_path, "heartmula"); HeartMuLaAssets assets; assets.resources = std::move(resources); assets.mula_config = parse_mula_config(assets.resources); diff --git a/src/models/heartmula/loader.cpp b/src/models/heartmula/loader.cpp deleted file mode 100644 index 408c2873..00000000 --- a/src/models/heartmula/loader.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include "engine/models/heartmula/loader.h" - -#include "engine/framework/model_spec/package.h" -#include "engine/models/heartmula/session.h" - -#include -#include - -namespace engine::models::heartmula { -namespace { - -runtime::CapabilitySet capabilities(const HeartMuLaAssets &) { - runtime::CapabilitySet capabilities; - capabilities.supported_tasks = { - {runtime::VoiceTaskKind::AudioGeneration, {runtime::RunMode::Offline}}, - }; - capabilities.languages = {"Auto"}; - capabilities.supports_style_condition = true; - return capabilities; -} - -runtime::ModelMetadata metadata(const HeartMuLaAssets & assets) { - runtime::ModelMetadata metadata; - metadata.family = "heartmula"; - metadata.variant = assets.mula_config.backbone_flavor + "-heartcodec"; - metadata.description = "HeartMuLa text-to-music model loaded from local assets."; - return metadata; -} - -runtime::ModelCliInterface cli(const HeartMuLaAssets &) { - runtime::ModelCliInterface out; - out.request_options = { - {"lyrics", "text", "Lyrics text."}, - {"tags", "text", "Comma-separated music tags."}, - {"duration_seconds", "seconds", "Maximum generated audio duration."}, - {"temperature", "float", "Audio token sampling temperature."}, - {"top_k", "n", "Audio token top-k sampling limit."}, - {"guidance_scale", "float", "MuLa classifier-free guidance scale."}, - {"codec_duration", "seconds", "Codec detokenization chunk duration."}, - {"num_inference_steps", "n", "Codec flow solver steps."}, - {"codec_guidance_scale", "float", "Codec classifier-free guidance scale."}, - {"infinite_mode", "bool", "Generate long outputs by splitting lyrics into bounded HeartMuLa requests."}, - {"text_chunk_size", "n", "Text chunk size for infinite mode."}, - {"infinite_chunk_audio_length_ms", "n", "Per-chunk audio cap for infinite mode."}, - {"seed", "n", "Torch RNG seed."}, - }; - out.session_options = { - {"heartmula.weight_type", "native|f32|f16|bf16|q8_0", "MuLa and codec weight storage type."}, - {"heartmula.mula_weight_type", "native|f32|f16|bf16|q8_0", "MuLa weight storage type."}, - {"heartmula.codec_weight_type", "native|f32|f16|bf16|q8_0", "Codec weight storage type."}, - {"heartmula.mula_weight_context_mb", "n", "MuLa weight context size."}, - {"heartmula.codec_weight_context_mb", "n", "Codec weight context size."}, - {"heartmula.mula_constant_context_mb", "n", "MuLa reusable constant context size."}, - {"heartmula.mula_backbone_prefill_graph_arena_mb", "n", "MuLa backbone prefill graph arena size."}, - {"heartmula.mula_backbone_step_graph_arena_mb", "n", "MuLa backbone cached-step graph arena size."}, - {"heartmula.mula_decoder_prefill_graph_arena_mb", "n", "MuLa decoder prefill graph arena size."}, - {"heartmula.mula_decoder_step_graph_arena_mb", "n", "MuLa decoder cached-step graph arena size."}, - {"heartmula.mula_frame_embedding_graph_arena_mb", "n", "MuLa frame-embedding graph arena size."}, - {"heartmula.codec_flow_estimator_graph_arena_mb", "n", "Codec flow-estimator graph arena size."}, - {"heartmula.codec_conditioning_graph_arena_mb", "n", "Codec conditioning graph arena size."}, - {"heartmula.codec_scalar_decoder_graph_arena_mb", "n", "Codec scalar-decoder graph arena size."}, - {"heartmula.mem_saver", "true|false", "Release staged runtime graphs after each request; default false."}, - }; - return out; -} - -class HeartMuLaLoader final : public runtime::IVoiceModelLoader { -public: - std::string family() const override { - return "heartmula"; - } - - runtime::CapabilitySet advertised_capabilities() const override { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::AudioGeneration, {runtime::RunMode::Offline}}, - }; - out.supports_style_condition = true; - return out; - } - - bool can_load(const runtime::ModelLoadRequest & request) const override { - try { - (void) engine::model_spec::load_resource_bundle( - request.model_path, - engine::model_spec::default_spec_path(family())); - return !request.family_hint.has_value() || *request.family_hint == family(); - } catch (...) { - return false; - } - } - - runtime::ModelInspection inspect(const runtime::ModelLoadRequest & request) const override { - const auto assets = load_heartmula_assets(request.model_path); - runtime::ModelInspection inspection; - inspection.model_root = assets->resources.model_root(); - inspection.metadata = metadata(*assets); - inspection.capabilities = capabilities(*assets); - inspection.cli = cli(*assets); - const auto spec_path = engine::model_spec::default_spec_path(family()); - inspection.discovered_configs = runtime::discover_named_assets_from_package_spec( - request.model_path, - spec_path, - engine::model_spec::ResourceKind::Files); - inspection.discovered_weights = runtime::discover_named_assets_from_package_spec( - request.model_path, - spec_path, - engine::model_spec::ResourceKind::Tensors); - return inspection; - } - - std::unique_ptr load(const runtime::ModelLoadRequest & request) const override { - return load_heartmula_model(request.model_path); - } -}; - -} // namespace - -HeartMuLaLoadedModel::HeartMuLaLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets) - : metadata_(std::move(metadata)), - capabilities_(std::move(capabilities)), - assets_(std::move(assets)) {} - -const runtime::ModelMetadata & HeartMuLaLoadedModel::metadata() const noexcept { - return metadata_; -} - -const runtime::CapabilitySet & HeartMuLaLoadedModel::capabilities() const noexcept { - return capabilities_; -} - -std::unique_ptr HeartMuLaLoadedModel::create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const { - if (task.mode != runtime::RunMode::Offline) { - throw std::runtime_error("HeartMuLa only supports offline sessions"); - } - if (task.task != runtime::VoiceTaskKind::AudioGeneration) { - throw std::runtime_error("HeartMuLa only supports the gen task"); - } - return std::make_unique(task, options, assets_); -} - -std::unique_ptr load_heartmula_model(const std::filesystem::path & model_path) { - auto assets = load_heartmula_assets(model_path); - return std::make_unique( - metadata(*assets), - capabilities(*assets), - std::move(assets)); -} - -std::shared_ptr make_heartmula_loader() { - return std::make_shared(); -} - -} // namespace engine::models::heartmula diff --git a/src/models/heartmula/session.cpp b/src/models/heartmula/session.cpp index 2694307a..7dce600b 100644 --- a/src/models/heartmula/session.cpp +++ b/src/models/heartmula/session.cpp @@ -3,6 +3,7 @@ #include "engine/framework/debug/profiler.h" #include "engine/framework/debug/trace.h" #include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" #include "engine/framework/text/chunking.h" #include @@ -15,6 +16,8 @@ namespace { using Clock = std::chrono::steady_clock; +constexpr const char * kFamily = "heartmula"; + std::shared_ptr require_assets(std::shared_ptr assets) { if (assets == nullptr) { throw std::runtime_error("HeartMuLa session requires assets"); @@ -22,95 +25,70 @@ std::shared_ptr require_assets(std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("HeartMuLa session requires a model contract"); } - throw std::runtime_error(std::string(option_name) + " currently supports only native, f32, f16, bf16, and q8_0"); + return contract; +} + +runtime::SessionOptions normalize_session_options(runtime::SessionOptions options) { + return runtime::apply_option_v1_compatibility( + std::move(options), + { + {"heartmula.mula_weight_type", "heartmula.generator_weight_type"}, + {"heartmula.mula_weight_context_mb", "heartmula.generator_weight_context_mb"}, + {"heartmula.mula_constant_context_mb", "heartmula.generator_constant_context_mb"}, + {"heartmula.mula_backbone_prefill_graph_arena_mb", "heartmula.backbone_prefill_graph_arena_mb"}, + {"heartmula.mula_backbone_step_graph_arena_mb", "heartmula.backbone_step_graph_arena_mb"}, + {"heartmula.mula_decoder_prefill_graph_arena_mb", "heartmula.decoder_prefill_graph_arena_mb"}, + {"heartmula.mula_decoder_step_graph_arena_mb", "heartmula.decoder_step_graph_arena_mb"}, + {"heartmula.mula_frame_embedding_graph_arena_mb", "heartmula.frame_embedding_graph_arena_mb"}, + {"heartmula.codec_flow_estimator_graph_arena_mb", "heartmula.flow_estimator_graph_arena_mb"}, + {"heartmula.codec_conditioning_graph_arena_mb", "heartmula.conditioning_graph_arena_mb"}, + {"heartmula.codec_scalar_decoder_graph_arena_mb", "heartmula.scalar_decoder_graph_arena_mb"}, + }, + "HeartMuLa"); } -void validate_session_options(const runtime::SessionOptions & options) { +runtime::SessionOptions require_supported_session_options( + runtime::SessionOptions options, + const std::shared_ptr & contract) { + options = normalize_session_options(std::move(options)); + const auto checked_contract = require_contract(contract); + runtime::validate_spec_backed_session_options(options, *checked_contract, kFamily, "HeartMuLa"); if (options.backend.threads <= 0) { throw std::runtime_error("HeartMuLa requires positive backend thread count"); } - for (const auto & [key, value] : options.options) { - if (key == "heartmula.weight_type" || - key == "heartmula.mula_weight_type" || - key == "heartmula.codec_weight_type") { - validate_weight_storage(engine::assets::parse_tensor_storage_type(value), key.c_str()); - } else if (key == "heartmula.mula_weight_context_mb" || - key == "heartmula.codec_weight_context_mb" || - key == "heartmula.mula_constant_context_mb" || - key == "heartmula.mula_backbone_prefill_graph_arena_mb" || - key == "heartmula.mula_backbone_step_graph_arena_mb" || - key == "heartmula.mula_decoder_prefill_graph_arena_mb" || - key == "heartmula.mula_decoder_step_graph_arena_mb" || - key == "heartmula.mula_frame_embedding_graph_arena_mb" || - key == "heartmula.codec_flow_estimator_graph_arena_mb" || - key == "heartmula.codec_conditioning_graph_arena_mb" || - key == "heartmula.codec_scalar_decoder_graph_arena_mb") { - (void) value; - } else if (key == "heartmula.mem_saver") { - (void) runtime::parse_bool_option(value, key); - } else if (key.rfind("heartmula.", 0) == 0) { - throw std::runtime_error("unknown HeartMuLa session option: " + key); - } + if (const auto mem_saver = runtime::find_option(options.options, {"heartmula.mem_saver"})) { + (void) runtime::parse_bool_option(*mem_saver, "heartmula.mem_saver"); } -} - -runtime::SessionOptions require_supported_options(runtime::SessionOptions options) { - validate_session_options(options); return options; } HeartMuLaGenerationOptions generation_options_from_request(const runtime::TaskRequest & request) { HeartMuLaGenerationOptions options; - if (const auto value = runtime::parse_finite_float_option(request.options, {"duration_seconds"})) { - if (*value <= 0.0F) { - throw std::runtime_error("HeartMuLa duration_seconds must be positive"); - } + if (const auto value = runtime::parse_positive_finite_float_option(request.options, {"duration_sec"})) { options.duration_seconds = *value; } - if (const auto value = runtime::parse_finite_float_option(request.options, {"temperature"})) { - if (*value <= 0.0F) { - throw std::runtime_error("HeartMuLa temperature must be positive"); - } + if (const auto value = runtime::parse_positive_finite_float_option(request.options, {"temperature"})) { options.temperature = *value; } - if (const auto value = runtime::parse_i64_option(request.options, {"top_k"})) { - if (*value <= 0) { - throw std::runtime_error("HeartMuLa top_k must be positive"); - } - options.top_k = *value; - } - if (const auto value = runtime::parse_finite_float_option(request.options, {"guidance_scale"})) { - if (*value <= 0.0F) { - throw std::runtime_error("HeartMuLa guidance_scale must be positive"); - } + options.top_k = runtime::parse_positive_i64_option(request.options, {"top_k"}, options.top_k); + if (const auto value = runtime::parse_positive_finite_float_option(request.options, {"guidance_scale"})) { options.guidance_scale = *value; } - if (const auto value = runtime::parse_finite_float_option(request.options, {"codec_duration"})) { - if (*value <= 0.0F) { - throw std::runtime_error("HeartMuLa codec_duration must be positive"); - } + if (const auto value = runtime::parse_positive_finite_float_option(request.options, {"codec_duration_sec"})) { options.codec_duration = *value; } - if (const auto value = runtime::parse_i64_option(request.options, {"num_inference_steps"})) { - if (*value <= 0) { - throw std::runtime_error("HeartMuLa num_inference_steps must be positive"); - } - options.num_inference_steps = *value; - } - if (const auto value = runtime::parse_finite_float_option( + options.num_inference_steps = runtime::parse_positive_i64_option( + request.options, + {"num_inference_steps"}, + options.num_inference_steps); + if (const auto value = runtime::parse_positive_finite_float_option( request.options, {"codec_guidance_scale"})) { - if (*value <= 0.0F) { - throw std::runtime_error("HeartMuLa codec_guidance_scale must be positive"); - } options.codec_guidance_scale = *value; } if (const auto value = runtime::find_option(request.options, {"infinite_mode"})) { @@ -119,17 +97,38 @@ HeartMuLaGenerationOptions generation_options_from_request(const runtime::TaskRe if (const auto value = engine::text::parse_text_chunk_size_override(request.options)) { options.text_chunk_size = *value; } - if (const auto value = runtime::parse_i64_option( - request.options, - {"infinite_chunk_audio_length_ms"})) { - if (*value <= 0) { - throw std::runtime_error("HeartMuLa infinite_chunk_audio_length_ms must be positive"); - } - options.infinite_chunk_audio_length_ms = *value; - } + options.infinite_chunk_audio_length_ms = runtime::parse_positive_i64_option( + request.options, + {"infinite_chunk_audio_duration_ms"}, + options.infinite_chunk_audio_length_ms); return options; } +runtime::TaskRequest normalize_request_options(runtime::TaskRequest request) { + request.options = runtime::apply_option_v1_compatibility( + std::move(request.options), + { + {"duration_seconds", "duration_sec"}, + {"codec_duration", "codec_duration_sec"}, + {"infinite_chunk_audio_length_ms", "infinite_chunk_audio_duration_ms"}, + }, + "HeartMuLa", + "request"); + return request; +} + +std::unique_ptr create_heartmula_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, + options, + std::move(assets), + std::move(contract)); +} + std::string request_tags(const runtime::TaskRequest & request) { if (const auto tags = runtime::find_option(request.options, {"tags"})) { return *tags; @@ -173,73 +172,89 @@ int64_t audio_duration_ms(const runtime::AudioBuffer & audio) { HeartMuLaSession::HeartMuLaSession( runtime::TaskSpec task, runtime::SessionOptions options, - std::shared_ptr assets) - : runtime::RuntimeSessionBase(require_supported_options(options)), + std::shared_ptr assets, + std::shared_ptr contract) + : runtime::RuntimeSessionBase(require_supported_session_options(std::move(options), contract)), task_(task), assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))), text_tokenizer_(assets_), mula_( assets_, - options.backend.type, - options.backend.device, - options.backend.threads, + RuntimeSessionBase::options().backend.type, + RuntimeSessionBase::options().backend.device, + RuntimeSessionBase::options().backend.threads, runtime::parse_size_mb_option( - options.options, - {"heartmula.mula_weight_context_mb"}, - mula_weight_context_bytes_), + RuntimeSessionBase::options().options, + {"heartmula.generator_weight_context_mb"}, + generator_weight_context_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.mula_constant_context_mb"}, - mula_constant_context_bytes_), + RuntimeSessionBase::options().options, + {"heartmula.generator_constant_context_mb"}, + generator_constant_context_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.mula_backbone_prefill_graph_arena_mb"}, - mula_backbone_prefill_graph_arena_bytes_), + RuntimeSessionBase::options().options, + {"heartmula.backbone_prefill_graph_arena_mb"}, + generator_backbone_prefill_graph_arena_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.mula_backbone_step_graph_arena_mb"}, - mula_backbone_step_graph_arena_bytes_), + RuntimeSessionBase::options().options, + {"heartmula.backbone_step_graph_arena_mb"}, + generator_backbone_step_graph_arena_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.mula_decoder_prefill_graph_arena_mb"}, - mula_decoder_prefill_graph_arena_bytes_), + RuntimeSessionBase::options().options, + {"heartmula.decoder_prefill_graph_arena_mb"}, + generator_decoder_prefill_graph_arena_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.mula_decoder_step_graph_arena_mb"}, - mula_decoder_step_graph_arena_bytes_), + RuntimeSessionBase::options().options, + {"heartmula.decoder_step_graph_arena_mb"}, + generator_decoder_step_graph_arena_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.mula_frame_embedding_graph_arena_mb"}, - mula_frame_embedding_graph_arena_bytes_), - options.options.find("heartmula.mula_weight_type") != options.options.end() - ? engine::assets::parse_tensor_storage_type(options.options.at("heartmula.mula_weight_type")) - : (options.options.find("heartmula.weight_type") != options.options.end() - ? engine::assets::parse_tensor_storage_type(options.options.at("heartmula.weight_type")) - : mula_weight_storage_type_)), + RuntimeSessionBase::options().options, + {"heartmula.frame_embedding_graph_arena_mb"}, + generator_frame_embedding_graph_arena_bytes_), + runtime::parse_tensor_storage_option( + RuntimeSessionBase::options().options, + "heartmula.generator_weight_type", + "heartmula.weight_type", + mula_weight_storage_type_, + { + engine::assets::TensorStorageType::Native, + engine::assets::TensorStorageType::F32, + engine::assets::TensorStorageType::F16, + engine::assets::TensorStorageType::BF16, + engine::assets::TensorStorageType::Q8_0, + })), codec_( assets_, execution_context(), runtime::parse_size_mb_option( - options.options, + RuntimeSessionBase::options().options, {"heartmula.codec_weight_context_mb"}, codec_weight_context_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.codec_flow_estimator_graph_arena_mb"}, + RuntimeSessionBase::options().options, + {"heartmula.flow_estimator_graph_arena_mb"}, codec_flow_estimator_graph_arena_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.codec_conditioning_graph_arena_mb"}, + RuntimeSessionBase::options().options, + {"heartmula.conditioning_graph_arena_mb"}, codec_conditioning_graph_arena_bytes_), runtime::parse_size_mb_option( - options.options, - {"heartmula.codec_scalar_decoder_graph_arena_mb"}, + RuntimeSessionBase::options().options, + {"heartmula.scalar_decoder_graph_arena_mb"}, codec_scalar_decoder_graph_arena_bytes_), - options.options.find("heartmula.codec_weight_type") != options.options.end() - ? engine::assets::parse_tensor_storage_type(options.options.at("heartmula.codec_weight_type")) - : (options.options.find("heartmula.weight_type") != options.options.end() - ? engine::assets::parse_tensor_storage_type(options.options.at("heartmula.weight_type")) - : codec_weight_storage_type_)) { + runtime::parse_tensor_storage_option( + RuntimeSessionBase::options().options, + "heartmula.codec_weight_type", + "heartmula.weight_type", + codec_weight_storage_type_, + { + engine::assets::TensorStorageType::Native, + engine::assets::TensorStorageType::F32, + engine::assets::TensorStorageType::F16, + engine::assets::TensorStorageType::BF16, + engine::assets::TensorStorageType::Q8_0, + })) { if (const auto mem_saver = runtime::find_option(RuntimeSessionBase::options().options, {"heartmula.mem_saver"})) { mem_saver_ = runtime::parse_bool_option(*mem_saver, "heartmula.mem_saver"); } @@ -252,7 +267,7 @@ HeartMuLaSession::HeartMuLaSession( } std::string HeartMuLaSession::family() const { - return "heartmula"; + return kFamily; } runtime::VoiceTaskKind HeartMuLaSession::task_kind() const { @@ -270,9 +285,11 @@ void HeartMuLaSession::prepare(const runtime::SessionPreparationRequest & reques runtime::TaskResult HeartMuLaSession::run(const runtime::TaskRequest & request) { require_prepared("HeartMuLa run"); + auto normalized_request = normalize_request_options(request); + runtime::validate_spec_backed_request_options(normalized_request.options, *contract_, "HeartMuLa"); const auto wall_start = Clock::now(); - const auto heartmula_request = make_request(request); - const uint32_t seed = request_seed(request); + const auto heartmula_request = make_request(normalized_request); + const uint32_t seed = request_seed(normalized_request); if (heartmula_request.options.infinite_mode) { const auto chunks = engine::text::split_text_chunks( heartmula_request.lyrics, @@ -427,4 +444,12 @@ HeartMuLaPromptRequest HeartMuLaSession::make_request(const runtime::TaskRequest return out; } +std::shared_ptr make_heartmula_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_heartmula_assets; + config.create_session = create_heartmula_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + } // namespace engine::models::heartmula diff --git a/src/models/hviske_asr/assets.cpp b/src/models/hviske_asr/assets.cpp index f5f250b5..31c8c65f 100644 --- a/src/models/hviske_asr/assets.cpp +++ b/src/models/hviske_asr/assets.cpp @@ -78,9 +78,7 @@ void validate_config(const HviskeConfig & config) { std::shared_ptr load_hviske_asr_assets(const std::filesystem::path & model_path) { auto assets = std::make_shared(); - assets->resources = engine::model_spec::load_resource_bundle( - model_path, - engine::model_spec::default_spec_path("hviske_asr")); + assets->resources = engine::model_spec::load_resource_bundle_for_family(model_path, "hviske_asr"); assets->config = parse_config(assets->resources); validate_config(assets->config); assets->tokenizer_pieces = engine::tokenizers::load_sentencepiece_model( diff --git a/src/models/hviske_asr/loader.cpp b/src/models/hviske_asr/loader.cpp deleted file mode 100644 index 4fd0bf38..00000000 --- a/src/models/hviske_asr/loader.cpp +++ /dev/null @@ -1,146 +0,0 @@ -#include "engine/models/hviske_asr/loader.h" - -#include "engine/framework/model_spec/package.h" -#include "engine/models/hviske_asr/session.h" - -#include -#include - -namespace engine::models::hviske_asr { -namespace { - -runtime::ModelMetadata metadata(const HviskeASRAssets & assets) { - runtime::ModelMetadata out; - out.family = "hviske_asr"; - out.variant = assets.config.variant.empty() ? assets.config.model_type : assets.config.variant; - out.description = "Hviske/Cohere ASR loaded from local tensor assets."; - return out; -} - -runtime::CapabilitySet capabilities(const HviskeASRAssets & assets) { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::Asr, {runtime::RunMode::Offline}}, - }; - out.languages = assets.config.supported_languages; - out.supports_timestamps = false; - return out; -} - -runtime::ModelCliInterface cli(const HviskeASRAssets &) { - runtime::ModelCliInterface out; - out.request_options = { - {"language", "code", "ASR language code; defaults to da."}, - {"punctuation", "bool", "Enable or disable punctuation tokens in the decoder prompt."}, - {"max_tokens", "n", "Maximum generated transcript tokens."}, - {"num_beams", "n", "Beam-search beam count; 1 uses greedy or sampling decode."}, - {"length_penalty", "float", "Beam-search length penalty."}, - {"do_sample", "bool", "Enable sampling instead of greedy decode when num_beams is 1."}, - {"temperature", "float", "Sampling temperature."}, - {"top_k", "n", "Top-k sampling limit; 0 disables top-k."}, - {"top_p", "float", "Nucleus sampling limit."}, - {"seed", "n", "Sampling seed."}, - }; - out.session_options = { - {"hviske_asr.weight_type", "native|f32|f16|bf16|q8_0", "Matmul weight storage type."}, - {"hviske_asr.conv_weight_type", "native|f32|f16", "Convolution weight storage type."}, - {"hviske_asr.weight_context_mb", "mb", "Weight context arena size."}, - {"hviske_asr.encoder_graph_arena_mb", "mb", "Encoder graph arena size."}, - {"hviske_asr.decoder_prefill_graph_arena_mb", "mb", "Decoder prefill graph arena size."}, - {"hviske_asr.decoder_decode_graph_arena_mb", "mb", "Decoder cached-step graph arena size."}, - }; - return out; -} - -class HviskeASRLoader final : public runtime::IVoiceModelLoader { -public: - std::string family() const override { - return "hviske_asr"; - } - - runtime::CapabilitySet advertised_capabilities() const override { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::Asr, {runtime::RunMode::Offline}}, - }; - out.supports_timestamps = true; - return out; - } - - bool can_load(const runtime::ModelLoadRequest & request) const override { - try { - const auto package_spec = engine::model_spec::default_spec_path(family()); - (void) engine::model_spec::load_resource_bundle(request.model_path, package_spec); - return !request.family_hint.has_value() || *request.family_hint == family(); - } catch (...) { - return false; - } - } - - runtime::ModelInspection inspect(const runtime::ModelLoadRequest & request) const override { - const auto assets = load_hviske_asr_assets(request.model_path); - runtime::ModelInspection inspection; - inspection.model_root = assets->resources.model_root(); - inspection.metadata = metadata(*assets); - inspection.capabilities = capabilities(*assets); - const auto package_spec = engine::model_spec::default_spec_path(family()); - inspection.discovered_configs = runtime::discover_named_assets_from_package_spec( - request.model_path, - package_spec, - engine::model_spec::ResourceKind::Files); - inspection.discovered_weights = runtime::discover_named_assets_from_package_spec( - request.model_path, - package_spec, - engine::model_spec::ResourceKind::Tensors); - inspection.cli = cli(*assets); - return inspection; - } - - std::unique_ptr load(const runtime::ModelLoadRequest & request) const override { - return load_hviske_asr_model(request.model_path); - } -}; - -} // namespace - -HviskeASRLoadedModel::HviskeASRLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets) - : metadata_(std::move(metadata)), - capabilities_(std::move(capabilities)), - assets_(std::move(assets)) {} - -const runtime::ModelMetadata & HviskeASRLoadedModel::metadata() const noexcept { - return metadata_; -} - -const runtime::CapabilitySet & HviskeASRLoadedModel::capabilities() const noexcept { - return capabilities_; -} - -std::unique_ptr HviskeASRLoadedModel::create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const { - if (task.task != runtime::VoiceTaskKind::Asr) { - throw std::runtime_error("Hviske ASR only supports the Asr task"); - } - if (task.mode != runtime::RunMode::Offline) { - throw std::runtime_error("Hviske ASR currently supports offline sessions"); - } - return std::make_unique(task, options, assets_); -} - -std::unique_ptr load_hviske_asr_model(const std::filesystem::path & model_path) { - auto assets = load_hviske_asr_assets(model_path); - return std::make_unique( - metadata(*assets), - capabilities(*assets), - std::move(assets)); -} - -std::shared_ptr make_hviske_asr_loader() { - return std::make_shared(); -} - -} // namespace engine::models::hviske_asr diff --git a/src/models/hviske_asr/session.cpp b/src/models/hviske_asr/session.cpp index e0412eba..3cc1f0fd 100644 --- a/src/models/hviske_asr/session.cpp +++ b/src/models/hviske_asr/session.cpp @@ -5,6 +5,7 @@ #include "engine/framework/debug/profiler.h" #include "engine/framework/io/text.h" #include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" #include #include @@ -18,11 +19,13 @@ namespace { using Clock = std::chrono::steady_clock; -constexpr size_t kDefaultWeightContextBytes = 512ull * 1024ull * 1024ull; +constexpr size_t kDefaultWeightContextBytes = 32ull * 1024ull * 1024ull; constexpr size_t kDefaultEncoderGraphArenaBytes = 512ull * 1024ull * 1024ull; constexpr size_t kDefaultDecoderPrefillGraphArenaBytes = 512ull * 1024ull * 1024ull; constexpr size_t kDefaultDecoderDecodeGraphArenaBytes = 512ull * 1024ull * 1024ull; +constexpr const char * kFamily = "hviske_asr"; + std::shared_ptr require_assets(std::shared_ptr assets) { if (assets == nullptr) { throw std::runtime_error("Hviske ASR session requires assets"); @@ -30,35 +33,33 @@ std::shared_ptr require_assets(std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("Hviske ASR session requires a model contract"); } - return engine::assets::parse_tensor_storage_type(it->second); + return contract; } -void validate_matmul_weight_storage(engine::assets::TensorStorageType storage_type, const char * option_name) { - if (storage_type == engine::assets::TensorStorageType::Native || - storage_type == engine::assets::TensorStorageType::F32 || - storage_type == engine::assets::TensorStorageType::F16 || - storage_type == engine::assets::TensorStorageType::BF16 || - storage_type == engine::assets::TensorStorageType::Q8_0) { - return; - } - throw std::runtime_error(std::string(option_name) + " supports only native, f32, f16, bf16, and q8_0"); +runtime::SessionOptions require_supported_session_options( + runtime::SessionOptions options, + const std::shared_ptr & contract) { + const auto checked_contract = require_contract(contract); + runtime::validate_spec_backed_session_options(options, *checked_contract, kFamily, "Hviske ASR"); + return options; } -void validate_conv_weight_storage(engine::assets::TensorStorageType storage_type, const char * option_name) { - if (storage_type == engine::assets::TensorStorageType::Native || - storage_type == engine::assets::TensorStorageType::F32 || - storage_type == engine::assets::TensorStorageType::F16) { - return; - } - throw std::runtime_error(std::string(option_name) + " supports only native, f32, and f16"); +std::unordered_map normalize_request_options( + std::unordered_map options) { + return runtime::apply_option_v1_compatibility( + std::move(options), + { + {"audio_chunk_seconds", "audio_chunk_duration_sec"}, + {"audio_chunk_duration_seconds", "audio_chunk_duration_sec"}, + {"audio_chunk_duration", "audio_chunk_duration_sec"}, + }, + "Hviske ASR", + "request"); } void ensure_supported_language(const HviskeASRAssets & assets, const std::string & language) { @@ -114,21 +115,54 @@ int64_t prepared_frontend_frames(const runtime::AudioPreparationContract & audio return std::max(request_frames, max_clip_frames); } +std::unique_ptr create_hviske_asr_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, + options, + std::move(assets), + std::move(contract)); +} + } // namespace HviskeASRSession::HviskeASRSession( runtime::TaskSpec task, runtime::SessionOptions options, - std::shared_ptr assets) - : RuntimeSessionBase(options), + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(require_supported_session_options(std::move(options), contract)), task_(task), assets_(require_assets(std::move(assets))), - weight_context_bytes_(runtime::parse_size_mb_option(options.options, {"hviske_asr.weight_context_mb"}, kDefaultWeightContextBytes)), - encoder_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"hviske_asr.encoder_graph_arena_mb"}, kDefaultEncoderGraphArenaBytes)), - decoder_prefill_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"hviske_asr.decoder_prefill_graph_arena_mb"}, kDefaultDecoderPrefillGraphArenaBytes)), - decoder_decode_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"hviske_asr.decoder_decode_graph_arena_mb"}, kDefaultDecoderDecodeGraphArenaBytes)), - matmul_weight_storage_type_(option_weight_type(options, "hviske_asr.weight_type", engine::assets::TensorStorageType::Native)), - conv_weight_storage_type_(option_weight_type(options, "hviske_asr.conv_weight_type", matmul_weight_storage_type_)), + contract_(require_contract(std::move(contract))), + weight_context_bytes_(runtime::parse_size_mb_option(RuntimeSessionBase::options().options, {"hviske_asr.weight_context_mb"}, kDefaultWeightContextBytes)), + encoder_graph_arena_bytes_(runtime::parse_size_mb_option(RuntimeSessionBase::options().options, {"hviske_asr.encoder_graph_arena_mb"}, kDefaultEncoderGraphArenaBytes)), + decoder_prefill_graph_arena_bytes_(runtime::parse_size_mb_option(RuntimeSessionBase::options().options, {"hviske_asr.decoder_prefill_graph_arena_mb"}, kDefaultDecoderPrefillGraphArenaBytes)), + decoder_decode_graph_arena_bytes_(runtime::parse_size_mb_option(RuntimeSessionBase::options().options, {"hviske_asr.decoder_decode_graph_arena_mb"}, kDefaultDecoderDecodeGraphArenaBytes)), + matmul_weight_storage_type_(runtime::parse_tensor_storage_option( + RuntimeSessionBase::options().options, + "hviske_asr.weight_type", + engine::assets::TensorStorageType::Native, + { + engine::assets::TensorStorageType::Native, + engine::assets::TensorStorageType::F32, + engine::assets::TensorStorageType::F16, + engine::assets::TensorStorageType::BF16, + engine::assets::TensorStorageType::Q8_0, + })), + conv_weight_storage_type_(runtime::parse_tensor_storage_option( + RuntimeSessionBase::options().options, + "hviske_asr.conv_weight_type", + "hviske_asr.weight_type", + matmul_weight_storage_type_, + { + engine::assets::TensorStorageType::Native, + engine::assets::TensorStorageType::F32, + engine::assets::TensorStorageType::F16, + })), frontend_(assets_) { if (task_.task != runtime::VoiceTaskKind::Asr) { throw std::runtime_error("Hviske ASR only supports VoiceTaskKind::Asr"); @@ -136,20 +170,6 @@ HviskeASRSession::HviskeASRSession( if (task_.mode != runtime::RunMode::Offline) { throw std::runtime_error("Hviske ASR currently supports offline sessions"); } - validate_matmul_weight_storage(matmul_weight_storage_type_, "hviske_asr.weight_type"); - validate_conv_weight_storage(conv_weight_storage_type_, "hviske_asr.conv_weight_type"); - for (const auto & [key, value] : options.options) { - (void)value; - if (key.rfind("hviske_asr.", 0) == 0 && - key != "hviske_asr.weight_context_mb" && - key != "hviske_asr.encoder_graph_arena_mb" && - key != "hviske_asr.decoder_prefill_graph_arena_mb" && - key != "hviske_asr.decoder_decode_graph_arena_mb" && - key != "hviske_asr.weight_type" && - key != "hviske_asr.conv_weight_type") { - throw std::runtime_error("unknown Hviske ASR session option: " + key); - } - } weights_ = load_hviske_weights( *assets_, execution_context().backend(), @@ -173,7 +193,7 @@ HviskeASRSession::HviskeASRSession( HviskeASRSession::~HviskeASRSession() = default; std::string HviskeASRSession::family() const { - return "hviske_asr"; + return kFamily; } runtime::VoiceTaskKind HviskeASRSession::task_kind() const { @@ -199,66 +219,69 @@ void HviskeASRSession::prepare(const runtime::SessionPreparationRequest & reques debug::trace_log_scalar("hviske_asr.prepare.max_input_samples", request.audio->max_input_samples); } -std::string HviskeASRSession::language_for_request(const runtime::TaskRequest & request) const { +std::string HviskeASRSession::language_for_request( + const runtime::TaskRequest & request, + const std::unordered_map & options) const { if (request.text_input.has_value() && !request.text_input->language.empty()) { return request.text_input->language; } - if (const auto language = runtime::find_option(request.options, {"language"})) { + if (const auto language = runtime::find_option(options, {"language"})) { return *language; } return "da"; } -bool HviskeASRSession::punctuation_for_request(const runtime::TaskRequest & request) const { - if (const auto value = runtime::find_option(request.options, {"punctuation"})) { +bool HviskeASRSession::punctuation_for_request(const std::unordered_map & options) const { + if (const auto value = runtime::find_option(options, {"punctuation"})) { return runtime::parse_bool_option(*value, "punctuation"); } return true; } -HviskeDecodingOptions HviskeASRSession::decoding_options_for_request(const runtime::TaskRequest & request) const { +HviskeDecodingOptions HviskeASRSession::decoding_options_for_request( + const std::unordered_map & request_options) const { HviskeDecodingOptions options; options.max_new_tokens = assets_->config.decoder.max_new_tokens; - if (const auto value = runtime::parse_i64_option(request.options, {"max_tokens"})) { + if (const auto value = runtime::parse_i64_option(request_options, {"max_tokens"})) { if (*value <= 0) { throw std::runtime_error("Hviske ASR max_tokens must be positive"); } options.max_new_tokens = *value; } - if (const auto value = runtime::parse_i64_option(request.options, {"num_beams"})) { + if (const auto value = runtime::parse_i64_option(request_options, {"num_beams"})) { if (*value <= 0) { throw std::runtime_error("Hviske ASR num_beams must be positive"); } options.num_beams = *value; } - if (const auto value = runtime::parse_finite_float_option(request.options, {"length_penalty"})) { + if (const auto value = runtime::parse_finite_float_option(request_options, {"length_penalty"})) { if (*value <= 0.0f) { throw std::runtime_error("Hviske ASR length_penalty must be positive"); } options.length_penalty = *value; } - if (const auto value = runtime::find_option(request.options, {"do_sample"})) { + if (const auto value = runtime::find_option(request_options, {"do_sample"})) { options.do_sample = runtime::parse_bool_option(*value, "do_sample"); } - if (const auto value = runtime::parse_finite_float_option(request.options, {"temperature"})) { + if (const auto value = runtime::parse_finite_float_option(request_options, {"temperature"})) { if (*value <= 0.0f) { throw std::runtime_error("Hviske ASR temperature must be positive"); } options.temperature = *value; } - if (const auto value = runtime::parse_i64_option(request.options, {"top_k"})) { + if (const auto value = runtime::parse_i64_option(request_options, {"top_k"})) { if (*value < 0) { throw std::runtime_error("Hviske ASR top_k must be non-negative"); } options.top_k = *value; } - if (const auto value = runtime::parse_finite_float_option(request.options, {"top_p"})) { + if (const auto value = runtime::parse_finite_float_option(request_options, {"top_p"})) { if (*value <= 0.0f || *value > 1.0f) { throw std::runtime_error("Hviske ASR top_p must be within (0, 1]"); } options.top_p = *value; } - if (const auto value = runtime::parse_u32_option(request.options, {"seed"})) { + if (const auto value = runtime::parse_u32_option(request_options, {"seed"})) { options.seed = *value; } return options; @@ -295,7 +318,7 @@ std::vector HviskeASRSession::prepare_segments( engine::audio::parse_audio_chunk_seconds_override(options).value_or( static_cast(assets_->config.max_audio_clip_seconds))); if (!(chunk_seconds > 0.0)) { - throw std::runtime_error("Hviske ASR audio_chunk_seconds must be positive"); + throw std::runtime_error("Hviske ASR audio_chunk_duration_sec must be positive"); } const int64_t chunk_size = std::max( 1, @@ -344,15 +367,17 @@ std::vector HviskeASRSession::prepare_segments( runtime::TaskResult HviskeASRSession::run(const runtime::TaskRequest & request) { require_prepared("Hviske ASR run()"); + auto request_options = normalize_request_options(request.options); + runtime::validate_spec_backed_request_options(request_options, *contract_, "Hviske ASR"); if (!request.audio_input.has_value()) { throw std::runtime_error("Hviske ASR run() requires audio_input"); } const auto wall_start = Clock::now(); const auto config_start = Clock::now(); - const std::string language = language_for_request(request); + const std::string language = language_for_request(request, request_options); ensure_supported_language(*assets_, language); - const bool punctuation = punctuation_for_request(request); - const auto decoding_options = decoding_options_for_request(request); + const bool punctuation = punctuation_for_request(request_options); + const auto decoding_options = decoding_options_for_request(request_options); debug::timing_log_scalar("hviske_asr.request_config_ms", engine::debug::elapsed_ms(config_start, Clock::now())); const auto prompt_start = Clock::now(); @@ -360,7 +385,7 @@ runtime::TaskResult HviskeASRSession::run(const runtime::TaskRequest & request) debug::timing_log_scalar("hviske_asr.prompt_tokenize_ms", engine::debug::elapsed_ms(prompt_start, Clock::now())); const auto segments_start = Clock::now(); - const auto segments = prepare_segments(*request.audio_input, request.options); + const auto segments = prepare_segments(*request.audio_input, request_options); debug::timing_log_scalar("hviske_asr.prepare_segments_ms", engine::debug::elapsed_ms(segments_start, Clock::now())); std::vector texts; @@ -404,4 +429,12 @@ runtime::TaskResult HviskeASRSession::run(const runtime::TaskRequest & request) return result; } +std::shared_ptr make_hviske_asr_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_hviske_asr_assets; + config.create_session = create_hviske_asr_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + } // namespace engine::models::hviske_asr diff --git a/src/models/roformer/loader.cpp b/src/models/roformer/loader.cpp deleted file mode 100644 index 06dfac35..00000000 --- a/src/models/roformer/loader.cpp +++ /dev/null @@ -1,160 +0,0 @@ -#include "engine/models/roformer/loader.h" - -#include "engine/framework/model_spec/package.h" -#include "engine/models/roformer/session.h" - -#include -#include - -namespace engine::models::roformer { -namespace { - -runtime::ModelMetadata metadata(const RoformerAssets & assets) { - runtime::ModelMetadata out; - out.family = assets.config.family; - out.variant = assets.resources.model_root().filename().string(); - out.description = assets.config.family == kBsRoformerFamily - ? "Band-Split RoFormer music source separation model." - : "Mel-band RoFormer music source separation model."; - return out; -} - -runtime::CapabilitySet capabilities(const RoformerAssets &) { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::SourceSeparation, {runtime::RunMode::Offline}}, - }; - return out; -} - -runtime::ModelCliInterface cli(const RoformerAssets & assets) { - runtime::ModelCliInterface out; - out.session_options = { - { - assets.config.family + ".weight_type", - "native|f32|f16|bf16|q8_0", - "RoFormer weight storage type.", - }, - }; - if (assets.config.family == kBsRoformerFamily) { - out.session_options.push_back({ - assets.config.family + ".num_overlap", - "n", - "Number of overlapping inference windows; defaults to the package " - "configuration. Lower values improve throughput but can reduce " - "boundary quality.", - }); - } - return out; -} - -runtime::ModelInspection inspect_model( - const runtime::ModelLoadRequest & request, - std::string_view family) { - const auto assets = load_roformer_assets(request, family); - const auto package_spec = - engine::model_spec::default_spec_path(std::string(family)); - runtime::ModelInspection inspection; - inspection.model_root = assets->resources.model_root(); - inspection.metadata = metadata(*assets); - inspection.capabilities = capabilities(*assets); - inspection.cli = cli(*assets); - inspection.discovered_configs = runtime::discover_named_assets_from_package_spec( - request.model_path, - package_spec, - engine::model_spec::ResourceKind::Files); - inspection.discovered_weights = runtime::discover_named_assets_from_package_spec( - request.model_path, - package_spec, - engine::model_spec::ResourceKind::Tensors); - return inspection; -} - -class RoformerLoader final : public runtime::IVoiceModelLoader { -public: - explicit RoformerLoader(std::string family) - : family_(std::move(family)) {} - - std::string family() const override { - return family_; - } - - runtime::CapabilitySet advertised_capabilities() const override { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::SourceSeparation, {runtime::RunMode::Offline}}, - }; - return out; - } - - bool can_load(const runtime::ModelLoadRequest & request) const override { - if (request.family_hint.has_value() && *request.family_hint != family()) { - return false; - } - try { - (void) load_roformer_assets(request, family_); - return true; - } catch (...) { - if (request.family_hint.has_value() && *request.family_hint == family()) { - throw; - } - return false; - } - } - - runtime::ModelInspection inspect(const runtime::ModelLoadRequest & request) const override { - return inspect_model(request, family_); - } - - std::unique_ptr load(const runtime::ModelLoadRequest & request) const override { - return load_roformer_model(request, family_); - } - -private: - std::string family_; -}; - -} // namespace - -RoformerLoadedModel::RoformerLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets) - : metadata_(std::move(metadata)), - capabilities_(std::move(capabilities)), - assets_(std::move(assets)) { - if (assets_ == nullptr) { - throw std::runtime_error("RoFormer loaded model requires assets"); - } -} - -const runtime::ModelMetadata & RoformerLoadedModel::metadata() const noexcept { - return metadata_; -} - -const runtime::CapabilitySet & RoformerLoadedModel::capabilities() const noexcept { - return capabilities_; -} - -std::unique_ptr RoformerLoadedModel::create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const { - return std::make_unique(task, options, assets_); -} - -std::unique_ptr load_roformer_model( - const runtime::ModelLoadRequest & request, - std::string_view family) { - auto assets = load_roformer_assets(request, family); - return std::make_unique( - metadata(*assets), - capabilities(*assets), - std::move(assets)); -} - -std::shared_ptr make_mel_band_roformer_loader() { - return std::make_shared( - std::string(kMelBandRoformerFamily)); -} - -} // namespace engine::models::roformer diff --git a/src/models/roformer/session.cpp b/src/models/roformer/session.cpp index cd06f30c..6323bdcc 100644 --- a/src/models/roformer/session.cpp +++ b/src/models/roformer/session.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -50,56 +51,80 @@ runtime::AudioBuffer derive_instrumental( return out; } -void validate_session_option_keys( - const runtime::SessionOptions & options, - const engine::model_spec::ModelContract & contract, - std::string_view family) { - const std::string family_prefix = std::string(family) + "."; - for (const auto & [key, _] : options.options) { - if (key.rfind(family_prefix, 0) == 0 && - contract.session_option_keys.find(key) == - contract.session_option_keys.end()) { - throw std::runtime_error( - "unknown " + std::string(family) + - " session option: " + key); - } +std::shared_ptr require_assets(std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("RoFormer session requires assets"); } + return assets; } -} // namespace +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("RoFormer session requires a model contract"); + } + return contract; +} -RoformerSession::RoformerSession( +runtime::SessionOptions apply_option_v1_compatibility( + runtime::SessionOptions options, + const std::shared_ptr & assets) { + if (assets == nullptr) { + throw std::runtime_error("RoFormer session requires assets"); + } + const auto & family = assets->config.family; + return runtime::apply_option_v1_compatibility( + std::move(options), + { + {"weight_type", family + ".weight_type"}, + {"num_overlap", family + ".num_overlap"}, + }, + family); +} + +std::shared_ptr load_roformer_assets_for_family( + const std::filesystem::path & model_path, + std::string_view family) { + runtime::ModelLoadRequest request; + request.model_path = model_path; + request.family_hint = std::string(family); + return load_roformer_assets(request, family); +} + +std::unique_ptr create_roformer_session( const runtime::TaskSpec & task, const runtime::SessionOptions & options, - std::shared_ptr assets) - : RoformerSession(task, options, std::move(assets), nullptr) {} + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, + options, + std::move(assets), + std::move(contract)); +} + +} // namespace RoformerSession::RoformerSession( const runtime::TaskSpec & task, - const runtime::SessionOptions & options, + runtime::SessionOptions options, std::shared_ptr assets, std::shared_ptr contract) - : RuntimeSessionBase(options), + : RuntimeSessionBase(apply_option_v1_compatibility(std::move(options), assets)), task_(task), - assets_(std::move(assets)), - contract_(std::move(contract)) { - if (assets_ == nullptr) { - throw std::runtime_error("RoFormer session requires assets"); - } + assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))) { + runtime::validate_spec_backed_session_options( + RuntimeSessionBase::options(), + *contract_, + assets_->config.family, + assets_->config.family); if (task_.task != runtime::VoiceTaskKind::SourceSeparation) { throw std::runtime_error("RoFormer models only support --task sep"); } if (task_.mode != runtime::RunMode::Offline) { throw std::runtime_error("RoFormer models only support offline mode"); } - if (assets_->config.family == kBsRoformerFamily) { - if (contract_ == nullptr) { - throw std::runtime_error( - "BS-RoFormer session requires a model contract"); - } - validate_session_option_keys( - options, *contract_, kBsRoformerFamily); - } const auto default_weight_storage = core::requested_backend_uses_host_graph_plan(RuntimeSessionBase::options().backend) ? assets::TensorStorageType::F32 @@ -177,6 +202,7 @@ void RoformerSession::prepare(const runtime::SessionPreparationRequest & request runtime::TaskResult RoformerSession::run(const runtime::TaskRequest & request) { require_prepared("RoFormer run()"); + runtime::validate_spec_backed_request_options(request.options, *contract_, assets_->config.family); if (!request.audio_input.has_value()) { throw std::runtime_error("RoFormer run() requires audio_input"); } @@ -330,26 +356,23 @@ runtime::TaskResult RoformerSession::run(const runtime::TaskRequest & request) { return result_task; } +std::shared_ptr make_mel_band_roformer_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = std::string(kMelBandRoformerFamily); + config.load_assets = [](const std::filesystem::path & model_path) { + return load_roformer_assets_for_family(model_path, kMelBandRoformerFamily); + }; + config.create_session = create_roformer_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + std::shared_ptr make_bs_roformer_loader() { runtime::SpecBackedVoiceModelConfig config; config.family = std::string(kBsRoformerFamily); config.load_assets = [](const std::filesystem::path & model_path) { - runtime::ModelLoadRequest request; - request.model_path = model_path; - request.family_hint = std::string(kBsRoformerFamily); - return load_bs_roformer_assets(request); - }; - config.create_session = []( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options, - std::shared_ptr assets, - std::shared_ptr contract) { - return std::make_unique( - task, - options, - std::move(assets), - std::move(contract)); + return load_roformer_assets_for_family(model_path, kBsRoformerFamily); }; + config.create_session = create_roformer_session; return runtime::make_spec_backed_voice_loader(std::move(config)); } diff --git a/src/models/seed_vc/assets.cpp b/src/models/seed_vc/assets.cpp index 40d53f5a..ba1623ef 100644 --- a/src/models/seed_vc/assets.cpp +++ b/src/models/seed_vc/assets.cpp @@ -228,9 +228,7 @@ SeedVcConfig parse_config(const assets::ResourceBundle & resources) { std::shared_ptr load_seed_vc_assets(const std::filesystem::path & model_path) { auto assets = std::make_shared(); - assets->resources = engine::model_spec::load_resource_bundle( - model_path, - engine::model_spec::default_spec_path("seed_vc")); + assets->resources = engine::model_spec::load_resource_bundle_for_family(model_path, "seed_vc"); assets->config = parse_config(assets->resources); assets->v2_ar_weights = assets->resources.open_tensor_source("v2_ar_weights"); assets->v2_cfm_weights = assets->resources.open_tensor_source("v2_cfm_weights"); diff --git a/src/models/seed_vc/loader.cpp b/src/models/seed_vc/loader.cpp deleted file mode 100644 index 57497aef..00000000 --- a/src/models/seed_vc/loader.cpp +++ /dev/null @@ -1,158 +0,0 @@ -#include "engine/models/seed_vc/loader.h" - -#include "engine/framework/model_spec/package.h" -#include "engine/models/seed_vc/assets.h" -#include "engine/models/seed_vc/session.h" - -#include -#include -#include - -namespace engine::models::seed_vc { -namespace { - -runtime::CapabilitySet capabilities(const SeedVcAssets &) { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::VoiceConversion, {runtime::RunMode::Offline}}, - {runtime::VoiceTaskKind::Svc, {runtime::RunMode::Offline}}, - }; - out.supports_speaker_reference = true; - out.supports_style_condition = true; - return out; -} - -runtime::ModelMetadata metadata(const SeedVcAssets &) { - runtime::ModelMetadata out; - out.family = "seed_vc"; - out.variant = "v2-vc-v1-svc"; - out.description = "SeedVC-MLX asset bundle for V2 voice conversion and V1 voice conversion/SVC variants."; - return out; -} - -runtime::ModelCliInterface cli(const SeedVcAssets &) { - runtime::ModelCliInterface out; - out.request_options = { - {"route", "v2_vc|v1_svc|v1_whisper_bigvgan_vc|v1_xlsr_hift_vc", - "Seed-VC route. Defaults to v2_vc for VC and v1_svc for SVC."}, - {"length_adjust", "float", "Duration multiplier; default 1.0."}, - {"num_inference_steps", "n", "Diffusion steps; default 30."}, - {"inference_cfg_rate", "float", "V1 CFG rate; default 0.7."}, - {"intelligibility_cfg_rate", "float", "V2 intelligibility CFG rate; default 0.7."}, - {"similarity_cfg_rate", "float", "V2 similarity CFG rate; default 0.7."}, - {"f0_condition", "true|false", "Enable V1 F0 conditioning."}, - {"auto_f0_adjust", "true|false", "Enable automatic V1 F0 adjustment."}, - {"semi_tone_shift", "n", "V1 semitone shift; default 0."}, - }; - out.session_options = { - {"seed_vc.weight_type", "native|f32|f16|bf16|q8_0", "Seed-VC weight storage type."}, - }; - return out; -} - -class SeedVcLoadedModel final : public runtime::ILoadedVoiceModel { -public: - SeedVcLoadedModel( - runtime::ModelMetadata metadata, - runtime::CapabilitySet capabilities, - std::shared_ptr assets) - : metadata_(std::move(metadata)), - capabilities_(std::move(capabilities)), - assets_(std::move(assets)) { - if (assets_ == nullptr) { - throw std::runtime_error("Seed-VC loaded model requires assets"); - } - } - - const runtime::ModelMetadata & metadata() const noexcept override { - return metadata_; - } - - const runtime::CapabilitySet & capabilities() const noexcept override { - return capabilities_; - } - - std::unique_ptr create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const override { - if (task.mode != runtime::RunMode::Offline) { - throw std::runtime_error("Seed-VC currently supports offline sessions"); - } - if (task.task != runtime::VoiceTaskKind::VoiceConversion && task.task != runtime::VoiceTaskKind::Svc) { - throw std::runtime_error("Seed-VC supports VoiceConversion and Svc tasks"); - } - return std::make_unique(task, options, assets_); - } - -private: - runtime::ModelMetadata metadata_; - runtime::CapabilitySet capabilities_; - std::shared_ptr assets_; -}; - -class SeedVcLoader final : public runtime::IVoiceModelLoader { -public: - std::string family() const override { - return "seed_vc"; - } - - runtime::CapabilitySet advertised_capabilities() const override { - runtime::CapabilitySet out; - out.supported_tasks = { - {runtime::VoiceTaskKind::VoiceConversion, {runtime::RunMode::Offline}}, - {runtime::VoiceTaskKind::Svc, {runtime::RunMode::Offline}}, - }; - out.supports_speaker_reference = true; - out.supports_style_condition = true; - return out; - } - - bool can_load(const runtime::ModelLoadRequest & request) const override { - if (request.family_hint.has_value() && *request.family_hint != family()) { - return false; - } - try { - (void) engine::model_spec::load_resource_bundle( - request.model_path, - engine::model_spec::default_spec_path(family())); - return true; - } catch (...) { - return false; - } - } - - runtime::ModelInspection inspect(const runtime::ModelLoadRequest & request) const override { - const auto assets = load_seed_vc_assets(request.model_path); - runtime::ModelInspection inspection; - inspection.model_root = assets->resources.model_root(); - inspection.metadata = metadata(*assets); - inspection.capabilities = capabilities(*assets); - inspection.cli = cli(*assets); - const auto spec_path = engine::model_spec::default_spec_path(family()); - inspection.discovered_configs = runtime::discover_named_assets_from_package_spec( - request.model_path, - spec_path, - engine::model_spec::ResourceKind::Files); - inspection.discovered_weights = runtime::discover_named_assets_from_package_spec( - request.model_path, - spec_path, - engine::model_spec::ResourceKind::Tensors); - return inspection; - } - - std::unique_ptr load(const runtime::ModelLoadRequest & request) const override { - auto assets = load_seed_vc_assets(request.model_path); - return std::make_unique( - metadata(*assets), - capabilities(*assets), - std::move(assets)); - } -}; - -} // namespace - -std::shared_ptr make_seed_vc_loader() { - return std::make_shared(); -} - -} // namespace engine::models::seed_vc diff --git a/src/models/seed_vc/session.cpp b/src/models/seed_vc/session.cpp index fa517eb8..28306ec3 100644 --- a/src/models/seed_vc/session.cpp +++ b/src/models/seed_vc/session.cpp @@ -10,6 +10,7 @@ #include "engine/framework/modules/speech_encoders/hubert_encoder.h" #include "engine/framework/io/binary.h" #include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" #include "engine/framework/sampling/torch_random.h" #include "engine/models/seed_vc/astral_quantizer.h" #include "engine/models/seed_vc/audio_features.h" @@ -39,6 +40,8 @@ namespace { constexpr int64_t kSeedVcBigVganActiveFrames = 1408; constexpr int64_t kSeedVcBigVganOverlapFrames = 32; +constexpr const char * kFamily = "seed_vc"; + std::shared_ptr require_assets(std::shared_ptr assets) { if (assets == nullptr) { throw std::runtime_error("Seed-VC session requires assets"); @@ -46,6 +49,69 @@ std::shared_ptr require_assets(std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("Seed-VC session requires a model contract"); + } + return contract; +} + +runtime::SessionOptions normalize_session_options(runtime::SessionOptions options) { + return runtime::apply_option_v1_compatibility( + std::move(options), + { + {"weight_type", "seed_vc.weight_type"}, + }, + "Seed-VC"); +} + +runtime::SessionOptions require_supported_session_options( + runtime::SessionOptions options, + const std::shared_ptr & contract) { + options = normalize_session_options(std::move(options)); + const auto checked_contract = require_contract(contract); + runtime::validate_spec_backed_session_options(options, *checked_contract, kFamily, "Seed-VC"); + return options; +} + +std::unordered_map normalize_request_options( + std::unordered_map options) { + options = runtime::apply_option_v1_compatibility( + std::move(options), + { + {"inference_cfg_rate", "inference_guidance_scale"}, + {"intelligibility_cfg_rate", "intelligibility_guidance_scale"}, + {"similarity_cfg_rate", "similarity_guidance_scale"}, + {"anonymization_only", "voice_anonymization"}, + {"noise_file", "noise_path"}, + {"semi_tone_shift", "semitone_shift"}, + }, + "Seed-VC", + "request"); + return options; +} + +std::unordered_map validated_request_options( + const std::unordered_map & options, + const engine::model_spec::ModelContract & contract) { + auto normalized = normalize_request_options(options); + runtime::validate_spec_backed_request_options(normalized, contract, "Seed-VC"); + return normalized; +} + +std::unique_ptr create_seed_vc_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, + options, + std::move(assets), + std::move(contract)); +} + } // namespace struct SeedVcRouteRuntime { @@ -78,23 +144,20 @@ struct SeedVcV2RequestConfig { float length_adjust = 1.0F; float intelligibility_cfg_rate = 0.7F; float similarity_cfg_rate = 0.7F; - float top_p = 0.9F; - float temperature = 1.0F; - float repetition_penalty = 1.0F; - bool convert_style = false; - bool anonymization_only = false; + bool voice_anonymization = false; uint64_t seed = 1234; - std::string noise_file; + std::string noise_path; }; struct SeedVcV1RequestConfig { int num_inference_steps = 30; float length_adjust = 1.0F; - float inference_cfg_rate = 0.7F; + float inference_guidance_scale = 0.7F; bool f0_condition = false; bool auto_f0_adjust = false; - int semi_tone_shift = 0; - bool fp16 = true; + int semitone_shift = 0; + uint64_t seed = 1234; + std::string noise_path; }; struct SeedVcExecutionPlan { @@ -109,8 +172,7 @@ struct SeedVcExecutionPlan { std::optional v1; }; -std::string request_route_or_default(const runtime::TaskRequest & request, runtime::VoiceTaskKind task); -std::string route_path_or_default_from_options( +std::string resolve_route_path( const std::unordered_map & options, runtime::VoiceTaskKind task); @@ -174,21 +236,16 @@ bool route_is_v1(const SeedVcRouteRuntime::Route route) { route == SeedVcRouteRuntime::Route::V1XlsrHiftVoiceConversion; } -std::optional parse_seed_vc_weight_type( - const runtime::SessionOptions & options) { - const auto it = options.options.find("seed_vc.weight_type"); - if (it == options.options.end()) { - return std::nullopt; - } - const auto storage_type = engine::assets::parse_tensor_storage_type(it->second); - if (storage_type == engine::assets::TensorStorageType::Native || - storage_type == engine::assets::TensorStorageType::F32 || - storage_type == engine::assets::TensorStorageType::F16 || - storage_type == engine::assets::TensorStorageType::BF16 || - storage_type == engine::assets::TensorStorageType::Q8_0) { - return storage_type; - } - throw std::runtime_error("seed_vc.weight_type currently supports only native, f32, f16, bf16, and q8_0"); +std::optional parse_seed_vc_weight_type(const runtime::SessionOptions & options) { + using T = engine::assets::TensorStorageType; + if (runtime::find_option(options.options, {"seed_vc.weight_type"}).has_value()) { + return runtime::parse_tensor_storage_option( + options.options, + "seed_vc.weight_type", + T::Native, + {T::Native, T::F32, T::F16, T::BF16, T::Q8_0}); + } + return std::nullopt; } const SeedVcMelConfig & v1_mel_config_for_path(const SeedVcAssets & assets, const std::string & path) { @@ -284,7 +341,7 @@ std::vector adjust_source_f0_like_python( const std::vector & source_f0, const std::vector & target_f0, bool auto_f0_adjust, - int semi_tone_shift) { + int semitone_shift) { std::vector source_voiced_log; std::vector target_voiced_log; source_voiced_log.reserve(source_f0.size()); @@ -301,7 +358,7 @@ std::vector adjust_source_f0_like_python( } const float source_median = median_like_torch_1d(std::move(source_voiced_log)); const float target_median = median_like_torch_1d(std::move(target_voiced_log)); - const float pitch_scale = std::pow(2.0F, static_cast(semi_tone_shift) / 12.0F); + const float pitch_scale = std::pow(2.0F, static_cast(semitone_shift) / 12.0F); std::vector shifted(source_f0.size(), 0.0F); for (size_t index = 0; index < source_f0.size(); ++index) { const float value = source_f0[index]; @@ -310,7 +367,7 @@ std::vector adjust_source_f0_like_python( log_value = log_value - source_median + target_median; } float out = std::exp(log_value); - if (value > 1.0F && semi_tone_shift != 0) { + if (value > 1.0F && semitone_shift != 0) { out *= pitch_scale; } shifted[index] = out; @@ -318,51 +375,31 @@ std::vector adjust_source_f0_like_python( return shifted; } -SeedVcV2RequestConfig parse_v2_config(const runtime::TaskRequest & request) { +SeedVcV2RequestConfig parse_v2_config(const std::unordered_map & options) { SeedVcV2RequestConfig config; config.num_inference_steps = runtime::parse_int_option( - request.options, + options, {"num_inference_steps"}) .value_or(config.num_inference_steps); config.length_adjust = runtime::parse_finite_float_option( - request.options, + options, {"length_adjust"}) .value_or(config.length_adjust); config.intelligibility_cfg_rate = runtime::parse_finite_float_option( - request.options, - {"intelligibility_cfg_rate"}) + options, + {"intelligibility_guidance_scale"}) .value_or(config.intelligibility_cfg_rate); config.similarity_cfg_rate = runtime::parse_finite_float_option( - request.options, - {"similarity_cfg_rate"}) + options, + {"similarity_guidance_scale"}) .value_or(config.similarity_cfg_rate); - config.top_p = runtime::parse_finite_float_option(request.options, {"top_p"}) - .value_or(config.top_p); - config.temperature = runtime::parse_finite_float_option(request.options, {"temperature"}) - .value_or(config.temperature); - config.repetition_penalty = runtime::parse_finite_float_option( - request.options, - {"repetition_penalty"}) - .value_or(config.repetition_penalty); - if (const auto value = runtime::find_option(request.options, {"convert_style"})) { - config.convert_style = runtime::parse_bool_option(*value, "convert_style"); - } - if (const auto value = runtime::find_option(request.options, {"anonymization_only"})) { - config.anonymization_only = runtime::parse_bool_option(*value, "anonymization_only"); - } - config.seed = runtime::parse_u64_option(request.options, {"seed"}) + if (const auto value = runtime::find_option(options, {"voice_anonymization"})) { + config.voice_anonymization = runtime::parse_bool_option(*value, "voice_anonymization"); + } + config.seed = runtime::parse_u64_option(options, {"seed"}) .value_or(runtime::random_u64_seed()); - config.noise_file = runtime::find_option(request.options, {"noise_file"}).value_or(""); + config.noise_path = runtime::find_option(options, {"noise_path"}).value_or(""); validate_common_generation_options(config.num_inference_steps, config.length_adjust); - if (!(config.top_p > 0.0F && config.top_p <= 1.0F)) { - throw std::runtime_error("Seed-VC top_p must be in (0, 1]"); - } - if (!(config.temperature > 0.0F)) { - throw std::runtime_error("Seed-VC temperature must be positive"); - } - if (!(config.repetition_penalty > 0.0F)) { - throw std::runtime_error("Seed-VC repetition_penalty must be positive"); - } return config; } @@ -525,18 +562,18 @@ std::vector synthesize_bigvgan_fixed_chunks( } std::vector load_seed_vc_noise_or_sample( - const std::string & noise_file, + const std::string & noise_path, size_t count, uint64_t seed, uint64_t offset) { - if (noise_file.empty()) { + if (noise_path.empty()) { return engine::sampling::generate_torch_cuda_randn( count, seed, engine::sampling::TorchRandnPrecision::Float32, offset); } - auto values = engine::io::read_f32_file(noise_file); + auto values = engine::io::read_f32_file(noise_path); if (values.size() < offset + count) { throw std::runtime_error( "Seed-VC noise file is too short: expected at least " + @@ -560,9 +597,6 @@ runtime::TaskResult run_v2_voice_conversion( throw std::runtime_error("Seed-VC V2 route requires V2 config"); } const auto & config = *plan.v2; - if (config.convert_style) { - throw std::runtime_error("Seed-VC V2 convert_style path requires AR generation and is not implemented yet"); - } if (sources.route != SeedVcRouteRuntime::Route::V2VoiceConversion) { throw std::runtime_error("Seed-VC V2 route requires V2 component sources"); } @@ -697,7 +731,7 @@ runtime::TaskResult run_v2_voice_conversion( const size_t noise_count = static_cast(assets.config.v2_cfm.in_channels * original_len); auto initial_noise = load_seed_vc_noise_or_sample( - config.noise_file, + config.noise_path, noise_count, config.seed, random_offset); @@ -715,7 +749,7 @@ runtime::TaskResult run_v2_voice_conversion( cfm_input.temperature = 1.0F; cfm_input.intelligibility_cfg_rate = config.intelligibility_cfg_rate; cfm_input.similarity_cfg_rate = config.similarity_cfg_rate; - cfm_input.random_voice = config.anonymization_only; + cfm_input.random_voice = config.voice_anonymization; timing_start = std::chrono::steady_clock::now(); const auto cfm_output = sources.v2_cfm_estimator.infer(cfm_input); @@ -884,7 +918,7 @@ runtime::TaskResult run_v1_singing_voice_conversion( raw_source_f0, target_f0, config.auto_f0_adjust, - config.semi_tone_shift); + config.semitone_shift); } timing_end = std::chrono::steady_clock::now(); engine::debug::timing_log_scalar( @@ -929,10 +963,8 @@ runtime::TaskResult run_v1_singing_voice_conversion( uint64_t random_offset = 0; std::vector generated_wave; std::vector previous_chunk; - const uint64_t seed = runtime::parse_u64_option(request.options, {"seed"}) - .value_or(runtime::random_u64_seed()); - const std::string noise_file = - runtime::find_option(request.options, {"noise_file"}).value_or(""); + const uint64_t seed = config.seed; + const std::string & noise_path = config.noise_path; while (processed_frames < source_condition.tokens) { const int64_t chunk_frames = std::min(max_source_window, source_condition.tokens - processed_frames); @@ -953,7 +985,7 @@ runtime::TaskResult run_v1_singing_voice_conversion( const size_t noise_count = static_cast(dit_config.in_channels * original_len); auto initial_noise = load_seed_vc_noise_or_sample( - noise_file, + noise_path, noise_count, seed, random_offset); @@ -969,7 +1001,7 @@ runtime::TaskResult run_v1_singing_voice_conversion( cfm_input.prompt_frames = target_mel.frames; cfm_input.num_inference_steps = config.num_inference_steps; cfm_input.temperature = 1.0F; - cfm_input.inference_cfg_rate = config.inference_cfg_rate; + cfm_input.inference_cfg_rate = config.inference_guidance_scale; const auto cfm_output = sources.v1_cfm_estimator.infer(cfm_input); timing_end = std::chrono::steady_clock::now(); @@ -1000,9 +1032,9 @@ runtime::TaskResult run_v1_singing_voice_conversion( static_cast(harmonics) + static_cast(harmonics) * source_samples; const size_t stream_random_count = used_random_count + source_samples; hift_source_advance_count = static_cast(stream_random_count); - if (!noise_file.empty()) { + if (!noise_path.empty()) { auto hift_random_stream = load_seed_vc_noise_or_sample( - noise_file, + noise_path, stream_random_count, seed, random_offset); @@ -1033,7 +1065,7 @@ runtime::TaskResult run_v1_singing_voice_conversion( use_hift ? "seed_vc.v1.hift_ms" : "seed_vc.v1.bigvgan_ms", engine::debug::elapsed_ms(timing_start, timing_end)); timing_start = timing_end; - if (use_hift && noise_file.empty()) { + if (use_hift && noise_path.empty()) { random_offset += hift_source_advance_count; } @@ -1071,40 +1103,43 @@ runtime::TaskResult run_v1_singing_voice_conversion( return result; } -SeedVcV1RequestConfig parse_v1_config(const runtime::TaskRequest & request) { +SeedVcV1RequestConfig parse_v1_config(const std::unordered_map & options) { SeedVcV1RequestConfig config; config.num_inference_steps = runtime::parse_int_option( - request.options, + options, {"num_inference_steps"}) .value_or(config.num_inference_steps); config.length_adjust = runtime::parse_finite_float_option( - request.options, + options, {"length_adjust"}) .value_or(config.length_adjust); - config.inference_cfg_rate = runtime::parse_finite_float_option( - request.options, - {"inference_cfg_rate"}) - .value_or(config.inference_cfg_rate); - if (const auto value = runtime::find_option(request.options, {"f0_condition"})) { + config.inference_guidance_scale = runtime::parse_finite_float_option( + options, + {"inference_guidance_scale"}) + .value_or(config.inference_guidance_scale); + if (const auto value = runtime::find_option(options, {"f0_condition"})) { config.f0_condition = runtime::parse_bool_option(*value, "f0_condition"); } - if (const auto value = runtime::find_option(request.options, {"auto_f0_adjust"})) { + if (const auto value = runtime::find_option(options, {"auto_f0_adjust"})) { config.auto_f0_adjust = runtime::parse_bool_option(*value, "auto_f0_adjust"); } - config.semi_tone_shift = runtime::parse_int_option( - request.options, - {"semi_tone_shift"}) - .value_or(config.semi_tone_shift); - if (const auto value = runtime::find_option(request.options, {"fp16"})) { - config.fp16 = runtime::parse_bool_option(*value, "fp16"); - } + config.semitone_shift = runtime::parse_int_option( + options, + {"semitone_shift"}) + .value_or(config.semitone_shift); + config.seed = runtime::parse_u64_option(options, {"seed"}) + .value_or(runtime::random_u64_seed()); + config.noise_path = runtime::find_option(options, {"noise_path"}).value_or(""); validate_common_generation_options(config.num_inference_steps, config.length_adjust); return config; } -SeedVcExecutionPlan make_execution_plan(const runtime::TaskRequest & request, runtime::VoiceTaskKind task) { +SeedVcExecutionPlan make_execution_plan( + const runtime::TaskRequest & request, + const std::unordered_map & options, + runtime::VoiceTaskKind task) { SeedVcExecutionPlan plan; - plan.path = request_route_or_default(request, task); + plan.path = resolve_route_path(options, task); const auto & source = require_source_audio(request); const auto & target = require_target_audio(request); plan.source_sample_rate = source.sample_rate; @@ -1114,15 +1149,14 @@ SeedVcExecutionPlan make_execution_plan(const runtime::TaskRequest & request, ru plan.source_frames = static_cast(source.samples.size() / static_cast(source.channels)); plan.target_frames = static_cast(target.samples.size() / static_cast(target.channels)); if (plan.path == "v2_vc") { - plan.v2 = parse_v2_config(request); + plan.v2 = parse_v2_config(options); } else if (is_v1_path(plan.path)) { - plan.v1 = parse_v1_config(request); + plan.v1 = parse_v1_config(options); } return plan; } std::shared_ptr open_route_runtime( - runtime::VoiceTaskKind task, const engine::core::BackendConfig & backend, const SeedVcAssets & assets, const std::string & route_path, @@ -1135,9 +1169,6 @@ std::shared_ptr open_route_runtime( backend, engine::modules::CampplusEncoderConfig{80, 192, default_weight_storage_type}); if (route_path == "v2_vc") { - if (task != runtime::VoiceTaskKind::VoiceConversion) { - throw std::runtime_error("Seed-VC v2_vc sources require a VoiceConversion session"); - } sources->route = SeedVcRouteRuntime::Route::V2VoiceConversion; sources->v2_ar_length_regulator = SeedVcDiscreteLengthRegulator( assets.v2_ar_weights, @@ -1186,9 +1217,6 @@ std::shared_ptr open_route_runtime( backend, hubert_config); } else if (route_path == "v1_svc") { - if (task != runtime::VoiceTaskKind::Svc) { - throw std::runtime_error("Seed-VC v1_svc sources require an Svc session"); - } sources->route = SeedVcRouteRuntime::Route::V1SingingVoiceConversion; sources->v1_length_regulator = SeedVcV1LengthRegulator( assets.v1_svc_weights, @@ -1215,9 +1243,6 @@ std::shared_ptr open_route_runtime( backend, make_bigvgan_config(assets.config.bigvgan_44k, default_weight_storage_type)); } else if (route_path == "v1_whisper_bigvgan_vc") { - if (task != runtime::VoiceTaskKind::VoiceConversion) { - throw std::runtime_error("Seed-VC v1_whisper_bigvgan_vc sources require a VoiceConversion session"); - } sources->route = SeedVcRouteRuntime::Route::V1WhisperBigVganVoiceConversion; sources->v1_length_regulator = SeedVcV1LengthRegulator( assets.v1_whisper_bigvgan_weights, @@ -1240,9 +1265,6 @@ std::shared_ptr open_route_runtime( backend, make_bigvgan_config(assets.config.bigvgan_22k, default_weight_storage_type)); } else if (route_path == "v1_xlsr_hift_vc") { - if (task != runtime::VoiceTaskKind::VoiceConversion) { - throw std::runtime_error("Seed-VC v1_xlsr_hift_vc sources require a VoiceConversion session"); - } sources->route = SeedVcRouteRuntime::Route::V1XlsrHiftVoiceConversion; sources->v1_length_regulator = SeedVcV1LengthRegulator( assets.v1_xlsr_hift_weights, @@ -1273,39 +1295,30 @@ std::shared_ptr open_route_runtime( return sources; } -std::string request_route_or_default(const runtime::TaskRequest & request, runtime::VoiceTaskKind task) { - return route_path_or_default_from_options(request.options, task); -} - -std::string route_path_or_default_from_options( +std::string resolve_route_path( const std::unordered_map & options, runtime::VoiceTaskKind task) { const auto it = options.find("route"); - if (it != options.end() && !it->second.empty()) { - return it->second; - } - return task == runtime::VoiceTaskKind::Svc ? "v1_svc" : "v2_vc"; -} - -void validate_request_route(const runtime::TaskRequest & request, runtime::VoiceTaskKind task) { - const auto route = request_route_or_default(request, task); + const std::string route = (it != options.end() && !it->second.empty()) + ? it->second + : (task == runtime::VoiceTaskKind::Svc ? "v1_svc" : "v2_vc"); if (route == "v2_vc") { if (task != runtime::VoiceTaskKind::VoiceConversion) { throw std::runtime_error("Seed-VC v2_vc request requires a VoiceConversion session"); } - return; + return route; } if (route == "v1_svc") { if (task != runtime::VoiceTaskKind::Svc) { throw std::runtime_error("Seed-VC v1_svc request requires an Svc session"); } - return; + return route; } if (is_v1_voice_conversion_path(route)) { if (task != runtime::VoiceTaskKind::VoiceConversion) { throw std::runtime_error("Seed-VC " + route + " request requires a VoiceConversion session"); } - return; + return route; } throw std::runtime_error("unsupported Seed-VC route: " + route); } @@ -1327,14 +1340,23 @@ std::string route_path_for_runtime(SeedVcRouteRuntime::Route route) { SeedVcSession::SeedVcSession( runtime::TaskSpec task, runtime::SessionOptions options, - std::shared_ptr assets) - : RuntimeSessionBase(options), - task_(task), + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(require_supported_session_options(std::move(options), contract)), + task_(std::move(task)), assets_(require_assets(std::move(assets))), - weight_storage_type_(parse_seed_vc_weight_type(this->options())) {} + contract_(require_contract(std::move(contract))), + weight_storage_type_(parse_seed_vc_weight_type(this->options())) { + if (task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("Seed-VC currently supports offline sessions"); + } + if (task_.task != runtime::VoiceTaskKind::VoiceConversion && task_.task != runtime::VoiceTaskKind::Svc) { + throw std::runtime_error("Seed-VC supports VoiceConversion and Svc tasks"); + } +} std::string SeedVcSession::family() const { - return "seed_vc"; + return kFamily; } runtime::VoiceTaskKind SeedVcSession::task_kind() const { @@ -1346,7 +1368,8 @@ runtime::RunMode SeedVcSession::run_mode() const { } void SeedVcSession::prepare(const runtime::SessionPreparationRequest & request) { - const std::string route_path = route_path_or_default_from_options(request.options, task_.task); + const auto request_options = validated_request_options(request.options, *contract_); + const std::string route_path = resolve_route_path(request_options, task_.task); if (route_runtime_ != nullptr) { const std::string prepared_route = route_path_for_runtime(route_runtime_->route); if (prepared_route != route_path) { @@ -1358,7 +1381,6 @@ void SeedVcSession::prepare(const runtime::SessionPreparationRequest & request) return; } route_runtime_ = open_route_runtime( - task_.task, options().backend, *assets_, route_path, @@ -1392,8 +1414,8 @@ void SeedVcSession::prepare(const runtime::SessionPreparationRequest & request) runtime::TaskResult SeedVcSession::run(const runtime::TaskRequest & request) { require_prepared("Seed-VC run"); const auto wall_start = std::chrono::steady_clock::now(); - validate_request_route(request, task_.task); - const auto plan = make_execution_plan(request, task_.task); + const auto request_options = validated_request_options(request.options, *contract_); + const auto plan = make_execution_plan(request, request_options, task_.task); if (route_runtime_ == nullptr) { throw std::runtime_error("Seed-VC session has no prepared route"); } @@ -1416,4 +1438,12 @@ runtime::TaskResult SeedVcSession::run(const runtime::TaskRequest & request) { throw std::runtime_error("Seed-VC " + plan.path + " graph execution is not implemented yet"); } +std::shared_ptr make_seed_vc_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_seed_vc_assets; + config.create_session = create_seed_vc_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + } // namespace engine::models::seed_vc diff --git a/tests/perf/model_perf_request_cases.json b/tests/perf/model_perf_request_cases.json index fe534cb2..f8acda05 100644 --- a/tests/perf/model_perf_request_cases.json +++ b/tests/perf/model_perf_request_cases.json @@ -533,8 +533,7 @@ "inference_cfg_rate": 0.7, "f0_condition": true, "auto_f0_adjust": true, - "semi_tone_shift": 0, - "fp16": false + "semi_tone_shift": 0 }, "seed": 1234, "num_inference_steps": 30 @@ -559,8 +558,7 @@ "route": "v1_whisper_bigvgan_vc", "options": { "length_adjust": 1.0, - "inference_cfg_rate": 0.7, - "fp16": false + "inference_cfg_rate": 0.7 }, "seed": 1234, "num_inference_steps": 30 @@ -585,8 +583,7 @@ "route": "v1_xlsr_hift_vc", "options": { "length_adjust": 1.0, - "inference_cfg_rate": 0.7, - "fp16": false + "inference_cfg_rate": 0.7 }, "seed": 1234, "num_inference_steps": 30 diff --git a/tests/seed_vc/seed_vc_warm_bench_cases.json b/tests/seed_vc/seed_vc_warm_bench_cases.json index 097cc865..17536e22 100644 --- a/tests/seed_vc/seed_vc_warm_bench_cases.json +++ b/tests/seed_vc/seed_vc_warm_bench_cases.json @@ -46,8 +46,7 @@ "inference_cfg_rate": 0.7, "f0_condition": true, "auto_f0_adjust": true, - "semi_tone_shift": 0, - "fp16": false + "semi_tone_shift": 0 } ] }, @@ -62,8 +61,7 @@ "inference_cfg_rate": 0.7, "f0_condition": true, "auto_f0_adjust": false, - "semi_tone_shift": 2, - "fp16": false + "semi_tone_shift": 2 } ] }, @@ -78,8 +76,7 @@ "inference_cfg_rate": 0.7, "f0_condition": true, "auto_f0_adjust": true, - "semi_tone_shift": 0, - "fp16": false + "semi_tone_shift": 0 } ] }, @@ -94,8 +91,7 @@ "inference_cfg_rate": 0.7, "f0_condition": false, "auto_f0_adjust": false, - "semi_tone_shift": 0, - "fp16": false + "semi_tone_shift": 0 } ] }, @@ -110,8 +106,7 @@ "inference_cfg_rate": 0.7, "f0_condition": false, "auto_f0_adjust": false, - "semi_tone_shift": 0, - "fp16": false + "semi_tone_shift": 0 } ] } diff --git a/tools/audiocpp_cli/audiocpp_cli_path_cases.json b/tools/audiocpp_cli/audiocpp_cli_path_cases.json index 17a258bb..8d0c46bb 100644 --- a/tools/audiocpp_cli/audiocpp_cli_path_cases.json +++ b/tools/audiocpp_cli/audiocpp_cli_path_cases.json @@ -1086,8 +1086,7 @@ "inference_cfg_rate": 0.7, "f0_condition": true, "auto_f0_adjust": true, - "semi_tone_shift": 0, - "fp16": false + "semi_tone_shift": 0 }, "seed": 1234, "num_inference_steps": 30 @@ -1112,8 +1111,7 @@ "route": "v1_whisper_bigvgan_vc", "options": { "length_adjust": 1.0, - "inference_cfg_rate": 0.7, - "fp16": false + "inference_cfg_rate": 0.7 }, "seed": 1234, "num_inference_steps": 30 @@ -1138,8 +1136,7 @@ "route": "v1_xlsr_hift_vc", "options": { "length_adjust": 1.0, - "inference_cfg_rate": 0.7, - "fp16": false + "inference_cfg_rate": 0.7 }, "seed": 1234, "num_inference_steps": 30