diff --git a/CMakeLists.txt b/CMakeLists.txt index 40221b37..3a4676aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -279,6 +279,7 @@ add_library(engine_core OBJECT src/framework/modules/vocoders/hift_vocoder.cpp src/framework/modules/speech_encoders/hubert_encoder.cpp src/framework/modules/speech_encoders/wavlm_encoder.cpp + src/framework/modules/speech_encoders/sanm.cpp src/framework/modules/optimizations/fast_conv_modules.cpp src/framework/modules/optimizations/fast_projection_modules.cpp src/framework/modules/recurrent_modules.cpp @@ -311,6 +312,7 @@ add_library(engine_core OBJECT external/llama_tokenizer/unicode-data.cpp src/framework/tokenizers/sentencepiece.cpp src/framework/audio/fft.cpp + src/framework/audio/kaldi_fbank.cpp src/framework/audio/activity.cpp src/framework/audio/chunking.cpp src/framework/audio/conversion.cpp @@ -689,6 +691,23 @@ audiocpp_add_model(dramabox engine::models::dramabox::make_dramabox_loader ) +audiocpp_add_model(fun_asr_nano + SOURCES + src/models/fun_asr_nano/assets.cpp + src/models/fun_asr_nano/frontend.cpp + src/models/fun_asr_nano/encoder.cpp + src/models/fun_asr_nano/adaptor.cpp + src/models/fun_asr_nano/tokenizer_text.cpp + src/models/fun_asr_nano/prompt.cpp + src/models/fun_asr_nano/decoder.cpp + src/models/fun_asr_nano/session.cpp + src/models/fun_asr_nano/loader.cpp + INCLUDES + engine/models/fun_asr_nano/loader.h + LOADERS + engine::models::fun_asr_nano::make_fun_asr_nano_loader +) + audiocpp_add_model(heartmula SOURCES src/models/heartmula/assets.cpp @@ -1330,6 +1349,7 @@ if (ENGINE_BUILD_WARMBENCH) add_engine_warmbench(citrinet_asr_warm_bench tests/citrinet_asr/citrinet_asr_warm_bench.cpp) add_engine_warmbench(confucius4_tts_warm_bench tests/confucius4_tts/confucius4_tts_warm_bench.cpp) add_engine_warmbench(dramabox_warm_bench tools/dramabox/dramabox_warm_bench.cpp) + add_engine_warmbench(fun_asr_nano_warm_bench tests/fun_asr_nano/fun_asr_nano_warm_bench.cpp) add_engine_warmbench(higgs_audio_stt_warm_bench tests/higgs_audio_stt/higgs_audio_stt_warm_bench.cpp) add_engine_warmbench(higgs_audio_tts_warm_bench tests/higgs_audio_tts/higgs_audio_tts_warm_bench.cpp) add_engine_warmbench(hviske_asr_warm_bench tests/hviske_asr/hviske_asr_warm_bench.cpp) @@ -1666,6 +1686,214 @@ if (ENGINE_BUILD_TESTS) COMMAND model_spec_system_test ) + add_engine_unittest(fun_asr_nano_assets_test tests/unittests/test_fun_asr_nano_assets.cpp) + target_sources(fun_asr_nano_assets_test PRIVATE src/models/fun_asr_nano/assets.cpp) + target_include_directories(fun_asr_nano_assets_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) + + add_test( + NAME fun_asr_nano_assets_test + COMMAND fun_asr_nano_assets_test + ) + + add_engine_unittest(fun_asr_nano_frontend_probe tests/fun_asr_nano/fun_asr_nano_frontend_probe.cpp) + + add_test( + NAME fun_asr_nano_frontend_probe + COMMAND fun_asr_nano_frontend_probe + ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/frontend_reference.json + ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/frontend_reference.bin + ${CMAKE_CURRENT_SOURCE_DIR}/assets/resources/sample_16k.wav + ) + + add_engine_unittest(fun_asr_nano_sanm_probe tests/fun_asr_nano/sanm_probe.cpp) + + add_test( + NAME fun_asr_nano_sanm_probe + COMMAND fun_asr_nano_sanm_probe + --backend cpu + --reference ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/sanm_reference.json + --data ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/sanm_reference.bin + ) + + add_engine_unittest(fun_asr_nano_encoder_probe tests/fun_asr_nano/fun_asr_nano_encoder_probe.cpp) + target_sources( + fun_asr_nano_encoder_probe + PRIVATE + src/models/fun_asr_nano/assets.cpp + src/models/fun_asr_nano/frontend.cpp + src/models/fun_asr_nano/encoder.cpp + ) + set( + AUDIOCPP_FUN_ASR_NANO_TEST_MODEL + "" + CACHE PATH + "Optional local Fun-ASR-Nano model directory for full runtime parity tests" + ) + set( + AUDIOCPP_FUN_ASR_NANO_TEST_GGUF + "" + CACHE FILEPATH + "Optional standalone Fun-ASR-Nano GGUF for full CLI transcript tests" + ) + if (AUDIOCPP_FUN_ASR_NANO_TEST_MODEL) + add_test( + NAME fun_asr_nano_encoder_probe_cpu + COMMAND fun_asr_nano_encoder_probe + --backend cpu + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_MODEL} + --reference ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/encoder_reference.json + --data ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/encoder_reference.bin + ) + if (GGML_CUDA) + add_test( + NAME fun_asr_nano_encoder_probe_cuda + COMMAND fun_asr_nano_encoder_probe + --backend cuda + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_MODEL} + --reference ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/encoder_reference.json + --data ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/encoder_reference.bin + ) + endif() + + add_engine_unittest(fun_asr_nano_session_probe tests/fun_asr_nano/fun_asr_nano_session_probe.cpp) + add_test( + NAME fun_asr_nano_session_probe_cpu + COMMAND fun_asr_nano_session_probe + --backend cpu + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_MODEL} + --audio ${CMAKE_CURRENT_SOURCE_DIR}/assets/resources/sample_16k.wav + ) + if (GGML_CUDA) + add_test( + NAME fun_asr_nano_session_probe_cuda + COMMAND fun_asr_nano_session_probe + --backend cuda + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_MODEL} + --audio ${CMAKE_CURRENT_SOURCE_DIR}/assets/resources/sample_16k.wav + ) + endif() + endif() + if (AUDIOCPP_FUN_ASR_NANO_TEST_GGUF) + add_test( + NAME fun_asr_nano_gguf_cli_cpu + COMMAND audiocpp_cli + --task asr + --family fun_asr_nano + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_GGUF} + --backend cpu + --audio ${CMAKE_CURRENT_SOURCE_DIR}/assets/resources/sample_16k.wav + ) + set_tests_properties( + fun_asr_nano_gguf_cli_cpu + PROPERTIES PASS_REGULAR_EXPRESSION + "text_output=Some call me nature, others call me mother nature" + ) + if (GGML_CUDA) + add_test( + NAME fun_asr_nano_gguf_cli_cuda + COMMAND audiocpp_cli + --task asr + --family fun_asr_nano + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_GGUF} + --backend cuda + --audio ${CMAKE_CURRENT_SOURCE_DIR}/assets/resources/sample_16k.wav + ) + set_tests_properties( + fun_asr_nano_gguf_cli_cuda + PROPERTIES PASS_REGULAR_EXPRESSION + "text_output=Some call me nature, others call me mother nature" + ) + add_test( + NAME fun_asr_nano_gguf_cli_cuda_shared_q8 + COMMAND audiocpp_cli + --task asr + --family fun_asr_nano + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_GGUF} + --backend cuda + --session-option fun_asr_nano.weight_type=q8_0 + --audio ${CMAKE_CURRENT_SOURCE_DIR}/assets/resources/sample_16k.wav + ) + set_tests_properties( + fun_asr_nano_gguf_cli_cuda_shared_q8 + PROPERTIES PASS_REGULAR_EXPRESSION + "text_output=Some call me nature, others call me mother nature" + ) + add_test( + NAME fun_asr_nano_gguf_cli_best_shared_q8 + COMMAND audiocpp_cli + --task asr + --family fun_asr_nano + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_GGUF} + --backend best + --session-option fun_asr_nano.weight_type=q8_0 + --audio ${CMAKE_CURRENT_SOURCE_DIR}/assets/resources/sample_16k.wav + ) + set_tests_properties( + fun_asr_nano_gguf_cli_best_shared_q8 + PROPERTIES PASS_REGULAR_EXPRESSION + "text_output=Some call me nature, others call me mother nature" + ) + endif() + endif() + + add_engine_unittest(fun_asr_nano_adaptor_probe tests/fun_asr_nano/fun_asr_nano_adaptor_probe.cpp) + target_sources( + fun_asr_nano_adaptor_probe + PRIVATE + src/models/fun_asr_nano/assets.cpp + src/models/fun_asr_nano/adaptor.cpp + ) + if (AUDIOCPP_FUN_ASR_NANO_TEST_MODEL) + add_test( + NAME fun_asr_nano_adaptor_probe_cpu + COMMAND fun_asr_nano_adaptor_probe + --backend cpu + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_MODEL} + --reference ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/adaptor_reference.json + --data ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/adaptor_reference.bin + ) + if (GGML_CUDA) + add_test( + NAME fun_asr_nano_adaptor_probe_cuda + COMMAND fun_asr_nano_adaptor_probe + --backend cuda + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_MODEL} + --reference ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/adaptor_reference.json + --data ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/adaptor_reference.bin + ) + endif() + endif() + + add_engine_unittest(fun_asr_nano_decoder_probe tests/fun_asr_nano/fun_asr_nano_decoder_probe.cpp) + target_sources( + fun_asr_nano_decoder_probe + PRIVATE + src/models/fun_asr_nano/assets.cpp + src/models/fun_asr_nano/tokenizer_text.cpp + src/models/fun_asr_nano/prompt.cpp + src/models/fun_asr_nano/decoder.cpp + ) + if (AUDIOCPP_FUN_ASR_NANO_TEST_MODEL) + add_test( + NAME fun_asr_nano_decoder_probe_cpu + COMMAND fun_asr_nano_decoder_probe + --backend cpu + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_MODEL} + --reference ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/decoder_reference.json + --data ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/decoder_reference.bin + ) + if (GGML_CUDA) + add_test( + NAME fun_asr_nano_decoder_probe_cuda + COMMAND fun_asr_nano_decoder_probe + --backend cuda + --model ${AUDIOCPP_FUN_ASR_NANO_TEST_MODEL} + --reference ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/decoder_reference.json + --data ${CMAKE_CURRENT_SOURCE_DIR}/tests/fun_asr_nano/decoder_reference.bin + ) + endif() + endif() + add_engine_unittest(tdt_decoder_duration_loop_test tests/unittests/test_tdt_decoder_duration_loop.cpp) target_include_directories(tdt_decoder_duration_loop_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) diff --git a/README.md b/README.md index 607db4b9..66a2ba2b 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ Runtime tags: safetensors is the default model loading path. `GGUF 16/Q8/Q4` mea | **chatterbox** | TTS, Clone, VC| ar, da, de, el, en, es, fi, fr, hi, it, ko, ms, nl, no, pl, pt, sv, sw, tr | Chatterbox with 0.5B backbone | GGUF 16/Q8 | | **citrinet_asr** | ASR | en | Citrinet-256 | GGUF Q8 | | **fish_audio** | TTS, Clone, Ctrl | auto, en, zh | Fish Audio S2 Pro | GGUF 16/Q8 | +| **fun_asr_nano** | ASR | auto, zh, en, ja | Fun-ASR-Nano-2512 | GGUF 16/Q8 | | **heartmula** | Music | zh, en, ja, ko, es | HeartMuLa-oss-3B with HeartCodec-oss | GGUF 16/Q8 | | **higgs_audio_stt** | ASR | en | Higgs Audio v3 STT | GGUF 16/Q8, Stream | | **higgs_audio_tts** | TTS, Clone, Ctrl | auto | Higgs Audio v3 TTS 4B | GGUF 16/Q8 | diff --git a/docs/asr.md b/docs/asr.md index f36b255e..cb96498e 100644 --- a/docs/asr.md +++ b/docs/asr.md @@ -2,6 +2,7 @@ | Model | Family | Mode(s) | Quick Start | |---|---|---|---| +| Fun-ASR-Nano | `fun_asr_nano` | offline | [Fun-ASR-Nano](#fun-asr-nano) | | Qwen3 ASR | `qwen3_asr` | offline | [Qwen3 ASR](#qwen3-asr) | | Citrinet ASR | `citrinet_asr` | offline | [Citrinet ASR](#citrinet-asr) | | Kroko Community ASR | `kroko_asr` | offline, streaming | [Kroko Community ASR](#kroko-community-asr) | @@ -21,6 +22,24 @@ audiocpp_cli --task asr --family --model --backend cuda --a When `--mode streaming` is used, the selected model provides its default streaming policy. +## Fun-ASR-Nano + +Fun-ASR-Nano provides offline multilingual transcription for Chinese, English, +and Japanese with automatic language selection. The recommended package is the +standalone Q8_0 GGUF published by FunAudioLLM. + +```bash +python3 tools/model_manager_v2.py install fun_asr_nano +audiocpp_cli --task asr --family fun_asr_nano \ + --model models/Fun-ASR-Nano-2512-GGUF/fun-asr-nano-2512-q8_0.gguf \ + --backend cuda --audio speech_16k.wav --text-out transcript.txt +``` + +The runtime supports fixed offline chunking and inverse text normalization. +Streaming and timestamp output are not exposed. See the +[Fun-ASR-Nano model guide](models/fun_asr_nano.md) for package, option, GGUF, +and server details. + ## Qwen3 ASR Qwen3 ASR transcribes speech and can be paired with Qwen3 Forced Aligner when timestamps are needed. See [Qwen3 models](models/qwen3.md) for the full ASR and alignment manual. diff --git a/docs/gguf.md b/docs/gguf.md index 4adc5078..dc2fd1f9 100644 --- a/docs/gguf.md +++ b/docs/gguf.md @@ -61,6 +61,7 @@ Status labels: | `chatterbox` | Done | Pass | --- | Pass (ASR match, drift) | Pass (ASR match, drift) | | `citrinet_asr` | Done | Pass | --- | --- | Pass | | `fish_audio` | Done | Pass | --- | Pass | Pass | +| `fun_asr_nano` | Done | Pass | --- | Pass | Pass | | `glm_tts` | Done | Pass (TTS + clone) | --- | --- | Pass (ASR match, drift) | | `heartmula` | Done | Pass | --- | Pass (drift) | Pass (drift) | | `higgs_audio_stt` | Done | Pass | --- | Pass | Pass | diff --git a/docs/model_manager.md b/docs/model_manager.md index 6a2c2de5..c4195248 100644 --- a/docs/model_manager.md +++ b/docs/model_manager.md @@ -14,6 +14,7 @@ If a model has a ready-to-use GGUF package, prefer that route first. Ready-to-use GGUF packages are published here: - Core released models: [audio-cpp/audio.cpp-gguf](https://huggingface.co/audio-cpp/audio.cpp-gguf) +- Fun-ASR-Nano: [FunAudioLLM/Fun-ASR-Nano-2512-GGUF](https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512-GGUF) - Community OuteTTS package: [mirek190/audio.cpp](https://huggingface.co/mirek190/audio.cpp/tree/main/Text%20to%20audio%20(TTS)) For support status and tested precision coverage, see the [GGUF guide](gguf.md). @@ -112,6 +113,8 @@ Packages whose loaders are not registered in the current release tree are listed | `chatterbox` | Chatterbox | **Yes** | | `citrinet_asr` | Citrinet ASR converted layout | No | | `fish_audio_s2_pro` | Fish Audio S2 Pro GGUF Q8_0 | **Yes** | +| `fun_asr_nano_2512_f16` | Fun-ASR-Nano-2512 F16 GGUF | **Yes** | +| `fun_asr_nano_2512_q8_0` | Fun-ASR-Nano-2512 Q8_0 GGUF | **Yes** | | `glm_tts` | GLM-TTS with converted Flow/HiFT/tokenizer and CAMPPlus assets | No | | `heartmula` | HeartMuLa | No | | `higgs_audio_stt` | Higgs Audio STT | No | diff --git a/docs/models/fun_asr_nano.md b/docs/models/fun_asr_nano.md new file mode 100644 index 00000000..aefdb8a9 --- /dev/null +++ b/docs/models/fun_asr_nano.md @@ -0,0 +1,153 @@ +# Fun-ASR-Nano + +audio.cpp runs Fun-ASR-Nano-2512 as a native offline ASR model on CPU and CUDA. +The runtime accepts the official Hugging Face safetensors checkpoint and +standalone F16 or Q8_0 GGUF files. + +| Field | Value | +|---|---| +| Family | `fun_asr_nano` | +| Task | `asr` | +| Mode | `offline` | +| Languages | `auto`, `zh`, `en`, `ja` | +| Audio | WAV; converted to mono 16 kHz internally | +| Output | Transcript text | +| Streaming | Not exposed | +| Timestamps | Not exposed | + +## Install + +The model-spec manager installs the Q8_0 standalone GGUF by default: + +```bash +python3 tools/model_manager_v2.py install fun_asr_nano +``` + +Select another published format explicitly: + +```bash +python3 tools/model_manager_v2.py install fun_asr_nano_2512_f16 +python3 tools/model_manager_v2.py install fun_asr_nano_2512_safetensors +``` + +Published GGUF files and their SHA256 manifest are available at +[FunAudioLLM/Fun-ASR-Nano-2512-GGUF](https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512-GGUF). +The package catalog pins an immutable Hub revision. + +## CLI + +```bash +audiocpp_cli \ + --task asr \ + --family fun_asr_nano \ + --model models/Fun-ASR-Nano-2512-GGUF/fun-asr-nano-2512-q8_0.gguf \ + --backend cuda \ + --audio speech.wav \ + --text-out transcript.txt +``` + +The same command accepts the F16 GGUF file or the official safetensors model +directory. + +### Request Options + +| Option | Values | Default | Meaning | +|---|---|---|---| +| `language` | `auto`, `zh`, `en`, `ja` | `auto` | Recognition language hint. | +| `enable_itn` | `true`, `false` | `true` | Enable inverse text normalization in the prompt. | +| `max_tokens` | positive integer | `512` | Maximum transcript tokens generated per chunk. | +| `audio_chunk_mode` | `auto`, `fixed`, `none` | `auto` | Offline audio chunking policy. | +| `audio_chunk_seconds` | positive seconds | `30` | Chunk duration for fixed/automatic chunking. | + +For example: + +```bash +audiocpp_cli --task asr --family fun_asr_nano \ + --model models/Fun-ASR-Nano-2512-GGUF/fun-asr-nano-2512-q8_0.gguf \ + --backend cuda --audio meeting.wav \ + --language zh \ + --request-option enable_itn=true \ + --audio-chunk-mode fixed \ + --audio-chunk-seconds 30 +``` + +### Weight Storage + +`fun_asr_nano.weight_type` sets a shared storage preference. Component overrides are +`fun_asr_nano.encoder_weight_type`, `fun_asr_nano.adaptor_weight_type`, and +`fun_asr_nano.decoder_weight_type`. Supported values are `native`, `f32`, +`f16`, `bf16`, and `q8_0`. + +On CUDA, native, F16, and Q8 shared preferences are promoted to BF16 for the +decoder to keep logits stable. Encoder and adaptor weights retain the shared +type, so a Q8_0 GGUF keeps those large matrix paths quantized. An explicit +`fun_asr_nano.decoder_weight_type` forces that component's storage type. CPU +uses the requested or native stored types without promotion. + +## Server + +`server.json`: + +```json +{ + "host": "127.0.0.1", + "port": 8080, + "backend": "cuda", + "device": 0, + "threads": 4, + "lazy_load": true, + "models": [ + { + "id": "fun-asr-nano", + "family": "fun_asr_nano", + "path": "models/Fun-ASR-Nano-2512-GGUF/fun-asr-nano-2512-q8_0.gguf", + "task": "asr", + "mode": "offline" + } + ] +} +``` + +```bash +audiocpp_server --config server.json + +curl http://127.0.0.1:8080/v1/audio/transcriptions \ + -F model=fun-asr-nano \ + -F language=auto \ + -F file=@speech.wav +``` + +The endpoint follows the OpenAI multipart transcription shape and returns +`{"text":"...","timing":{...}}`. + +## GGUF Conversion + +```bash +audiocpp_gguf \ + --input /path/to/Fun-ASR-Nano-2512-hf/model.safetensors \ + --root /path/to/Fun-ASR-Nano-2512-hf \ + --output fun-asr-nano-2512-q8_0.gguf \ + --type q8_0 \ + --family fun_asr_nano \ + --model-spec model_specs/fun_asr_nano.json +``` + +The standalone output embeds the config, processor config, tokenizer, chat +template, and model package specification. Inspect it with: + +```bash +audiocpp_gguf --inspect fun-asr-nano-2512-q8_0.gguf +``` + +## Validation + +The published F16 and Q8_0 files were checked with the same 14.07-second +reference WAV on CPU and NVIDIA H100 CUDA. CLI and OpenAI-compatible server +transcripts matched the official safetensors path. The Q8_0 server validation +completed at approximately 0.0725 RTF on H100. + +## License + +The checkpoint and converted weights are governed by the FunASR Model Open +Source License Agreement v1.1 distributed with the official model. Review that +agreement before use or redistribution. diff --git a/docs/superpowers/plans/2026-07-29-fun-asr-nano.md b/docs/superpowers/plans/2026-07-29-fun-asr-nano.md new file mode 100644 index 00000000..6eef819d --- /dev/null +++ b/docs/superpowers/plans/2026-07-29-fun-asr-nano.md @@ -0,0 +1,457 @@ +# Fun-ASR-Nano Native Runtime Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add complete offline Fun-ASR-Nano transcription to audio.cpp for official HF safetensors and audio.cpp GGUF on CPU and CUDA. + +**Architecture:** Parse the official Transformers checkpoint into typed assets, run a shared Kaldi-LFR frontend and SAN-M encoder, project through the bidirectional adaptor, and reuse audio.cpp's Qwen causal decoder for generation. Register the completed family through the existing loader, model-spec, CLI, server, and GGUF systems. + +**Tech Stack:** C++20, CMake, GGML, audio.cpp runtime/model-spec APIs, Python reference probes, safetensors, GGUF, Hugging Face Hub. + +## Global Constraints + +- Family id is exactly `fun_asr_nano`; first supported variant is `FunAudioLLM/Fun-ASR-Nano-2512-hf`. +- Support offline ASR only; do not advertise native streaming, timestamps, translation, diarization, or CTC decoding. +- CPU and CUDA are release gates; Metal remains supported through generic GGML operations but is not a first-release gate. +- Official safetensors and audio.cpp-owned F16, Q8_0, and Q4_K_M GGUF use one logical tensor-name layout. +- Preserve MIT attribution for math adapted from transcribe.cpp commit `c87109304c42707867f926fb7b9c378d9f46df8a`. +- Preserve the FunASR Model Open Source License Agreement v1.1 in model package metadata and docs. +- Use TDD for every task and keep `.mcp-tasks/` untracked and untouched. +- Back up the worktree with a git bundle and patch before every commit. + +--- + +## File Map + +- `include/engine/framework/audio/kaldi_fbank.h`, `src/framework/audio/kaldi_fbank.cpp`: parameterized Kaldi fbank and LFR frontend. +- `include/engine/framework/modules/speech_encoders/sanm.h`, `src/framework/modules/speech_encoders/sanm.cpp`: reusable SAN-M graph primitives. +- `include/engine/models/fun_asr_nano/assets.h`, `src/models/fun_asr_nano/assets.cpp`: typed config, resources, and tensor source. +- `include/engine/models/fun_asr_nano/types.h`: frontend, embedding, prompt, generation, and result value types. +- `include/engine/models/fun_asr_nano/frontend.h`, `src/models/fun_asr_nano/frontend.cpp`: fixed Fun-ASR frontend wrapper. +- `include/engine/models/fun_asr_nano/encoder.h`, `src/models/fun_asr_nano/encoder.cpp`: 70-block SAN-M runtime. +- `include/engine/models/fun_asr_nano/adaptor.h`, `src/models/fun_asr_nano/adaptor.cpp`: projector and two-layer bidirectional adaptor. +- `include/engine/models/fun_asr_nano/tokenizer_text.h`, `src/models/fun_asr_nano/tokenizer_text.cpp`: Qwen tokenizer wrapper. +- `include/engine/models/fun_asr_nano/prompt.h`, `src/models/fun_asr_nano/prompt.cpp`: prompt construction and ITN selection. +- `include/engine/models/fun_asr_nano/decoder.h`, `src/models/fun_asr_nano/decoder.cpp`: Qwen prefill/decode runtime. +- `include/engine/models/fun_asr_nano/session.h`, `src/models/fun_asr_nano/session.cpp`: offline task orchestration and timings. +- `include/engine/models/fun_asr_nano/loader.h`, `src/models/fun_asr_nano/loader.cpp`: runtime registration and capabilities. +- `model_specs/fun_asr_nano.json`: packages, sources, capabilities, and options. +- `tests/fun_asr_nano/`: config, frontend, stage parity, warm bench, and reference scripts. +- `docs/models/fun_asr_nano.md`, `docs/asr.md`, `docs/gguf.md`: user-facing install, CLI, server, and GGUF instructions. + +### Task 1: Typed Assets And Model Spec + +**Files:** +- Create: `include/engine/models/fun_asr_nano/assets.h` +- Create: `src/models/fun_asr_nano/assets.cpp` +- Create: `model_specs/fun_asr_nano.json` +- Create: `tests/unittests/test_fun_asr_nano_assets.cpp` +- Modify: `CMakeLists.txt` + +**Interfaces:** +- Produces: `FunAsrNanoConfig`, `FunAsrNanoAssets`, and `load_fun_asr_nano_assets(path)`. +- Consumes: `ResourceBundle`, JSON helpers, `TensorSource`, and the existing model-spec loader. + +- [ ] **Step 1: Write failing config and spec tests** + +Create fixtures with nested current config and legacy flat adaptor fields. Assert the parser yields encoder `(560, 512, 50, 20, 4, 2048, 11)`, adaptor `(1024, 8, 256, 2)`, projector hidden size `2048`, and Qwen `(1024, 3072, 28, 16, 8, 128, 151936)`. + +```cpp +engine::test::require(config.encoder.input_size == 560, "Fun-ASR input size"); +engine::test::require(config.adaptor.ffn_dim == 256, "Fun-ASR adaptor FFN"); +engine::test::require(config.text.rope_theta == 1000000.0F, "Fun-ASR RoPE"); +``` + +- [ ] **Step 2: Run the focused test and confirm it fails** + +Run: `cmake --build build/debug --target test_fun_asr_nano_assets -j 8 && build/debug/bin/test_fun_asr_nano_assets` + +Expected: compilation fails because `FunAsrNanoConfig` is not defined. + +- [ ] **Step 3: Implement strict config parsing and the typed spec** + +Reject dimension mismatches, unsupported activation functions, untied embeddings, absent tokenizer resources, and a tensor source missing the stem or token embedding. Define the HF package and safetensors source in the typed schema. + +- [ ] **Step 4: Run asset and model-spec checks** + +Run: `cmake --build build/debug --target test_fun_asr_nano_assets model_spec_demo -j 8` + +Run: `build/debug/bin/model_spec_demo model_specs/fun_asr_nano.json tests/fixtures/fun_asr_nano_package` + +Expected: both commands pass and report family `fun_asr_nano`. + +- [ ] **Step 5: Back up and commit** + +```bash +git add CMakeLists.txt include/engine/models/fun_asr_nano/assets.h \ + src/models/fun_asr_nano/assets.cpp model_specs/fun_asr_nano.json \ + tests/unittests/test_fun_asr_nano_assets.cpp +git commit -s -m "Add Fun-ASR-Nano assets and model spec" +``` + +### Task 2: Shared Kaldi Fbank And LFR Frontend + +**Files:** +- Create: `include/engine/framework/audio/kaldi_fbank.h` +- Create: `src/framework/audio/kaldi_fbank.cpp` +- Create: `tests/fun_asr_nano/reference_frontend.py` +- Create: `tests/fun_asr_nano/fun_asr_nano_frontend_probe.cpp` +- Modify: `CMakeLists.txt` + +**Interfaces:** +- Produces: `KaldiFbankOptions`, `KaldiFbankFeatures`, and `extract_kaldi_fbank(audio, options)`. +- Output layout: time-major `frames * (num_mels * lfr_m)` float values and a valid-frame count. + +- [ ] **Step 1: Generate pinned reference values** + +The Python probe must use the Transformers `FunAsrNanoFeatureExtractor` from PR #46180 and print JSON summaries plus the first and last 32 values for silence, impulse, 440 Hz sine, and `assets/resources/sample_16k.wav`. + +- [ ] **Step 2: Write the C++ probe assertions** + +Assert frame counts, output dimension `560`, finite values, and per-fixture numerical parity at `atol=2e-4`, `rtol=2e-4`. + +- [ ] **Step 3: Run and confirm the frontend probe fails** + +Run: `cmake --build build/debug --target fun_asr_nano_frontend_probe -j 8` + +Expected: compilation fails because `extract_kaldi_fbank` is not defined. + +- [ ] **Step 4: Implement the frontend** + +Implement pre-emphasis `0.97`, 25 ms Hamming frames, 10 ms shift, 80 HTK mel bins, natural log, centered LFR `7/6`, replicated edge frames, and optional CMVN. Keep Fun-ASR's CMVN flag off. + +- [ ] **Step 5: Run parity and existing audio tests** + +Run: `build/debug/bin/fun_asr_nano_frontend_probe tests/fun_asr_nano/frontend_reference.json` + +Run: `ctest --test-dir build/debug -R "audio|frontend" --output-on-failure` + +Expected: all pass. + +- [ ] **Step 6: Back up and commit** + +```bash +git add CMakeLists.txt include/engine/framework/audio/kaldi_fbank.h \ + src/framework/audio/kaldi_fbank.cpp tests/fun_asr_nano +git commit -s -m "Add Kaldi LFR audio frontend" +``` + +### Task 3: Shared SAN-M Graph Primitives + +**Files:** +- Create: `include/engine/framework/modules/speech_encoders/sanm.h` +- Create: `src/framework/modules/speech_encoders/sanm.cpp` +- Create: `tests/fun_asr_nano/sanm_probe.cpp` +- Create: `tests/fun_asr_nano/reference_sanm.py` +- Modify: `CMakeLists.txt` + +**Interfaces:** +- Produces: `SanmBlockWeightsView`, `SanmBlockConfig`, `sanm_projection_block`, `sanm_residual_block`, and `sanm_layer_norm`. +- Consumes: GGML graph context and non-owning weight tensors. + +- [ ] **Step 1: Write one-block reference and failing probe** + +Use deterministic small tensors with `d_input=12`, `d_model=8`, two heads, FFN 16, and kernel 3. Compare projection and residual variants separately. + +- [ ] **Step 2: Run the probe and confirm missing symbols** + +Run: `cmake --build build/debug --target fun_asr_nano_sanm_probe -j 8` + +Expected: link or compilation failure for the SAN-M API. + +- [ ] **Step 3: Implement SAN-M math** + +Implement Q/K/V projection, scaled dot-product attention, depthwise FSMN over V, residual addition, affine layer norm, and ReLU FFN. Request F32 accumulation for F16 matrix multiplication. + +- [ ] **Step 4: Run CPU and CUDA probe variants** + +Run: `build/debug/bin/fun_asr_nano_sanm_probe --backend cpu` + +Run: `build/debug/bin/fun_asr_nano_sanm_probe --backend cuda` + +Expected: both pass their configured tolerances. + +- [ ] **Step 5: Back up and commit** + +```bash +git add CMakeLists.txt include/engine/framework/modules/speech_encoders/sanm.h \ + src/framework/modules/speech_encoders/sanm.cpp tests/fun_asr_nano +git commit -s -m "Add reusable SAN-M encoder blocks" +``` + +### Task 4: Fun-ASR Encoder Runtime + +**Files:** +- Create: `include/engine/models/fun_asr_nano/types.h` +- Create: `include/engine/models/fun_asr_nano/frontend.h` +- Create: `src/models/fun_asr_nano/frontend.cpp` +- Create: `include/engine/models/fun_asr_nano/encoder.h` +- Create: `src/models/fun_asr_nano/encoder.cpp` +- Create: `tests/fun_asr_nano/fun_asr_nano_encoder_probe.cpp` +- Create: `tests/fun_asr_nano/reference_encoder.py` +- Modify: `CMakeLists.txt` + +**Interfaces:** +- Produces: `FunAsrNanoAudioFeatures` and `FunAsrNanoEncoderRuntime::encode(features)` returning `[frames, 512]`. +- Consumes: Tasks 1-3. + +- [ ] **Step 1: Write tensor catalog and stage-parity failures** + +Assert required tensor names and shapes. Capture stem, main blocks 0/24/48, timestamp blocks 0/10/19, and final normalization from the HF model. + +- [ ] **Step 2: Run the probe and confirm the runtime is absent** + +Run: `cmake --build build/debug --target fun_asr_nano_encoder_probe -j 8` + +Expected: compilation fails for `FunAsrNanoEncoderRuntime`. + +- [ ] **Step 3: Implement weight loading and shape-keyed graph caching** + +Load split Q/K/V tensors from the HF namespaces. Build `sqrt(512)` scaling, sinusoidal positions, one 560 -> 512 stem, 49 main blocks, normalization, 20 timestamp blocks, and final normalization. + +- [ ] **Step 4: Verify stage parity on CPU and CUDA** + +Run: `build/debug/bin/fun_asr_nano_encoder_probe --backend cpu --reference tests/fun_asr_nano/encoder_reference` + +Run: `build/debug/bin/fun_asr_nano_encoder_probe --backend cuda --reference tests/fun_asr_nano/encoder_reference` + +Expected: every captured stage passes `atol=2e-3`, `rtol=2e-3`. + +- [ ] **Step 5: Back up and commit** + +```bash +git add CMakeLists.txt include/engine/models/fun_asr_nano \ + src/models/fun_asr_nano tests/fun_asr_nano +git commit -s -m "Implement Fun-ASR-Nano SAN-M encoder" +``` + +### Task 5: Projector And Bidirectional Adaptor + +**Files:** +- Create: `include/engine/models/fun_asr_nano/adaptor.h` +- Create: `src/models/fun_asr_nano/adaptor.cpp` +- Create: `tests/fun_asr_nano/fun_asr_nano_adaptor_probe.cpp` +- Create: `tests/fun_asr_nano/reference_adaptor.py` +- Modify: `CMakeLists.txt` + +**Interfaces:** +- Produces: `FunAsrNanoAdaptorRuntime::adapt(encoder_embeddings, mask)` returning packed valid `[tokens, 1024]` audio embeddings. +- Consumes: `FunAsrNanoEncoderEmbeddings` and Task 1 config. + +- [ ] **Step 1: Write projector and block parity tests** + +Capture `linear_1`, `linear_2`, adaptor block 0, adaptor block 1, and packed valid embeddings from Transformers. + +- [ ] **Step 2: Confirm the probe fails before implementation** + +Run: `cmake --build build/debug --target fun_asr_nano_adaptor_probe -j 8` + +- [ ] **Step 3: Implement projector, mask, and two pre-norm blocks** + +Use 512 -> 2048 -> 1024 projector linears with ReLU, eight-head bidirectional attention, 256-wide ReLU FFN, residuals, and padding removal. + +- [ ] **Step 4: Run CPU/CUDA parity and padding cases** + +Expected: stage tensors pass `atol=2e-3`, `rtol=2e-3`; padded batches match individual inference. + +- [ ] **Step 5: Back up and commit** + +```bash +git add CMakeLists.txt include/engine/models/fun_asr_nano/adaptor.h \ + src/models/fun_asr_nano/adaptor.cpp tests/fun_asr_nano +git commit -s -m "Implement Fun-ASR-Nano audio adaptor" +``` + +### Task 6: Tokenizer, Prompt, And Qwen Decoder + +**Files:** +- Create: `include/engine/models/fun_asr_nano/tokenizer_text.h` +- Create: `src/models/fun_asr_nano/tokenizer_text.cpp` +- Create: `include/engine/models/fun_asr_nano/prompt.h` +- Create: `src/models/fun_asr_nano/prompt.cpp` +- Create: `include/engine/models/fun_asr_nano/decoder.h` +- Create: `src/models/fun_asr_nano/decoder.cpp` +- Create: `tests/fun_asr_nano/fun_asr_nano_decoder_probe.cpp` +- Create: `tests/fun_asr_nano/reference_decoder.py` +- Modify: `CMakeLists.txt` + +**Interfaces:** +- Produces: `FunAsrNanoPromptBuilder::build(request, audio_tokens)` and `FunAsrNanoDecoderRuntime::generate(prompt, audio_embeddings, options)`. +- Consumes: `HfTokenizerJson`, `QwenCausalDecoderModule`, generic HF sampler, and Task 5 embeddings. + +- [ ] **Step 1: Write prompt token and logit parity tests** + +Assert the exact system/user/assistant token sequence, audio token positions, ITN prompt variant, prefill logits, and first three greedy decode steps. + +- [ ] **Step 2: Run and confirm missing decoder APIs** + +Run: `cmake --build build/debug --target fun_asr_nano_decoder_probe -j 8` + +- [ ] **Step 3: Implement tokenizer and prompt splice** + +Resolve `<|im_start|>`, `<|im_end|>`, and audio token id `151646` from tokenizer assets. Insert one placeholder per valid audio embedding. + +- [ ] **Step 4: Wrap the existing Qwen causal decoder** + +Configure hidden size 1024, 28 layers, 16 query heads, 8 KV heads, head dim 128, FFN 3072, RoPE theta `1e6`, and tied embeddings. Do not duplicate Qwen layer math. + +- [ ] **Step 5: Run token, prefill, decode, and KV-cache parity** + +Expected: prompt ids are exact and logits pass `atol=3e-3`, `rtol=3e-3`. + +- [ ] **Step 6: Back up and commit** + +```bash +git add CMakeLists.txt include/engine/models/fun_asr_nano \ + src/models/fun_asr_nano tests/fun_asr_nano +git commit -s -m "Add Fun-ASR-Nano Qwen decoding" +``` + +### Task 7: Offline Session And Loader Registration + +**Files:** +- Create: `include/engine/models/fun_asr_nano/session.h` +- Create: `src/models/fun_asr_nano/session.cpp` +- Create: `include/engine/models/fun_asr_nano/loader.h` +- Create: `src/models/fun_asr_nano/loader.cpp` +- Create: `tests/fun_asr_nano/fun_asr_nano_warm_bench.cpp` +- Create: `tests/fun_asr_nano/fun_asr_nano_warm_bench_cases.json` +- Modify: `CMakeLists.txt` + +**Interfaces:** +- Produces: `make_fun_asr_nano_loader()` and an `IOfflineVoiceTaskSession` for task `asr`. +- Consumes: Tasks 1-6 and framework audio chunking. + +- [ ] **Step 1: Write failing loader, option, and transcript tests** + +Assert family/task/mode, rejection of streaming and timestamps, `fun_asr_nano.itn`, unknown-option errors, and deterministic transcripts for zh/en/ja fixtures. + +- [ ] **Step 2: Confirm the family is unavailable** + +Run: `build/debug/bin/audiocpp_cli --inspect --family fun_asr_nano --model tests/fixtures/fun_asr_nano_package` + +Expected: loader registry reports unknown family. + +- [ ] **Step 3: Implement session orchestration and timings** + +Execute resample -> frontend -> encoder -> adaptor -> prompt -> decoder -> decode. Emit `fun_asr_nano.frontend_ms`, `encoder_ms`, `adaptor_ms`, `decoder_ms`, token count, audio frame count, and `session.wall_ms`. + +- [ ] **Step 4: Register only the complete family** + +Add an `audiocpp_add_model(fun_asr_nano ...)` entry with loader include and `make_fun_asr_nano_loader`. + +- [ ] **Step 5: Run CLI and server smoke tests** + +Run: `build/debug/bin/audiocpp_cli --task asr --family fun_asr_nano --model "$FUN_ASR_NANO_MODEL" --audio assets/resources/sample_16k.wav --text-out /tmp/fun_asr.txt` + +Run the existing OpenAI transcription route test with family `fun_asr_nano` and assert the JSON transcript equals the CLI result. + +- [ ] **Step 6: Back up and commit** + +```bash +git add CMakeLists.txt include/engine/models/fun_asr_nano \ + src/models/fun_asr_nano tests/fun_asr_nano +git commit -s -m "Register Fun-ASR-Nano offline ASR" +``` + +### Task 8: GGUF Conversion And Package Validation + +**Files:** +- Modify: `model_specs/fun_asr_nano.json` +- Create: `tests/fun_asr_nano/test_gguf_roundtrip.cpp` +- Create: `tests/fun_asr_nano/compare_transcripts.py` +- Modify: `tools/model_manager.py` +- Modify: `docs/gguf.md` + +**Interfaces:** +- Produces: installable F16, Q8_0, and Q4_K_M packages with embedded sidecars and spec. +- Consumes: generic `audiocpp_gguf` and the safetensors loader. + +- [ ] **Step 1: Write a failing round-trip test** + +Convert a tiny fixture, reopen it as a standalone GGUF, and assert exact config, tokenizer sidecars, tensor logical names, and family selection. + +- [ ] **Step 2: Run the generic converter on the official checkpoint** + +```bash +build/release/bin/audiocpp_gguf --family fun_asr_nano \ + --model "$FUN_ASR_NANO_MODEL" --type f16 --output fun-asr-nano-f16.gguf +``` + +Repeat with `q8_0` and `q4_k_m` after F16 parity passes. + +- [ ] **Step 3: Validate transcript and logit drift** + +Compare safetensors, F16, Q8_0, and Q4_K_M on the zh/en/ja fixture matrix. Require exact greedy transcripts for F16 and Q8_0; record any Q4_K_M text differences. + +- [ ] **Step 4: Add package records after artifacts exist** + +Use `audio-cpp/audio.cpp-gguf`, exact filenames, SHA-backed Hub revisions, model license metadata, and Q8_0 as the default package. + +- [ ] **Step 5: Back up and commit** + +```bash +git add model_specs/fun_asr_nano.json tools/model_manager.py \ + tests/fun_asr_nano docs/gguf.md +git commit -s -m "Add Fun-ASR-Nano GGUF packages" +``` + +### Task 9: Documentation, Full Verification, And Draft PR + +**Files:** +- Create: `docs/models/fun_asr_nano.md` +- Modify: `docs/asr.md` +- Modify: `docs/model_manager.md` +- Modify: `README.md` +- Modify: `tools/audiocpp_cli/audiocpp_cli_path_cases.json` + +**Interfaces:** +- Produces: complete install, CLI, server, GGUF, limitations, licensing, and validation documentation. + +- [ ] **Step 1: Add executable CLI path coverage** + +Add `fun_asr_nano_offline` with `task=asr`, `mode=offline`, `family=fun_asr_nano`, a 16 kHz fixture, text output, and expected non-empty transcript. + +- [ ] **Step 2: Document user workflows and limits** + +Include model-manager install, safetensors and GGUF commands, OpenAI-compatible transcription endpoint, ITN option, CPU/CUDA selection, 30-second chunking, language coverage, and unsupported timestamps/streaming. + +- [ ] **Step 3: Run repository verification** + +```bash +cmake -S . -B build/release -DCMAKE_BUILD_TYPE=Release +cmake --build build/release --target audiocpp_cli audiocpp_server audiocpp_gguf \ + fun_asr_nano_frontend_probe fun_asr_nano_encoder_probe \ + fun_asr_nano_adaptor_probe fun_asr_nano_decoder_probe \ + fun_asr_nano_warm_bench -j 8 +ctest --test-dir build/release --output-on-failure +python3 tools/check_loader_catalog_sync.py +python3 tools/audiocpp_cli/run_audiocpp_cli_path_tests.py \ + --binary build/release/bin/audiocpp_cli --case fun_asr_nano_offline +``` + +Expected: every command passes. + +- [ ] **Step 4: Run release parity matrix** + +Run zh/en/ja transcripts on CPU and CUDA for safetensors, F16, Q8_0, and Q4_K_M. Record timing, peak memory, transcript, and known limitations in `docs/reports/fun_asr_nano_validation.md`. + +- [ ] **Step 5: Self-review scope and attribution** + +Confirm no incomplete CTC/timestamp/streaming claims, no unrelated refactor, transcribe.cpp MIT attribution is present, and model license text is linked from every package. + +- [ ] **Step 6: Back up, commit, push, and open a draft PR** + +```bash +git add README.md docs tools tests +git commit -s -m "Document and validate Fun-ASR-Nano support" +git push -u origin codex/add-fun-asr-nano-audiocpp-20260729 +``` + +Open a draft PR linked to modelscope/FunASR#3439 with exact local verification results and remaining release gates. Mark ready only after CPU/CUDA and GGUF parity are green. + +## Self-Review + +- Spec coverage: assets, official HF loading, frontend, SAN-M, adaptor, Qwen decoding, session, CLI/server, GGUF, CPU/CUDA, parity, docs, licensing, and rollback each map to a task. +- Placeholder scan: all steps name concrete files, interfaces, commands, and expected outcomes. +- Type consistency: frontend emits `FunAsrNanoAudioFeatures`; encoder emits 512-wide embeddings; adaptor emits packed 1024-wide embeddings; decoder consumes those embeddings and returns token ids; session returns a framework transcript. diff --git a/docs/superpowers/specs/2026-07-29-fun-asr-nano-design.md b/docs/superpowers/specs/2026-07-29-fun-asr-nano-design.md new file mode 100644 index 00000000..a95014f5 --- /dev/null +++ b/docs/superpowers/specs/2026-07-29-fun-asr-nano-design.md @@ -0,0 +1,176 @@ +# Fun-ASR-Nano Native Runtime Design + +## Goal + +Add production-grade offline transcription for `FunAudioLLM/Fun-ASR-Nano-2512` +to audio.cpp under the family id `fun_asr_nano`. The first complete release must +load the official Hugging Face safetensors checkpoint, convert it with the normal +audio.cpp GGUF path, run on CPU and CUDA, and serve through the existing CLI and +OpenAI-compatible server. + +## Scope + +The first release includes: + +- 16 kHz mono audio input with Kaldi-compatible 80-bin fbank extraction; +- pre-emphasis `0.97`, 25 ms frames, 10 ms shift, and LFR `m=7`, `n=6`; +- the 70-block SenseVoice SAN-M encoder (1 projection stem, 49 main blocks, + 20 timestamp-prediction blocks); +- the 512 -> 2048 -> 1024 projector and two-layer bidirectional adaptor; +- the bundled Qwen3-0.6B causal decoder with KV cache; +- Chinese, English, and Japanese transcription plus the model's Chinese + dialect and regional-accent coverage; +- prompt-controlled inverse text normalization through `fun_asr_nano.itn`; +- native safetensors and audio.cpp-owned F16, Q8_0, and Q4_K_M GGUF packages; +- deterministic greedy decoding, timing metrics, and explicit input limits; +- reference parity against Transformers and the MIT-licensed transcribe.cpp + implementation pinned at `c87109304c42707867f926fb7b9c378d9f46df8a`. + +Streaming, built-in timestamps, translation, diarization, and the unpublished +CTC path are outside the first release. The published checkpoint contains no +complete CTC decoder, so the runtime is intentionally LLM-only. + +## Source Of Truth + +The implementation uses these pinned references: + +- official HF checkpoint: `FunAudioLLM/Fun-ASR-Nano-2512-hf` at + `854d88f94205cd17d2afdb24332130d86fbe654a`; +- Transformers integration PR #46180 and its `fun_asr_nano` config, frontend, + model, and conversion mapping; +- transcribe.cpp at `c87109304c42707867f926fb7b9c378d9f46df8a` + for independently validated SAN-M, adaptor, and frontend math; +- transcribe.cpp BF16 result of 1.78% WER on LibriSpeech test-clean as an + external acceptance reference, not as a replacement for audio.cpp parity. + +Adapted MIT code must retain attribution in the affected source files and in +the repository's third-party notices. Model packages retain the FunASR Model +Open Source License Agreement v1.1. + +## Architecture + +### Assets And Configuration + +`FunAsrNanoAssets` owns a `ResourceBundle`, parsed configuration, tokenizer +assets, and one tensor source. The parser accepts the current nested +Transformers config and the legacy flat adaptor fields already published by +the HF checkpoint. Validation rejects mismatched dimensions before allocating +weights. + +The runtime reads the converted HF tensor namespaces directly: + +- `model.audio_tower.stem`; +- `model.audio_tower.layers`; +- `model.audio_tower.timestamp_prediction_layers`; +- `model.audio_tower.layer_norm` and + `model.audio_tower.timestamp_prediction_layer_norm`; +- `model.multi_modal_projector` and `model.audio_adaptor.blocks`; +- `model.language_model`, `lm_head`, and tied token embeddings. + +The audio.cpp GGUF converter preserves these logical names and embeds config, +processor config, generation config, tokenizer JSON, chat template, and the +typed `fun_asr_nano` model spec. This keeps safetensors and GGUF on one loader +path. + +### Shared Frontend + +A parameterized Kaldi fbank frontend belongs in `engine/framework/audio`, not +inside the model. It supports the exact options needed by Fun-ASR-Nano and can +later replace duplicated SenseVoice-style frontend code: + +- pre-emphasis and Hamming window; +- HTK mel scale and natural-log output; +- configurable snip-edges behavior; +- centered LFR stacking with first/last frame replication; +- optional CMVN, disabled for Fun-ASR-Nano. + +The model wrapper fixes the published values and returns time-major LFR frames, +an attention mask, and valid lengths. + +### Shared SAN-M Module + +The SAN-M graph primitives belong in +`engine/framework/modules/speech_encoders/sanm`. A non-owning weight view feeds +three operations: + +- the 560 -> 512 projection stem; +- a residual SAN-M block; +- affine layer normalization. + +Each block computes projected multi-head self-attention, the depthwise FSMN +branch over value states with kernel size 11, a residual connection, and the +ReLU feed-forward branch. F16 matrix multiplication requests F32 accumulation +where the validated reference requires it. + +### Encoder And Adaptor + +`FunAsrNanoEncoderRuntime` owns typed weight storage and a shape-keyed graph +cache. It applies `sqrt(512)` scaling, sinusoidal positions, the stem, 49 main +blocks, normalization, 20 timestamp-prediction blocks, and final normalization. + +`FunAsrNanoAdaptorRuntime` applies the two projector linears and two pre-norm +bidirectional Transformer blocks. The adaptor preserves one output embedding +per valid LFR frame; padded frames are masked and removed before prompt splice. + +### Decoder And Prompt + +The text path reuses audio.cpp's `QwenCausalDecoderModule`, Qwen3 tokenizer, +sampling code, and KV-cache conventions. The Fun-ASR wrapper supplies the +1024-hidden, 28-layer, 16-query/8-KV-head configuration with RoPE theta +`1e6` and tied embeddings. + +The prompt builder emits the checkpoint chat template, inserts one audio token +per adaptor output, and supports the model's two Chinese normalization prompts. +Greedy decoding is the default. Existing request sampling fields remain +available without creating model-specific duplicates. + +### Session And Serving + +`FunAsrNanoSession` implements `IOfflineVoiceTaskSession`. It validates task and +run mode, resamples input to 16 kHz, executes frontend -> encoder -> adaptor -> +decoder -> tokenizer, and publishes per-stage timings. It uses the framework's +existing audio chunking for long files, with 30-second default chunks and no +claim of model-native streaming. + +Registration through `audiocpp_add_model` automatically exposes the family to +`audiocpp_cli`, `audiocpp_server`, loader inspection, and generic GGUF tooling. + +## Model Spec And Packages + +`model_specs/fun_asr_nano.json` is the source of truth. It advertises: + +- task `asr`, mode `offline`, languages `zh`, `en`, and `ja`; +- official HF safetensors package from + `FunAudioLLM/Fun-ASR-Nano-2512-hf`; +- audio.cpp-hosted F16, Q8_0, and Q4_K_M packages after parity validation; +- request option `language` and session option `itn`; +- explicit lack of timestamps and streaming. + +The package license and attribution are displayed in the model documentation +and release metadata. + +## Verification + +Tests proceed in increasing cost: + +1. config, tensor-name, tokenizer, and model-spec unit tests; +2. frontend parity on fixed waveforms, including short and padded audio; +3. encoder stage parity at stem, main block 0/24/48, and timestamp block + 0/10/19; +4. adaptor parity after each projector and adaptor block; +5. decoder prefill and first three decode-step logits parity; +6. end-to-end transcript parity on Chinese, English, and Japanese fixtures; +7. CPU and CUDA smoke tests for safetensors and GGUF; +8. WER evaluation on LibriSpeech test-clean before publishing quantized files. + +F16/BF16 stage tensors target `atol=2e-3`, `rtol=2e-3`; end-to-end greedy +transcripts must match the HF reference on the fixture set. Quantized packages +must remain within 0.20 absolute WER percentage points of the F16 audio.cpp +baseline. + +## Rollback + +Work stays on `codex/add-fun-asr-nano-audiocpp-20260729` with a bundle and +patch backup before each commit. Each commit is independently buildable. The +model is registered only in the final end-to-end commit, so earlier commits do +not expose an incomplete family to users. diff --git a/include/engine/framework/audio/kaldi_fbank.h b/include/engine/framework/audio/kaldi_fbank.h new file mode 100644 index 00000000..bbcc0912 --- /dev/null +++ b/include/engine/framework/audio/kaldi_fbank.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +namespace engine::audio { + +struct KaldiFbankOptions { + int sample_rate = 16000; + int num_mels = 80; + float frame_length_ms = 25.0F; + float frame_shift_ms = 10.0F; + int lfr_m = 7; + int lfr_n = 6; + float preemphasis = 0.97F; + float low_frequency = 20.0F; + float high_frequency = 0.0F; + bool remove_dc_offset = true; + bool upscale_samples = false; + bool apply_cmvn = false; + std::vector cmvn_shift; + std::vector cmvn_scale; +}; + +struct KaldiFbankFeatures { + int frames = 0; + int feature_dim = 0; + std::vector values; +}; + +// Extracts snip-edges Kaldi log-mel features followed by centered LFR stacking. +KaldiFbankFeatures extract_kaldi_fbank(const std::vector &audio, + const KaldiFbankOptions &options = {}); + +} // namespace engine::audio diff --git a/include/engine/framework/modules/speech_encoders/sanm.h b/include/engine/framework/modules/speech_encoders/sanm.h new file mode 100644 index 00000000..64a1f522 --- /dev/null +++ b/include/engine/framework/modules/speech_encoders/sanm.h @@ -0,0 +1,49 @@ +#pragma once + +#include "engine/framework/core/module.h" +#include "engine/framework/modules/attention/scaled_dot_product_attention.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" + +#include + +namespace engine::modules { + +struct SanmBlockWeightsView { + NormWeights self_attention_norm; + LinearWeights query_projection; + LinearWeights key_projection; + LinearWeights value_projection; + LinearWeights attention_output_projection; + core::TensorValue fsmn_weight; + NormWeights final_norm; + LinearWeights ffn_input_projection; + LinearWeights ffn_output_projection; +}; + +struct SanmBlockConfig { + int64_t input_size = 0; + int64_t model_size = 0; + int64_t num_heads = 0; + int64_t ffn_size = 0; + int64_t fsmn_kernel_size = 0; + float layer_norm_eps = 1.0e-5F; + ScaledDotProductAttentionLowering attention_lowering = + ScaledDotProductAttentionLowering::Explicit; +}; + +core::TensorValue sanm_layer_norm(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const NormWeights &weights, float epsilon); + +core::TensorValue sanm_projection_block(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const SanmBlockWeightsView &weights, + const SanmBlockConfig &config); + +core::TensorValue sanm_residual_block(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const SanmBlockWeightsView &weights, + const SanmBlockConfig &config); + +} // namespace engine::modules diff --git a/include/engine/models/fun_asr_nano/adaptor.h b/include/engine/models/fun_asr_nano/adaptor.h new file mode 100644 index 00000000..06955342 --- /dev/null +++ b/include/engine/models/fun_asr_nano/adaptor.h @@ -0,0 +1,40 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/models/fun_asr_nano/assets.h" +#include "engine/models/fun_asr_nano/types.h" + +#include +#include +#include +#include + +namespace engine::models::fun_asr_nano { + +class FunAsrNanoAdaptorRuntime { +public: + FunAsrNanoAdaptorRuntime(std::shared_ptr assets, + engine::core::ExecutionContext &execution_context, + size_t graph_arena_bytes, + engine::assets::TensorStorageType weight_storage = + engine::assets::TensorStorageType::F32); + ~FunAsrNanoAdaptorRuntime(); + + FunAsrNanoAdaptorRuntime(const FunAsrNanoAdaptorRuntime &) = delete; + FunAsrNanoAdaptorRuntime & + operator=(const FunAsrNanoAdaptorRuntime &) = delete; + FunAsrNanoAdaptorRuntime(FunAsrNanoAdaptorRuntime &&) noexcept; + FunAsrNanoAdaptorRuntime &operator=(FunAsrNanoAdaptorRuntime &&) noexcept; + + void prepare_capacity(int64_t valid_frames); + FunAsrNanoAdaptorEmbeddings + adapt(const FunAsrNanoEncoderEmbeddings &encoder_embeddings, + const std::vector &mask, bool capture_stages = false); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/assets.h b/include/engine/models/fun_asr_nano/assets.h new file mode 100644 index 00000000..fe98bd7e --- /dev/null +++ b/include/engine/models/fun_asr_nano/assets.h @@ -0,0 +1,88 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" + +#include +#include +#include +#include +#include + +namespace engine::assets { +class TensorSource; +} + +namespace engine::models::fun_asr_nano { + +struct FunAsrNanoFrontendConfig { + int sample_rate = 16000; + int64_t feature_size = 80; + int64_t frame_length_ms = 25; + int64_t frame_shift_ms = 10; + int64_t lfr_m = 7; + int64_t lfr_n = 6; + float preemphasis = 0.97F; +}; + +struct FunAsrNanoEncoderConfig { + int64_t num_mel_bins = 80; + int64_t num_stacked_frames = 7; + int64_t input_size = 560; + int64_t d_model = 512; + int64_t attention_heads = 4; + int64_t ffn_dim = 2048; + int64_t layers = 50; + int64_t timestamp_prediction_layers = 20; + int64_t kernel_size = 11; + int64_t max_position_embeddings = 2049; + std::string activation = "relu"; +}; + +struct FunAsrNanoAdaptorConfig { + int64_t d_model = 1024; + int64_t attention_heads = 8; + int64_t ffn_dim = 256; + int64_t layers = 2; + std::string activation = "relu"; +}; + +struct FunAsrNanoTextConfig { + int64_t vocab_size = 151936; + int64_t hidden_size = 1024; + int64_t intermediate_size = 3072; + int64_t layers = 28; + int64_t attention_heads = 16; + int64_t key_value_heads = 8; + int64_t head_dim = 128; + int64_t max_position_embeddings = 40960; + int64_t bos_token_id = 151643; + int64_t eos_token_id = 151645; + float rms_norm_eps = 1.0e-6F; + float rope_theta = 1000000.0F; + bool tie_word_embeddings = true; +}; + +struct FunAsrNanoConfig { + std::string model_type; + int64_t audio_token_id = 151646; + int64_t projector_hidden_size = 2048; + bool tie_word_embeddings = true; + FunAsrNanoFrontendConfig frontend; + FunAsrNanoEncoderConfig encoder; + FunAsrNanoAdaptorConfig adaptor; + FunAsrNanoTextConfig text; + std::vector supported_languages = {"zh", "en", "ja"}; +}; + +struct FunAsrNanoAssets { + assets::ResourceBundle resources; + FunAsrNanoConfig config; + std::shared_ptr model_weights; +}; + +std::shared_ptr +load_fun_asr_nano_assets(const std::filesystem::path &model_path); +std::shared_ptr +load_fun_asr_nano_assets(assets::ResourceBundle resources); + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/decoder.h b/include/engine/models/fun_asr_nano/decoder.h new file mode 100644 index 00000000..36e9f43b --- /dev/null +++ b/include/engine/models/fun_asr_nano/decoder.h @@ -0,0 +1,39 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/models/fun_asr_nano/assets.h" +#include "engine/models/fun_asr_nano/types.h" + +#include +#include +#include + +namespace engine::models::fun_asr_nano { + +using FunAsrNanoTokenCallback = + std::function; + +class FunAsrNanoDecoderRuntime { +public: + struct Impl; + + FunAsrNanoDecoderRuntime(std::shared_ptr assets, + core::ExecutionContext &execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + assets::TensorStorageType weight_storage_type); + ~FunAsrNanoDecoderRuntime(); + + FunAsrNanoGeneratedTokens + generate(const FunAsrNanoPrompt &prompt, + const FunAsrNanoAdaptorEmbeddings &audio_embeddings, + const FunAsrNanoGenerationOptions &options, + const FunAsrNanoTokenCallback &token_callback = {}); + +private: + std::unique_ptr impl_; +}; + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/encoder.h b/include/engine/models/fun_asr_nano/encoder.h new file mode 100644 index 00000000..bba833fa --- /dev/null +++ b/include/engine/models/fun_asr_nano/encoder.h @@ -0,0 +1,38 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/models/fun_asr_nano/assets.h" +#include "engine/models/fun_asr_nano/types.h" + +#include +#include +#include + +namespace engine::models::fun_asr_nano { + +class FunAsrNanoEncoderRuntime { +public: + FunAsrNanoEncoderRuntime(std::shared_ptr assets, + engine::core::ExecutionContext &execution_context, + size_t graph_arena_bytes, + engine::assets::TensorStorageType weight_storage = + engine::assets::TensorStorageType::F32); + ~FunAsrNanoEncoderRuntime(); + + FunAsrNanoEncoderRuntime(const FunAsrNanoEncoderRuntime &) = delete; + FunAsrNanoEncoderRuntime & + operator=(const FunAsrNanoEncoderRuntime &) = delete; + FunAsrNanoEncoderRuntime(FunAsrNanoEncoderRuntime &&) noexcept; + FunAsrNanoEncoderRuntime &operator=(FunAsrNanoEncoderRuntime &&) noexcept; + + void prepare_capacity(int64_t frames); + FunAsrNanoEncoderEmbeddings encode(const FunAsrNanoAudioFeatures &features, + bool capture_stages = false); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/frontend.h b/include/engine/models/fun_asr_nano/frontend.h new file mode 100644 index 00000000..1864cfd4 --- /dev/null +++ b/include/engine/models/fun_asr_nano/frontend.h @@ -0,0 +1,21 @@ +#pragma once + +#include "engine/models/fun_asr_nano/assets.h" +#include "engine/models/fun_asr_nano/types.h" + +#include + +namespace engine::models::fun_asr_nano { + +class FunAsrNanoFrontend { +public: + explicit FunAsrNanoFrontend(FunAsrNanoFrontendConfig config); + + FunAsrNanoAudioFeatures extract(const std::vector &audio, + int sample_rate) const; + +private: + FunAsrNanoFrontendConfig config_; +}; + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/loader.h b/include/engine/models/fun_asr_nano/loader.h new file mode 100644 index 00000000..791b86cd --- /dev/null +++ b/include/engine/models/fun_asr_nano/loader.h @@ -0,0 +1,32 @@ +#pragma once + +#include "engine/framework/runtime/model.h" +#include "engine/models/fun_asr_nano/assets.h" + +#include + +namespace engine::models::fun_asr_nano { + +class FunAsrNanoLoadedModel final : public runtime::ILoadedVoiceModel { +public: + FunAsrNanoLoadedModel(runtime::ModelMetadata metadata, + runtime::CapabilitySet capabilities, + std::shared_ptr assets); + + const runtime::ModelMetadata &metadata() const noexcept override; + const runtime::CapabilitySet &capabilities() const noexcept override; + std::unique_ptr + create_task_session(const runtime::TaskSpec &task, + const runtime::SessionOptions &options) const override; + +private: + runtime::ModelMetadata metadata_; + runtime::CapabilitySet capabilities_; + std::shared_ptr assets_; +}; + +std::unique_ptr +load_fun_asr_nano_model(const std::filesystem::path &model_path); +std::shared_ptr make_fun_asr_nano_loader(); + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/prompt.h b/include/engine/models/fun_asr_nano/prompt.h new file mode 100644 index 00000000..da5958ec --- /dev/null +++ b/include/engine/models/fun_asr_nano/prompt.h @@ -0,0 +1,26 @@ +#pragma once + +#include "engine/models/fun_asr_nano/tokenizer_text.h" +#include "engine/models/fun_asr_nano/types.h" + +#include +#include +#include + +namespace engine::models::fun_asr_nano { + +class FunAsrNanoPromptBuilder { +public: + explicit FunAsrNanoPromptBuilder( + std::shared_ptr assets); + + FunAsrNanoPrompt build(const FunAsrNanoPromptRequest &request, + int64_t audio_tokens) const; + std::string prompt_text(const FunAsrNanoPromptRequest &request) const; + +private: + std::shared_ptr assets_; + std::shared_ptr tokenizer_; +}; + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/session.h b/include/engine/models/fun_asr_nano/session.h new file mode 100644 index 00000000..29376fcc --- /dev/null +++ b/include/engine/models/fun_asr_nano/session.h @@ -0,0 +1,69 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/runtime/session_base.h" +#include "engine/models/fun_asr_nano/adaptor.h" +#include "engine/models/fun_asr_nano/assets.h" +#include "engine/models/fun_asr_nano/decoder.h" +#include "engine/models/fun_asr_nano/encoder.h" +#include "engine/models/fun_asr_nano/frontend.h" +#include "engine/models/fun_asr_nano/prompt.h" +#include "engine/models/fun_asr_nano/tokenizer_text.h" + +#include +#include +#include + +namespace engine::models::fun_asr_nano { + +class FunAsrNanoSession final : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession { +public: + FunAsrNanoSession(runtime::TaskSpec task, runtime::SessionOptions options, + std::shared_ptr assets); + ~FunAsrNanoSession() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest &request) override; + runtime::TaskResult run(const runtime::TaskRequest &request) override; + +private: + struct AudioChunkPlan { + runtime::TimeSpan source_span; + }; + + struct AsrRequest { + runtime::AudioBuffer audio; + FunAsrNanoPromptRequest prompt; + FunAsrNanoGenerationOptions generation; + }; + + AsrRequest make_request(const runtime::TaskRequest &request) const; + std::vector + audio_chunk_plan(const runtime::TaskRequest &request) const; + runtime::TaskResult run_single(const AsrRequest &request); + + runtime::TaskSpec task_; + std::shared_ptr assets_; + size_t encoder_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; + size_t adaptor_graph_arena_bytes_ = 128ull * 1024ull * 1024ull; + size_t decoder_prefill_graph_arena_bytes_ = 256ull * 1024ull * 1024ull; + size_t decoder_decode_graph_arena_bytes_ = 128ull * 1024ull * 1024ull; + size_t decoder_weight_context_bytes_ = 32ull * 1024ull * 1024ull; + assets::TensorStorageType encoder_weight_storage_type_ = + assets::TensorStorageType::Native; + assets::TensorStorageType adaptor_weight_storage_type_ = + assets::TensorStorageType::Native; + assets::TensorStorageType decoder_weight_storage_type_ = + assets::TensorStorageType::Native; + FunAsrNanoTextTokenizer tokenizer_; + FunAsrNanoFrontend frontend_; + FunAsrNanoEncoderRuntime encoder_; + FunAsrNanoAdaptorRuntime adaptor_; + FunAsrNanoPromptBuilder prompt_builder_; + FunAsrNanoDecoderRuntime decoder_; +}; + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/tokenizer_text.h b/include/engine/models/fun_asr_nano/tokenizer_text.h new file mode 100644 index 00000000..f81c0201 --- /dev/null +++ b/include/engine/models/fun_asr_nano/tokenizer_text.h @@ -0,0 +1,28 @@ +#pragma once + +#include "engine/models/fun_asr_nano/assets.h" + +#include +#include +#include +#include + +namespace engine::models::fun_asr_nano { + +class FunAsrNanoTextTokenizer { +public: + struct Impl; + + explicit FunAsrNanoTextTokenizer( + std::shared_ptr assets); + + std::vector encode(const std::string &text) const; + std::string decode(const std::vector &token_ids) const; + int32_t require_token_id(const std::string &token) const; + +private: + std::shared_ptr assets_; + std::shared_ptr impl_; +}; + +} // namespace engine::models::fun_asr_nano diff --git a/include/engine/models/fun_asr_nano/types.h b/include/engine/models/fun_asr_nano/types.h new file mode 100644 index 00000000..adc773ac --- /dev/null +++ b/include/engine/models/fun_asr_nano/types.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include + +namespace engine::models::fun_asr_nano { + +struct FunAsrNanoAudioFeatures { + std::vector values; + int64_t frames = 0; + int64_t feature_dim = 0; + int64_t valid_frames = 0; +}; + +struct FunAsrNanoEncoderStage { + std::string name; + std::vector values; +}; + +struct FunAsrNanoEncoderEmbeddings { + std::vector values; + int64_t frames = 0; + int64_t valid_frames = 0; + int64_t hidden_size = 0; + std::vector stages; +}; + +struct FunAsrNanoAdaptorStage { + std::string name; + std::vector values; +}; + +struct FunAsrNanoAdaptorEmbeddings { + std::vector values; + int64_t tokens = 0; + int64_t hidden_size = 0; + std::vector stages; +}; + +struct FunAsrNanoPromptRequest { + std::string prompt; + std::string language = "auto"; + bool enable_itn = true; +}; + +struct FunAsrNanoPrompt { + std::vector input_ids; + std::vector audio_token_positions; + std::vector attention_mask; +}; + +struct FunAsrNanoGenerationOptions { + int64_t max_new_tokens = 512; + bool capture_logits = false; +}; + +struct FunAsrNanoGeneratedTokens { + std::vector token_ids; + std::vector> step_logits; +}; + +} // namespace engine::models::fun_asr_nano diff --git a/model_specs/fun_asr_nano.json b/model_specs/fun_asr_nano.json new file mode 100644 index 00000000..973bfc25 --- /dev/null +++ b/model_specs/fun_asr_nano.json @@ -0,0 +1,201 @@ +{ + "schema_version": 1, + "family": "fun_asr_nano", + "display_name": "Fun-ASR-Nano", + "description": "Offline multilingual speech recognition with the FunAudioLLM Fun-ASR-Nano-2512 model.", + "category": "asr", + "status": "wip", + "tasks": [ + "asr" + ], + "modes": [ + "offline" + ], + "languages": [ + "auto", + "zh", + "en", + "ja" + ], + "capabilities": {}, + "options": { + "request": [ + { + "name": "language", + "type": "string", + "description": "Recognition language, or auto to let the model infer it.", + "required": false, + "default": "auto" + }, + { + "name": "enable_itn", + "type": "bool", + "description": "Enable inverse text normalization in the transcription prompt.", + "required": false, + "default": true + }, + { + "name": "max_tokens", + "type": "int", + "description": "Maximum number of generated transcript tokens.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "audio_chunk_mode", + "type": "enum", + "description": "Audio chunking mode: auto, fixed, or none.", + "values": [ + "auto", + "fixed", + "none" + ], + "required": false, + "default": "auto" + }, + { + "name": "audio_chunk_seconds", + "type": "float", + "description": "Fixed chunk duration in seconds.", + "required": false, + "min": 0.001, + "default": 30 + } + ], + "session": [ + { + "name": "weight_type", + "type": "enum", + "description": "Shared model weight storage type.", + "preset": "weight_type_full", + "required": false, + "default": "native" + } + ], + "load": [] + }, + "runtime": { + "tags": [ + "gguf", + "server", + "cuda", + "metal", + "cpu" + ] + }, + "packages": [ + { + "id": "fun_asr_nano_2512_q8_0", + "display_name": "Fun-ASR-Nano-2512 Q8_0 GGUF", + "description": "Standalone audio.cpp GGUF built from the pinned official checkpoint; governed by the FunASR Model Open Source License Agreement v1.1.", + "default": true, + "format": "gguf", + "precision": "q8_0", + "target_directory": "Fun-ASR-Nano-2512-GGUF", + "files": [ + "fun-asr-nano-2512-q8_0.gguf" + ], + "download": { + "kind": "huggingface_snapshot", + "repo": "FunAudioLLM/Fun-ASR-Nano-2512-GGUF", + "revision": "ce72677f84900f0dc57f498ace253bfb3c9155b6", + "gated": false + } + }, + { + "id": "fun_asr_nano_2512_f16", + "display_name": "Fun-ASR-Nano-2512 F16 GGUF", + "description": "Standalone audio.cpp GGUF built from the pinned official checkpoint; governed by the FunASR Model Open Source License Agreement v1.1.", + "format": "gguf", + "precision": "f16", + "target_directory": "Fun-ASR-Nano-2512-GGUF", + "files": [ + "fun-asr-nano-2512-f16.gguf" + ], + "download": { + "kind": "huggingface_snapshot", + "repo": "FunAudioLLM/Fun-ASR-Nano-2512-GGUF", + "revision": "ce72677f84900f0dc57f498ace253bfb3c9155b6", + "gated": false + } + }, + { + "id": "fun_asr_nano_2512_safetensors", + "display_name": "Fun-ASR-Nano-2512 HF Safetensors", + "description": "Official checkpoint governed by the FunASR Model Open Source License Agreement v1.1.", + "format": "safetensors", + "precision": "native", + "target_directory": "Fun-ASR-Nano-2512-hf", + "files": [ + "chat_template.jinja", + "config.json", + "generation_config.json", + "model.safetensors", + "processor_config.json", + "tokenizer.json", + "tokenizer_config.json" + ], + "download": { + "kind": "huggingface_snapshot", + "repo": "FunAudioLLM/Fun-ASR-Nano-2512-hf", + "revision": "854d88f94205cd17d2afdb24332130d86fbe654a", + "gated": false + } + } + ], + "dependencies": [], + "ui": { + "recommended_package": "fun_asr_nano_2512_q8_0", + "tags": [ + "ASR", + "GGUF" + ], + "docs": [ + "docs/asr.md", + "docs/gguf.md" + ], + "summary": "Offline Fun-ASR-Nano transcription from official safetensors or audio.cpp GGUF." + }, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "processor_config": "model:processor_config.json", + "tokenizer_json": "model:tokenizer.json" + }, + "optional_files": { + "chat_template_jinja": "model:chat_template.jinja", + "tokenizer_config": "model:tokenizer_config.json" + }, + "tensors": { + "weights": "weights:" + } + }, + { + "format": "safetensors", + "roots": { + "model": "." + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "processor_config": "model:processor_config.json", + "tokenizer_json": "model:tokenizer.json" + }, + "optional_files": { + "chat_template_jinja": "model:chat_template.jinja", + "tokenizer_config": "model:tokenizer_config.json" + }, + "tensors": { + "weights": "model:model.safetensors" + } + } + ] +} diff --git a/src/framework/audio/kaldi_fbank.cpp b/src/framework/audio/kaldi_fbank.cpp new file mode 100644 index 00000000..a3f4ea21 --- /dev/null +++ b/src/framework/audio/kaldi_fbank.cpp @@ -0,0 +1,288 @@ +#include "engine/framework/audio/kaldi_fbank.h" + +#include "engine/framework/audio/fft.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Framing and LFR structure adapted from handy-computer/transcribe.cpp at +// c87109304c42707867f926fb7b9c378d9f46df8a. SPDX-License-Identifier: MIT; +// Copyright (c) 2026 The transcribe.cpp authors. + +namespace engine::audio { +namespace { + +int next_power_of_two(int value) { + int result = 1; + while (result < value) { + if (result > std::numeric_limits::max() / 2) { + throw std::runtime_error("Kaldi fbank FFT size overflow"); + } + result *= 2; + } + return result; +} + +double hz_to_mel(double frequency) { + return 1127.0 * std::log(1.0 + frequency / 700.0); +} + +size_t checked_product(size_t left, size_t right, const char *context) { + if (left != 0 && right > std::numeric_limits::max() / left) { + throw std::runtime_error(std::string("Kaldi fbank size overflow: ") + + context); + } + return left * right; +} + +void validate_options(const KaldiFbankOptions &options) { + if (options.sample_rate <= 0) { + throw std::runtime_error("Kaldi fbank sample_rate must be > 0"); + } + if (options.num_mels <= 3) { + throw std::runtime_error("Kaldi fbank num_mels must be > 3"); + } + if (!std::isfinite(options.frame_length_ms) || + !std::isfinite(options.frame_shift_ms) || + !(options.frame_length_ms > 0.0F) || !(options.frame_shift_ms > 0.0F)) { + throw std::runtime_error("Kaldi fbank frame length and shift must be > 0"); + } + if (options.lfr_m <= 0 || options.lfr_n <= 0) { + throw std::runtime_error("Kaldi fbank LFR factors must be > 0"); + } + if (!std::isfinite(options.preemphasis) || options.preemphasis < 0.0F || + options.preemphasis > 1.0F) { + throw std::runtime_error("Kaldi fbank preemphasis must be in [0, 1]"); + } + const float nyquist = static_cast(options.sample_rate) * 0.5F; + const float high_frequency = options.high_frequency <= 0.0F + ? nyquist + options.high_frequency + : options.high_frequency; + if (!std::isfinite(options.low_frequency) || + !std::isfinite(options.high_frequency) || + !std::isfinite(high_frequency) || options.low_frequency < 0.0F || + options.low_frequency >= high_frequency || high_frequency > nyquist) { + throw std::runtime_error("Kaldi fbank frequency bounds are invalid"); + } +} + +std::vector make_hamming_window(int size) { + std::vector window(static_cast(size), 1.0F); + if (size <= 1) { + return window; + } + const double radians_per_sample = + 2.0 * std::acos(-1.0) / static_cast(size - 1); + for (int index = 0; index < size; ++index) { + window[static_cast(index)] = + static_cast(0.54 - 0.46 * std::cos(radians_per_sample * + static_cast(index))); + } + return window; +} + +std::vector make_mel_filterbank(int sample_rate, int fft_size, + int num_mels, float low_frequency, + float high_frequency) { + const int spectrum_bins = fft_size / 2 + 1; + const int filtered_bins = fft_size / 2; + const double nyquist = static_cast(sample_rate) * 0.5; + const double high = + high_frequency <= 0.0F ? nyquist + high_frequency : high_frequency; + const double low_mel = hz_to_mel(low_frequency); + const double high_mel = hz_to_mel(high); + const double mel_step = + (high_mel - low_mel) / static_cast(num_mels + 1); + const double bin_width = + static_cast(sample_rate) / static_cast(fft_size); + + std::vector filters(checked_product(static_cast(num_mels), + static_cast(spectrum_bins), + "mel filters"), + 0.0F); + for (int mel_bin = 0; mel_bin < num_mels; ++mel_bin) { + const double left = low_mel + static_cast(mel_bin) * mel_step; + const double center = left + mel_step; + const double right = center + mel_step; + float *row = filters.data() + static_cast(mel_bin) * spectrum_bins; + for (int fft_bin = 0; fft_bin < filtered_bins; ++fft_bin) { + const double mel = hz_to_mel(static_cast(fft_bin) * bin_width); + const double rising = (mel - left) / (center - left); + const double falling = (right - mel) / (right - center); + row[fft_bin] = + static_cast(std::max(0.0, std::min(rising, falling))); + } + } + return filters; +} + +} // namespace + +KaldiFbankFeatures extract_kaldi_fbank(const std::vector &audio, + const KaldiFbankOptions &options) { + validate_options(options); + if (options.num_mels > std::numeric_limits::max() / options.lfr_m) { + throw std::runtime_error("Kaldi fbank feature dimension overflow"); + } + const int feature_dim = options.num_mels * options.lfr_m; + if (options.apply_cmvn && + (options.cmvn_shift.size() != static_cast(feature_dim) || + options.cmvn_scale.size() != static_cast(feature_dim))) { + throw std::runtime_error( + "Kaldi fbank CMVN vectors must match the LFR feature dimension"); + } + + const double window_size_f64 = static_cast(options.sample_rate) * + static_cast(options.frame_length_ms) / + 1000.0; + const double window_shift_f64 = static_cast(options.sample_rate) * + static_cast(options.frame_shift_ms) / + 1000.0; + if (window_size_f64 > std::numeric_limits::max() || + window_shift_f64 > std::numeric_limits::max()) { + throw std::runtime_error( + "Kaldi fbank frame settings exceed the supported sample count"); + } + const int window_size = static_cast(window_size_f64); + const int window_shift = static_cast(window_shift_f64); + if (window_size <= 0 || window_shift <= 0) { + throw std::runtime_error( + "Kaldi fbank frame settings round to zero samples"); + } + + KaldiFbankFeatures result; + result.feature_dim = feature_dim; + if (audio.size() < static_cast(window_size)) { + return result; + } + + const size_t mel_frames_size = + 1 + (audio.size() - static_cast(window_size)) / + static_cast(window_shift); + if (mel_frames_size > static_cast(std::numeric_limits::max())) { + throw std::runtime_error( + "Kaldi fbank frame count exceeds the supported range"); + } + const int mel_frames = static_cast(mel_frames_size); + const int fft_size = next_power_of_two(window_size); + const int spectrum_bins = fft_size / 2 + 1; + const float sample_scale = options.upscale_samples ? 32768.0F : 1.0F; + const auto window = make_hamming_window(window_size); + const auto filters = + make_mel_filterbank(options.sample_rate, fft_size, options.num_mels, + options.low_frequency, options.high_frequency); + + std::vector framed(checked_product(static_cast(mel_frames), + static_cast(fft_size), + "framed audio"), + 0.0F); + for (int frame_index = 0; frame_index < mel_frames; ++frame_index) { + float *frame = framed.data() + static_cast(frame_index) * fft_size; + const size_t sample_offset = + static_cast(frame_index) * window_shift; + for (int sample = 0; sample < window_size; ++sample) { + frame[sample] = + audio[sample_offset + static_cast(sample)] * sample_scale; + } + if (options.remove_dc_offset) { + double sum = 0.0; + for (int sample = 0; sample < window_size; ++sample) { + sum += frame[sample]; + } + const float mean = + static_cast(sum / static_cast(window_size)); + for (int sample = 0; sample < window_size; ++sample) { + frame[sample] -= mean; + } + } + if (options.preemphasis != 0.0F) { + for (int sample = window_size - 1; sample >= 1; --sample) { + frame[sample] -= options.preemphasis * frame[sample - 1]; + } + frame[0] *= 1.0F - options.preemphasis; + } + for (int sample = 0; sample < window_size; ++sample) { + frame[sample] *= window[static_cast(sample)]; + } + } + + std::vector> spectrum( + checked_product(static_cast(mel_frames), + static_cast(spectrum_bins), "spectrum")); + real_fft_forward( + {static_cast(mel_frames), static_cast(fft_size)}, + { + static_cast(fft_size) * + static_cast(sizeof(float)), + static_cast(sizeof(float)), + }, + { + static_cast(spectrum_bins) * + static_cast(sizeof(std::complex)), + static_cast(sizeof(std::complex)), + }, + 1, framed.data(), spectrum.data()); + + std::vector mel_features( + checked_product(static_cast(mel_frames), + static_cast(options.num_mels), "mel features"), + 0.0F); + constexpr float kMelEpsilon = std::numeric_limits::epsilon(); + for (int frame_index = 0; frame_index < mel_frames; ++frame_index) { + const auto *frame_spectrum = + spectrum.data() + static_cast(frame_index) * spectrum_bins; + float *mel_row = mel_features.data() + + static_cast(frame_index) * options.num_mels; + for (int mel_bin = 0; mel_bin < options.num_mels; ++mel_bin) { + const float *filter = + filters.data() + static_cast(mel_bin) * spectrum_bins; + float energy = 0.0F; + for (int fft_bin = 0; fft_bin < spectrum_bins; ++fft_bin) { + energy += filter[fft_bin] * std::norm(frame_spectrum[fft_bin]); + } + mel_row[mel_bin] = std::log(std::max(energy, kMelEpsilon)); + } + } + + const int left_pad = (options.lfr_m - 1) / 2; + result.frames = 1 + (mel_frames - 1) / options.lfr_n; + result.values.assign(checked_product(static_cast(result.frames), + static_cast(feature_dim), + "LFR features"), + 0.0F); + for (int output_frame = 0; output_frame < result.frames; ++output_frame) { + float *output = + result.values.data() + static_cast(output_frame) * feature_dim; + for (int stacked_frame = 0; stacked_frame < options.lfr_m; + ++stacked_frame) { + const int64_t padded_index = + static_cast(output_frame) * options.lfr_n + stacked_frame; + const int source_frame = static_cast(std::clamp( + padded_index - left_pad, 0, static_cast(mel_frames - 1))); + const float *source = + mel_features.data() + + static_cast(source_frame) * options.num_mels; + std::memcpy( + output + static_cast(stacked_frame) * options.num_mels, + source, static_cast(options.num_mels) * sizeof(float)); + } + if (options.apply_cmvn) { + for (int feature = 0; feature < feature_dim; ++feature) { + output[feature] = (output[feature] + + options.cmvn_shift[static_cast(feature)]) * + options.cmvn_scale[static_cast(feature)]; + } + } + } + return result; +} + +} // namespace engine::audio diff --git a/src/framework/modules/attention/scaled_dot_product_attention.cpp b/src/framework/modules/attention/scaled_dot_product_attention.cpp index b339b0f7..c0d4cb66 100644 --- a/src/framework/modules/attention/scaled_dot_product_attention.cpp +++ b/src/framework/modules/attention/scaled_dot_product_attention.cpp @@ -56,9 +56,11 @@ core::TensorValue build_explicit( const core::TensorValue & v_heads, const std::optional & attention_mask, float scale, + ggml_prec precision, AttentionCausality causality) { const MatMulModule matmul; auto scores = matmul.build(ctx, q_heads, TransposeModule({{0, 1, 3, 2}, k_heads.shape.rank}).build(ctx, k_heads)); + ggml_mul_mat_set_prec(scores.tensor, precision); scores = core::ensure_backend_addressable_layout(ctx, scores); core::TensorValue attn; if (attention_mask.has_value()) { @@ -75,6 +77,7 @@ core::TensorValue build_explicit( attn = core::wrap_tensor(ggml_soft_max(ctx.ggml, scores.tensor), scores.shape, GGML_TYPE_F32); } auto context = matmul.build(ctx, attn, v_heads); + ggml_mul_mat_set_prec(context.tensor, precision); return TransposeModule({{0, 2, 1, 3}, context.shape.rank}).build(ctx, context); } @@ -150,7 +153,7 @@ core::TensorValue ScaledDotProductAttentionModule::build( const float scale = 1.0F / std::sqrt(static_cast(config_.head_dim)); switch (config_.lowering) { case ScaledDotProductAttentionLowering::Explicit: - return build_explicit(ctx, q_heads, k_heads, v_heads, attention_mask, scale, config_.causality); + return build_explicit(ctx, q_heads, k_heads, v_heads, attention_mask, scale, config_.precision, config_.causality); case ScaledDotProductAttentionLowering::Flash: return build_flash(ctx, q_heads, k_heads, v_heads, attention_mask, scale, config_.precision, config_.causality); case ScaledDotProductAttentionLowering::FlashPreserveViews: diff --git a/src/framework/modules/speech_encoders/sanm.cpp b/src/framework/modules/speech_encoders/sanm.cpp new file mode 100644 index 00000000..0b927d83 --- /dev/null +++ b/src/framework/modules/speech_encoders/sanm.cpp @@ -0,0 +1,177 @@ +#include "engine/framework/modules/speech_encoders/sanm.h" + +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/structural_modules.h" + +#include +#include +#include + +namespace engine::modules { +namespace { + +void validate_config(const SanmBlockConfig &config) { + if (config.input_size <= 0 || config.model_size <= 0 || + config.num_heads <= 0 || config.ffn_size <= 0 || + config.fsmn_kernel_size <= 0) { + throw std::runtime_error("SAN-M block dimensions must be positive"); + } + if (config.model_size % config.num_heads != 0) { + throw std::runtime_error( + "SAN-M model size must be divisible by the head count"); + } + if (config.fsmn_kernel_size % 2 == 0) { + throw std::runtime_error("SAN-M FSMN kernel size must be odd"); + } + if (!std::isfinite(config.layer_norm_eps) || + !(config.layer_norm_eps > 0.0F)) { + throw std::runtime_error( + "SAN-M layer norm epsilon must be finite and positive"); + } +} + +core::TensorValue reshape_heads(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + int64_t num_heads, int64_t head_dim) { + const auto contiguous = core::ensure_backend_addressable_layout(ctx, input); + return core::reshape_tensor( + ctx, contiguous, + core::TensorShape::from_dims( + {input.shape.dims[0], input.shape.dims[1], num_heads, head_dim})); +} + +struct SanmAttentionBranch { + core::TensorValue output; +}; + +SanmAttentionBranch build_attention_branch(core::ModuleBuildContext &ctx, + const core::TensorValue &normalized, + const SanmBlockWeightsView &weights, + const SanmBlockConfig &config) { + const int64_t head_dim = config.model_size / config.num_heads; + const LinearModule q_projection( + {config.input_size, config.model_size, true, GGML_PREC_F32}); + const LinearModule k_projection( + {config.input_size, config.model_size, true, GGML_PREC_F32}); + const LinearModule v_projection( + {config.input_size, config.model_size, true, GGML_PREC_F32}); + const LinearModule output_projection( + {config.model_size, config.model_size, true, GGML_PREC_F32}); + + auto query = q_projection.build(ctx, normalized, weights.query_projection); + auto key = k_projection.build(ctx, normalized, weights.key_projection); + auto value = v_projection.build(ctx, normalized, weights.value_projection); + + query = reshape_heads(ctx, query, config.num_heads, head_dim); + key = reshape_heads(ctx, key, config.num_heads, head_dim); + auto value_heads = reshape_heads(ctx, value, config.num_heads, head_dim); + query = TransposeModule({{0, 2, 1, 3}, query.shape.rank}).build(ctx, query); + key = TransposeModule({{0, 2, 1, 3}, key.shape.rank}).build(ctx, key); + value_heads = TransposeModule({{0, 2, 1, 3}, value_heads.shape.rank}) + .build(ctx, value_heads); + + auto attention = + ScaledDotProductAttentionModule({ + head_dim, + config.attention_lowering, + GGML_PREC_F32, + AttentionCausality::NonCausal, + }) + .build(ctx, query, key, value_heads); + attention = core::ensure_backend_addressable_layout(ctx, attention); + attention = + core::reshape_tensor(ctx, attention, + core::TensorShape::from_dims( + {normalized.shape.dims[0], + normalized.shape.dims[1], config.model_size})); + attention = output_projection.build(ctx, attention, + weights.attention_output_projection); + + auto value_bct = + TransposeModule({{0, 2, 1, 3}, value.shape.rank}).build(ctx, value); + value_bct = core::ensure_backend_addressable_layout(ctx, value_bct); + auto fsmn = DepthwiseConv1dModule( + { + config.model_size, + config.fsmn_kernel_size, + 1, + static_cast((config.fsmn_kernel_size - 1) / 2), + 1, + false, + }) + .build(ctx, value_bct, {weights.fsmn_weight, std::nullopt}); + fsmn = TransposeModule({{0, 2, 1, 3}, fsmn.shape.rank}).build(ctx, fsmn); + fsmn = AddModule{}.build(ctx, fsmn, value); + return {AddModule{}.build(ctx, attention, fsmn)}; +} + +core::TensorValue build_ffn_residual(core::ModuleBuildContext &ctx, + const core::TensorValue &residual, + const SanmBlockWeightsView &weights, + const SanmBlockConfig &config) { + auto hidden = + sanm_layer_norm(ctx, residual, weights.final_norm, config.layer_norm_eps); + hidden = + LinearModule({config.model_size, config.ffn_size, true, GGML_PREC_F32}) + .build(ctx, hidden, weights.ffn_input_projection); + hidden = ReluModule{}.build(ctx, hidden); + hidden = + LinearModule({config.ffn_size, config.model_size, true, GGML_PREC_F32}) + .build(ctx, hidden, weights.ffn_output_projection); + return AddModule{}.build(ctx, residual, hidden); +} + +core::TensorValue build_block(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const SanmBlockWeightsView &weights, + const SanmBlockConfig &config, + bool add_input_residual) { + validate_config(config); + core::validate_rank_between(input, 3, 3, "SAN-M input"); + core::validate_last_dim(input, config.input_size, "SAN-M input"); + if (add_input_residual && config.input_size != config.model_size) { + throw std::runtime_error( + "SAN-M residual block requires input_size == model_size"); + } + + const auto normalized = sanm_layer_norm( + ctx, input, weights.self_attention_norm, config.layer_norm_eps); + auto residual = + build_attention_branch(ctx, normalized, weights, config).output; + if (add_input_residual) { + residual = AddModule{}.build(ctx, input, residual); + } + return build_ffn_residual(ctx, residual, weights, config); +} + +} // namespace + +core::TensorValue sanm_layer_norm(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const NormWeights &weights, float epsilon) { + core::validate_rank_between(input, 2, 4, "SAN-M layer norm input"); + if (!std::isfinite(epsilon) || !(epsilon > 0.0F)) { + throw std::runtime_error( + "SAN-M layer norm epsilon must be finite and positive"); + } + return LayerNormModule({input.shape.last_dim(), epsilon, true, true}) + .build(ctx, input, weights); +} + +core::TensorValue sanm_projection_block(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const SanmBlockWeightsView &weights, + const SanmBlockConfig &config) { + return build_block(ctx, input, weights, config, false); +} + +core::TensorValue sanm_residual_block(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const SanmBlockWeightsView &weights, + const SanmBlockConfig &config) { + return build_block(ctx, input, weights, config, true); +} + +} // namespace engine::modules diff --git a/src/models/fun_asr_nano/adaptor.cpp b/src/models/fun_asr_nano/adaptor.cpp new file mode 100644 index 00000000..e5ae099c --- /dev/null +++ b/src/models/fun_asr_nano/adaptor.cpp @@ -0,0 +1,471 @@ +#include "engine/models/fun_asr_nano/adaptor.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/attention/scaled_dot_product_attention.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace engine::models::fun_asr_nano { +namespace { + +using Clock = std::chrono::steady_clock; + +constexpr size_t kAdaptorGraphNodes = 16384; +constexpr size_t kWeightContextBytes = 8 * 1024 * 1024; +constexpr float kLayerNormEpsilon = 1.0e-5F; + +engine::modules::LinearWeights +load_linear(engine::core::BackendWeightStore &store, + const engine::assets::TensorSource &source, + const std::string &prefix, int64_t input_size, int64_t output_size, + engine::assets::TensorStorageType storage_type) { + return { + store.load_tensor(source, prefix + ".weight", storage_type, + {output_size, input_size}), + store.load_f32_tensor(source, prefix + ".bias", {output_size}), + }; +} + +engine::modules::NormWeights +load_norm(engine::core::BackendWeightStore &store, + const engine::assets::TensorSource &source, const std::string &prefix, + int64_t size) { + return { + store.load_f32_tensor(source, prefix + ".weight", {size}), + store.load_f32_tensor(source, prefix + ".bias", {size}), + }; +} + +struct AdaptorLayerWeights { + engine::modules::NormWeights attention_norm; + engine::modules::LinearWeights q_proj; + engine::modules::LinearWeights k_proj; + engine::modules::LinearWeights v_proj; + engine::modules::LinearWeights out_proj; + engine::modules::NormWeights final_norm; + engine::modules::LinearWeights fc1; + engine::modules::LinearWeights fc2; +}; + +struct AdaptorWeights { + std::unique_ptr store; + engine::modules::LinearWeights projector_1; + engine::modules::LinearWeights projector_2; + std::vector layers; +}; + +AdaptorLayerWeights +load_adaptor_layer(engine::core::BackendWeightStore &store, + const engine::assets::TensorSource &source, + const std::string &prefix, + const FunAsrNanoAdaptorConfig &config, + engine::assets::TensorStorageType storage_type) { + return { + load_norm(store, source, prefix + ".self_attn_layer_norm", + config.d_model), + load_linear(store, source, prefix + ".self_attn.q_proj", config.d_model, + config.d_model, storage_type), + load_linear(store, source, prefix + ".self_attn.k_proj", config.d_model, + config.d_model, storage_type), + load_linear(store, source, prefix + ".self_attn.v_proj", config.d_model, + config.d_model, storage_type), + load_linear(store, source, prefix + ".self_attn.out_proj", config.d_model, + config.d_model, storage_type), + load_norm(store, source, prefix + ".final_layer_norm", config.d_model), + load_linear(store, source, prefix + ".fc1", config.d_model, + config.ffn_dim, storage_type), + load_linear(store, source, prefix + ".fc2", config.ffn_dim, + config.d_model, storage_type), + }; +} + +std::unique_ptr +load_adaptor_weights(const FunAsrNanoAssets &assets, + engine::core::ExecutionContext &execution_context, + engine::assets::TensorStorageType storage_type) { + auto weights = std::make_unique(); + weights->store = std::make_unique( + execution_context.backend(), execution_context.backend_type(), + "Fun-ASR-Nano adaptor weights", kWeightContextBytes); + const auto &source = *assets.model_weights; + const auto &encoder = assets.config.encoder; + const auto &config = assets.config.adaptor; + const std::string projector_root = "model.multi_modal_projector."; + weights->projector_1 = load_linear( + *weights->store, source, projector_root + "linear_1", encoder.d_model, + assets.config.projector_hidden_size, storage_type); + weights->projector_2 = load_linear( + *weights->store, source, projector_root + "linear_2", + assets.config.projector_hidden_size, config.d_model, storage_type); + + const std::string current_root = "model.audio_adaptor.blocks."; + const std::string legacy_root = "model.multi_modal_projector.blocks."; + const std::string block_root = + source.has_tensor(current_root + "0.self_attn.q_proj.weight") + ? current_root + : legacy_root; + weights->layers.reserve(static_cast(config.layers)); + for (int64_t index = 0; index < config.layers; ++index) { + weights->layers.push_back(load_adaptor_layer( + *weights->store, source, block_root + std::to_string(index), config, + storage_type)); + } + weights->store->upload(); + return weights; +} + +engine::core::TensorValue linear(engine::core::ModuleBuildContext &context, + const engine::core::TensorValue &input, + const engine::modules::LinearWeights &weights, + int64_t input_size, int64_t output_size) { + return engine::modules::LinearModule( + {input_size, output_size, true, GGML_PREC_F32}) + .build(context, input, weights); +} + +engine::core::TensorValue +layer_norm(engine::core::ModuleBuildContext &context, + const engine::core::TensorValue &input, + const engine::modules::NormWeights &weights, int64_t hidden_size) { + return engine::modules::LayerNormModule( + {hidden_size, kLayerNormEpsilon, true, true}) + .build(context, input, weights); +} + +engine::core::TensorValue split_heads(engine::core::ModuleBuildContext &context, + const engine::core::TensorValue &input, + int64_t heads, int64_t head_dim) { + const auto contiguous = + engine::core::ensure_backend_addressable_layout(context, input); + const auto reshaped = engine::core::reshape_tensor( + context, contiguous, + engine::core::TensorShape::from_dims( + {input.shape.dims[0], input.shape.dims[1], heads, head_dim})); + return engine::modules::TransposeModule({{0, 2, 1, 3}, 4}) + .build(context, reshaped); +} + +engine::core::TensorValue merge_heads(engine::core::ModuleBuildContext &context, + const engine::core::TensorValue &input, + int64_t hidden_size) { + const auto contiguous = + engine::core::ensure_backend_addressable_layout(context, input); + return engine::core::reshape_tensor( + context, contiguous, + engine::core::TensorShape::from_dims( + {input.shape.dims[0], input.shape.dims[1], hidden_size})); +} + +engine::core::TensorValue +adaptor_block(engine::core::ModuleBuildContext &context, + const engine::core::TensorValue &input, + const AdaptorLayerWeights &weights, + const FunAsrNanoAdaptorConfig &config) { + const int64_t head_dim = config.d_model / config.attention_heads; + auto normalized = + layer_norm(context, input, weights.attention_norm, config.d_model); + auto q = split_heads(context, + linear(context, normalized, weights.q_proj, + config.d_model, config.d_model), + config.attention_heads, head_dim); + auto k = split_heads(context, + linear(context, normalized, weights.k_proj, + config.d_model, config.d_model), + config.attention_heads, head_dim); + auto v = split_heads(context, + linear(context, normalized, weights.v_proj, + config.d_model, config.d_model), + config.attention_heads, head_dim); + auto attention = + engine::modules::ScaledDotProductAttentionModule( + {head_dim, + engine::modules::ScaledDotProductAttentionLowering::Explicit, + GGML_PREC_F32, engine::modules::AttentionCausality::NonCausal}) + .build(context, q, k, v); + attention = merge_heads(context, attention, config.d_model); + attention = linear(context, attention, weights.out_proj, config.d_model, + config.d_model); + auto hidden = + engine::modules::ResidualAddModule{}.build(context, input, attention); + + normalized = layer_norm(context, hidden, weights.final_norm, config.d_model); + auto feed_forward = + linear(context, normalized, weights.fc1, config.d_model, config.ffn_dim); + feed_forward = engine::modules::ReluModule{}.build(context, feed_forward); + feed_forward = linear(context, feed_forward, weights.fc2, config.ffn_dim, + config.d_model); + return engine::modules::ResidualAddModule{}.build(context, hidden, + feed_forward); +} + +} // namespace + +struct FunAsrNanoAdaptorRuntime::Impl { + struct Graph { + int64_t frames = 0; + ggml_backend_t backend = nullptr; + ggml_context *ggml = nullptr; + ggml_gallocr_t allocator = nullptr; + ggml_cgraph *graph = nullptr; + engine::core::HostGraphPlan host_plan; + engine::core::TensorValue input; + engine::core::TensorValue output; + std::vector> checkpoints; + + ~Graph() { + host_plan.reset(); + if (backend != nullptr && graph != nullptr) { + engine::core::release_backend_graph_resources(backend, graph); + } + if (allocator != nullptr) { + ggml_gallocr_free(allocator); + } + if (ggml != nullptr) { + ggml_free(ggml); + } + } + }; + + Impl(std::shared_ptr assets_value, + engine::core::ExecutionContext &execution_context_value, + size_t graph_arena_bytes_value, + engine::assets::TensorStorageType weight_storage) + : assets(std::move(assets_value)), + execution_context(&execution_context_value), + graph_arena_bytes(graph_arena_bytes_value) { + if (assets == nullptr || assets->model_weights == nullptr) { + throw std::runtime_error( + "Fun-ASR-Nano adaptor requires model assets and weights"); + } + if (graph_arena_bytes == 0) { + throw std::runtime_error( + "Fun-ASR-Nano adaptor graph arena must be non-zero"); + } + const auto &config = assets->config.adaptor; + if (config.d_model % config.attention_heads != 0) { + throw std::runtime_error( + "Fun-ASR-Nano adaptor width must be divisible by its head count"); + } + weights = load_adaptor_weights(*assets, *execution_context, weight_storage); + } + + void ensure_graph(int64_t frames) { + if (frames <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano adaptor graph requires positive frames"); + } + if (cached_graph != nullptr && cached_graph->frames == frames && + cached_graph->backend == execution_context->backend()) { + engine::debug::trace_log_scalar("fun_asr_nano.adaptor.graph_cache_hit", + true); + return; + } + + const auto build_start = Clock::now(); + const auto &encoder = assets->config.encoder; + const auto &config = assets->config.adaptor; + auto next = std::make_unique(); + next->frames = frames; + next->backend = execution_context->backend(); + ggml_init_params params{}; + params.mem_size = graph_arena_bytes; + params.mem_buffer = nullptr; + params.no_alloc = true; + next->ggml = ggml_init(params); + if (next->ggml == nullptr) { + throw std::runtime_error( + "failed to initialize Fun-ASR-Nano adaptor graph context"); + } + + engine::core::ModuleBuildContext context{next->ggml, "fun_asr_nano.adaptor", + execution_context->backend_type()}; + const auto input_shape = + engine::core::TensorShape::from_dims({1, frames, encoder.d_model}); + next->input = + engine::core::make_tensor(context, GGML_TYPE_F32, input_shape); + ggml_set_input(next->input.tensor); + ggml_set_output(next->input.tensor); + + auto linear_1 = + linear(context, next->input, weights->projector_1, encoder.d_model, + assets->config.projector_hidden_size); + auto linear_1_checkpoint = engine::core::wrap_tensor( + ggml_dup(context.ggml, linear_1.tensor), linear_1.shape, linear_1.type); + next->checkpoints.push_back({"linear_1", linear_1_checkpoint}); + auto hidden = engine::modules::ReluModule{}.build(context, linear_1); + hidden = linear(context, hidden, weights->projector_2, + assets->config.projector_hidden_size, config.d_model); + auto linear_2_checkpoint = engine::core::wrap_tensor( + ggml_dup(context.ggml, hidden.tensor), hidden.shape, hidden.type); + next->checkpoints.push_back({"linear_2", linear_2_checkpoint}); + for (size_t index = 0; index < weights->layers.size(); ++index) { + hidden = adaptor_block(context, hidden, weights->layers[index], config); + auto checkpoint = + index + 1 == weights->layers.size() + ? hidden + : engine::core::wrap_tensor(ggml_dup(context.ggml, hidden.tensor), + hidden.shape, hidden.type); + next->checkpoints.push_back( + {"block_" + std::to_string(index), checkpoint}); + } + next->output = hidden; + next->checkpoints.push_back({"packed_valid", next->output}); + for (const auto &checkpoint : next->checkpoints) { + ggml_set_output(checkpoint.second.tensor); + } + + next->graph = ggml_new_graph_custom(next->ggml, kAdaptorGraphNodes, false); + for (const auto &checkpoint : next->checkpoints) { + ggml_build_forward_expand(next->graph, checkpoint.second.tensor); + } + ggml_build_forward_expand(next->graph, next->output.tensor); + engine::core::validate_backend_graph_supported(next->backend, next->graph, + "Fun-ASR-Nano adaptor"); + next->allocator = + ggml_gallocr_new(ggml_backend_get_default_buffer_type(next->backend)); + if (next->allocator == nullptr || + !ggml_gallocr_reserve(next->allocator, next->graph) || + !ggml_gallocr_alloc_graph(next->allocator, next->graph)) { + throw std::runtime_error( + "failed to allocate Fun-ASR-Nano adaptor graph tensors"); + } + engine::core::prepare_host_graph_plan(*execution_context, next->graph, + next->host_plan); + cached_graph = std::move(next); + engine::debug::timing_log_scalar( + "fun_asr_nano.adaptor.graph_build_ms", + engine::debug::elapsed_ms(build_start, Clock::now())); + engine::debug::trace_log_scalar("fun_asr_nano.adaptor.graph_cache_hit", + false); + engine::debug::trace_log_scalar("fun_asr_nano.adaptor.graph_frames", + frames); + } + + FunAsrNanoAdaptorEmbeddings + adapt(const FunAsrNanoEncoderEmbeddings &encoder_embeddings, + const std::vector &mask, bool capture_stages) { + const auto &encoder = assets->config.encoder; + const auto &config = assets->config.adaptor; + if (encoder_embeddings.frames <= 0 || + encoder_embeddings.hidden_size != encoder.d_model) { + throw std::runtime_error("Fun-ASR-Nano adaptor input shape is invalid"); + } + if (encoder_embeddings.frames > + std::numeric_limits::max() / encoder.d_model) { + throw std::runtime_error("Fun-ASR-Nano adaptor input shape is invalid"); + } + if (static_cast(encoder_embeddings.values.size()) != + encoder_embeddings.frames * encoder.d_model) { + throw std::runtime_error( + "Fun-ASR-Nano adaptor input value count mismatch"); + } + if (static_cast(mask.size()) != encoder_embeddings.frames) { + throw std::runtime_error("Fun-ASR-Nano adaptor mask size mismatch"); + } + + bool saw_padding = false; + int64_t valid_frames = 0; + for (const int32_t value : mask) { + if (value == 0) { + saw_padding = true; + } else if (value == 1 && !saw_padding) { + ++valid_frames; + } else if (value == 1) { + throw std::runtime_error( + "Fun-ASR-Nano adaptor mask must be right-padded"); + } else { + throw std::runtime_error( + "Fun-ASR-Nano adaptor mask values must be zero or one"); + } + } + if (valid_frames <= 0 || valid_frames != encoder_embeddings.valid_frames) { + throw std::runtime_error( + "Fun-ASR-Nano adaptor mask does not match valid frames"); + } + + const auto adapt_start = Clock::now(); + ensure_graph(valid_frames); + const size_t valid_value_count = + static_cast(valid_frames * encoder.d_model); + std::vector valid_values(encoder_embeddings.values.begin(), + encoder_embeddings.values.begin() + + valid_value_count); + engine::core::write_tensor_f32(cached_graph->input, valid_values); + const auto status = engine::core::compute_graph( + *execution_context, cached_graph->graph, cached_graph->host_plan, + "Fun-ASR-Nano adaptor"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Fun-ASR-Nano adaptor graph execution failed"); + } + + FunAsrNanoAdaptorEmbeddings output; + output.values = engine::core::read_tensor_f32(cached_graph->output.tensor); + output.tokens = valid_frames; + output.hidden_size = config.d_model; + if (capture_stages) { + output.stages.reserve(cached_graph->checkpoints.size()); + for (const auto &checkpoint : cached_graph->checkpoints) { + output.stages.push_back( + {checkpoint.first, + engine::core::read_tensor_f32(checkpoint.second.tensor)}); + } + } + engine::debug::timing_log_scalar( + "fun_asr_nano.adaptor_ms", + engine::debug::elapsed_ms(adapt_start, Clock::now())); + return output; + } + + std::shared_ptr assets; + engine::core::ExecutionContext *execution_context = nullptr; + size_t graph_arena_bytes = 0; + std::unique_ptr weights; + std::unique_ptr cached_graph; +}; + +FunAsrNanoAdaptorRuntime::FunAsrNanoAdaptorRuntime( + std::shared_ptr assets, + engine::core::ExecutionContext &execution_context, size_t graph_arena_bytes, + engine::assets::TensorStorageType weight_storage) + : impl_(std::make_unique(std::move(assets), execution_context, + graph_arena_bytes, weight_storage)) {} + +FunAsrNanoAdaptorRuntime::~FunAsrNanoAdaptorRuntime() = default; +FunAsrNanoAdaptorRuntime::FunAsrNanoAdaptorRuntime( + FunAsrNanoAdaptorRuntime &&) noexcept = default; +FunAsrNanoAdaptorRuntime &FunAsrNanoAdaptorRuntime::operator=( + FunAsrNanoAdaptorRuntime &&) noexcept = default; + +void FunAsrNanoAdaptorRuntime::prepare_capacity(int64_t valid_frames) { + if (impl_ == nullptr) { + throw std::runtime_error("Fun-ASR-Nano adaptor runtime is moved from"); + } + impl_->ensure_graph(valid_frames); +} + +FunAsrNanoAdaptorEmbeddings FunAsrNanoAdaptorRuntime::adapt( + const FunAsrNanoEncoderEmbeddings &encoder_embeddings, + const std::vector &mask, bool capture_stages) { + if (impl_ == nullptr) { + throw std::runtime_error("Fun-ASR-Nano adaptor runtime is moved from"); + } + return impl_->adapt(encoder_embeddings, mask, capture_stages); +} + +} // namespace engine::models::fun_asr_nano diff --git a/src/models/fun_asr_nano/assets.cpp b/src/models/fun_asr_nano/assets.cpp new file mode 100644 index 00000000..3e320cd0 --- /dev/null +++ b/src/models/fun_asr_nano/assets.cpp @@ -0,0 +1,260 @@ +#include "engine/models/fun_asr_nano/assets.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/io/json.h" +#include "engine/framework/model_spec/package.h" + +#include +#include +#include + +namespace engine::models::fun_asr_nano { +namespace json = engine::io::json; +namespace { + +FunAsrNanoEncoderConfig parse_encoder(const json::Value &value) { + FunAsrNanoEncoderConfig config; + config.num_mel_bins = json::require_i64(value, "num_mel_bins"); + config.num_stacked_frames = json::optional_i64(value, "num_stacked_frames", + config.num_stacked_frames); + config.input_size = config.num_mel_bins * config.num_stacked_frames; + config.d_model = json::require_i64(value, "d_model"); + config.attention_heads = json::require_i64(value, "encoder_attention_heads"); + config.ffn_dim = json::require_i64(value, "encoder_ffn_dim"); + config.layers = json::require_i64(value, "encoder_layers"); + config.timestamp_prediction_layers = + json::require_i64(value, "num_timestamp_prediction_blocks"); + config.kernel_size = json::require_i64(value, "kernel_size"); + config.max_position_embeddings = json::optional_i64( + value, "max_position_embeddings", config.max_position_embeddings); + config.activation = + json::optional_string(value, "activation_function", config.activation); + if (config.num_mel_bins <= 0 || config.num_stacked_frames <= 0 || + config.d_model <= 0 || config.attention_heads <= 0 || + config.ffn_dim <= 0 || config.layers <= 0 || + config.timestamp_prediction_layers < 0 || config.kernel_size <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano encoder dimensions must be positive"); + } + if (config.input_size != 560 || config.d_model != 512 || + config.attention_heads != 4 || config.ffn_dim != 2048 || + config.layers != 50 || config.timestamp_prediction_layers != 20 || + config.kernel_size != 11 || config.max_position_embeddings != 2049 || + config.activation != "relu") { + throw std::runtime_error( + "Fun-ASR-Nano config does not match the published encoder " + "architecture: 560-wide ReLU encoder required"); + } + if (config.d_model % config.attention_heads != 0) { + throw std::runtime_error( + "Fun-ASR-Nano encoder width must be divisible by attention heads"); + } + return config; +} + +FunAsrNanoTextConfig parse_text(const json::Value &value) { + FunAsrNanoTextConfig config; + config.vocab_size = json::require_i64(value, "vocab_size"); + config.hidden_size = json::require_i64(value, "hidden_size"); + config.intermediate_size = json::require_i64(value, "intermediate_size"); + config.layers = json::require_i64(value, "num_hidden_layers"); + config.attention_heads = json::require_i64(value, "num_attention_heads"); + config.key_value_heads = json::require_i64(value, "num_key_value_heads"); + if (config.attention_heads <= 0) { + throw std::runtime_error("Fun-ASR-Nano text dimensions must be positive"); + } + config.head_dim = json::optional_i64( + value, "head_dim", config.hidden_size / config.attention_heads); + config.max_position_embeddings = + json::require_i64(value, "max_position_embeddings"); + config.bos_token_id = + json::optional_i64(value, "bos_token_id", config.bos_token_id); + config.eos_token_id = + json::optional_i64(value, "eos_token_id", config.eos_token_id); + config.rms_norm_eps = + json::optional_f32(value, "rms_norm_eps", config.rms_norm_eps); + config.tie_word_embeddings = json::optional_bool(value, "tie_word_embeddings", + config.tie_word_embeddings); + const auto *rope = value.find("rope_parameters"); + if (rope != nullptr && rope->is_object()) { + config.rope_theta = + json::optional_f32(*rope, "rope_theta", config.rope_theta); + } else { + config.rope_theta = + json::optional_f32(value, "rope_theta", config.rope_theta); + } + if (config.hidden_size <= 0 || config.intermediate_size <= 0 || + config.layers <= 0 || config.attention_heads <= 0 || + config.key_value_heads <= 0 || config.head_dim <= 0 || + config.vocab_size <= 0 || config.max_position_embeddings <= 0) { + throw std::runtime_error("Fun-ASR-Nano text dimensions must be positive"); + } + if (config.vocab_size != 151936 || config.hidden_size != 1024 || + config.intermediate_size != 3072 || config.layers != 28 || + config.attention_heads != 16 || config.key_value_heads != 8 || + config.head_dim != 128 || config.max_position_embeddings != 40960 || + config.rope_theta != 1000000.0F) { + throw std::runtime_error( + "Fun-ASR-Nano config does not match the published Qwen architecture"); + } + if (config.attention_heads % config.key_value_heads != 0) { + throw std::runtime_error( + "Fun-ASR-Nano query heads must be divisible by key/value heads"); + } + return config; +} + +FunAsrNanoAdaptorConfig parse_adaptor(const json::Value &root, + const FunAsrNanoTextConfig &text) { + FunAsrNanoAdaptorConfig config; + const auto *nested = root.find("adaptor_config"); + if (nested != nullptr && nested->is_object()) { + config.d_model = json::optional_i64(*nested, "d_model", text.hidden_size); + config.attention_heads = json::optional_i64( + *nested, "encoder_attention_heads", config.attention_heads); + config.ffn_dim = + json::optional_i64(*nested, "encoder_ffn_dim", text.hidden_size / 4); + config.layers = + json::optional_i64(*nested, "encoder_layers", config.layers); + config.activation = json::optional_string(*nested, "activation_function", + config.activation); + } else { + config.d_model = text.hidden_size; + config.attention_heads = json::optional_i64( + root, "adaptor_num_attention_heads", config.attention_heads); + config.ffn_dim = text.hidden_size / 4; + config.layers = + json::optional_i64(root, "adaptor_num_hidden_layers", config.layers); + config.activation = + json::optional_string(root, "activation_function", config.activation); + } + if (config.d_model != text.hidden_size) { + throw std::runtime_error( + "Fun-ASR-Nano adaptor width must match the text hidden size"); + } + if (config.ffn_dim != text.hidden_size / 4) { + throw std::runtime_error("Fun-ASR-Nano adaptor FFN must equal one quarter " + "of the text hidden size"); + } + if (config.attention_heads != 8 || config.layers != 2 || + config.activation != "relu") { + throw std::runtime_error("Fun-ASR-Nano config does not match the published " + "adaptor architecture"); + } + return config; +} + +FunAsrNanoFrontendConfig +parse_frontend(const assets::ResourceBundle &resources, + const FunAsrNanoEncoderConfig &encoder) { + const auto processor = resources.parse_json("processor_config"); + const auto *nested = processor.find("feature_extractor"); + const auto &value = + nested != nullptr && nested->is_object() ? *nested : processor; + FunAsrNanoFrontendConfig config; + config.sample_rate = + json::optional_i32(value, "sampling_rate", config.sample_rate); + config.feature_size = + json::optional_i64(value, "feature_size", config.feature_size); + config.frame_length_ms = + json::optional_i64(value, "frame_length", config.frame_length_ms); + config.frame_shift_ms = + json::optional_i64(value, "frame_shift", config.frame_shift_ms); + config.lfr_m = json::optional_i64(value, "lfr_m", config.lfr_m); + config.lfr_n = json::optional_i64(value, "lfr_n", config.lfr_n); + config.preemphasis = + json::optional_f32(value, "preemphasis", config.preemphasis); + if (config.sample_rate != 16000 || + config.feature_size != encoder.num_mel_bins || + config.lfr_m != encoder.num_stacked_frames || + config.frame_length_ms != 25 || config.frame_shift_ms != 10 || + config.lfr_n != 6 || config.preemphasis != 0.97F) { + throw std::runtime_error( + "Fun-ASR-Nano config does not match the published frontend"); + } + return config; +} + +FunAsrNanoConfig parse_config(const assets::ResourceBundle &resources) { + const auto root = resources.parse_json("config"); + FunAsrNanoConfig config; + config.model_type = json::require_string(root, "model_type"); + if (config.model_type != "fun_asr_nano") { + throw std::runtime_error( + "Fun-ASR-Nano config has an unexpected model_type"); + } + config.audio_token_id = + json::optional_i64(root, "audio_token_id", config.audio_token_id); + config.encoder = parse_encoder(root.require("encoder_config")); + config.text = parse_text(root.require("text_config")); + config.adaptor = parse_adaptor(root, config.text); + config.projector_hidden_size = + json::optional_i64(root, "projector_hidden_size", + json::optional_i64(root, "adaptor_intermediate_size", + config.projector_hidden_size)); + config.tie_word_embeddings = json::optional_bool( + root, "tie_word_embeddings", config.text.tie_word_embeddings); + if (!config.tie_word_embeddings || !config.text.tie_word_embeddings) { + throw std::runtime_error( + "Fun-ASR-Nano currently requires tied text embeddings"); + } + if (config.projector_hidden_size != 2048) { + throw std::runtime_error("Fun-ASR-Nano projector hidden size must be 2048"); + } + if (config.audio_token_id < 0) { + throw std::runtime_error( + "Fun-ASR-Nano audio token configuration is invalid"); + } + config.frontend = parse_frontend(resources, config.encoder); + const auto generation = resources.parse_json("generation_config"); + config.text.bos_token_id = + json::optional_i64(generation, "bos_token_id", config.text.bos_token_id); + const auto eos_ids = + json::require_i64_array_or_scalar(generation, "eos_token_id"); + if (eos_ids.size() != 1) { + throw std::runtime_error( + "Fun-ASR-Nano currently requires one EOS token id"); + } + config.text.eos_token_id = eos_ids.front(); + return config; +} + +std::shared_ptr +make_assets(assets::ResourceBundle resources) { + for (const char *id : + {"config", "generation_config", "processor_config", "tokenizer_json"}) { + if (!resources.has_file(id)) { + throw std::runtime_error( + std::string("Fun-ASR-Nano is missing required resource: ") + id); + } + } + FunAsrNanoAssets assets; + assets.resources = std::move(resources); + assets.config = parse_config(assets.resources); + assets.model_weights = assets.resources.open_tensor_source("weights"); + for (const char *tensor : { + "model.audio_tower.stem.self_attn.q_proj.weight", + "model.language_model.embed_tokens.weight", + }) { + if (!assets.model_weights->has_tensor(tensor)) { + throw std::runtime_error( + std::string("Fun-ASR-Nano is missing required tensor: ") + tensor); + } + } + return std::make_shared(std::move(assets)); +} + +} // namespace + +std::shared_ptr +load_fun_asr_nano_assets(const std::filesystem::path &model_path) { + return make_assets(engine::model_spec::load_resource_bundle_for_family( + model_path, "fun_asr_nano")); +} + +std::shared_ptr +load_fun_asr_nano_assets(assets::ResourceBundle resources) { + return make_assets(std::move(resources)); +} + +} // namespace engine::models::fun_asr_nano diff --git a/src/models/fun_asr_nano/decoder.cpp b/src/models/fun_asr_nano/decoder.cpp new file mode 100644 index 00000000..0d841c30 --- /dev/null +++ b/src/models/fun_asr_nano/decoder.cpp @@ -0,0 +1,751 @@ +#include "engine/models/fun_asr_nano/decoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/positional_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/transformers/qwen_causal_decoder.h" +#include "engine/framework/runtime/kv_cache.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::fun_asr_nano { +namespace { + +namespace modules = engine::modules; +using Clock = std::chrono::steady_clock; + +struct GgmlContextDeleter { + void operator()(ggml_context *ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +struct TextLayerWeights { + core::TensorValue input_norm; + core::TensorValue q_proj; + core::TensorValue k_proj; + core::TensorValue v_proj; + core::TensorValue o_proj; + core::TensorValue q_norm; + core::TensorValue k_norm; + core::TensorValue post_norm; + core::TensorValue gate_proj; + core::TensorValue up_proj; + core::TensorValue down_proj; +}; + +struct TextDecoderWeights { + std::shared_ptr store; + core::TensorValue token_embedding; + std::vector layers; + core::TensorValue norm; + core::TensorValue lm_head; +}; + +struct PrefillOutput { + std::vector logits; + runtime::TransformerKVState kv_state; +}; + +modules::QwenDecoderLayerWeights +to_qwen_layer_weights(const TextLayerWeights &weights) { + modules::QwenDecoderLayerWeights out; + out.input_norm = {weights.input_norm, std::nullopt}; + out.self_attention.q_weight = weights.q_proj; + out.self_attention.k_weight = weights.k_proj; + out.self_attention.v_weight = weights.v_proj; + out.self_attention.out_weight = weights.o_proj; + out.q_norm = {weights.q_norm, std::nullopt}; + out.k_norm = {weights.k_norm, std::nullopt}; + out.post_norm = {weights.post_norm, std::nullopt}; + out.mlp.gate_proj = {weights.gate_proj, std::nullopt}; + out.mlp.up_proj = {weights.up_proj, std::nullopt}; + out.mlp.down_proj = {weights.down_proj, std::nullopt}; + return out; +} + +modules::QwenCausalDecoderConfig +make_qwen_decoder_config(const FunAsrNanoTextConfig &config) { + modules::QwenCausalDecoderConfig out; + out.stack.hidden_size = config.hidden_size; + out.stack.num_attention_heads = config.attention_heads; + out.stack.num_key_value_heads = config.key_value_heads; + out.stack.head_dim = config.head_dim; + out.stack.intermediate_size = config.intermediate_size; + out.stack.layers = config.layers; + out.stack.rms_norm_eps = config.rms_norm_eps; + out.stack.rope_theta = config.rope_theta; + out.stack.use_qk_norm = true; + out.stack.runtime.static_cache.update_mode = + modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; + out.logits_size = config.vocab_size; + out.logits_mode = modules::QwenCausalDecoderLogitsMode::LastStep; + return out; +} + +modules::QwenCausalDecoderWeights +make_qwen_decoder_weights(const TextDecoderWeights &weights) { + modules::QwenCausalDecoderWeights out; + out.stack.layers.reserve(weights.layers.size()); + for (const auto &layer : weights.layers) { + out.stack.layers.push_back(to_qwen_layer_weights(layer)); + } + out.final_norm = {weights.norm, std::nullopt}; + out.lm_head = {weights.lm_head, std::nullopt}; + return out; +} + +int64_t head_dim(const FunAsrNanoTextConfig &config) { + if (config.attention_heads <= 0 || config.key_value_heads <= 0 || + config.head_dim <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano decoder attention config is invalid"); + } + return config.head_dim; +} + +core::TensorValue +prompt_embeddings(core::ModuleBuildContext &ctx, + const TextDecoderWeights &weights, + const FunAsrNanoTextConfig &config, ggml_tensor *token_ids, + ggml_tensor *audio_embeddings, ggml_tensor *audio_positions, + int64_t prompt_steps, int64_t audio_tokens) { + auto ids = core::wrap_tensor( + token_ids, core::TensorShape::from_dims({prompt_steps}), GGML_TYPE_I32); + auto x = modules::EmbeddingModule({config.vocab_size, config.hidden_size}) + .build(ctx, ids, weights.token_embedding); + if (audio_tokens > 0) { + auto audio = core::wrap_tensor( + audio_embeddings, + core::TensorShape::from_dims({audio_tokens, config.hidden_size}), + GGML_TYPE_F32); + auto positions = core::wrap_tensor( + audio_positions, core::TensorShape::from_dims({audio_tokens}), + GGML_TYPE_I64); + x = core::wrap_tensor( + ggml_set_rows(ctx.ggml, x.tensor, audio.tensor, positions.tensor), + x.shape, GGML_TYPE_F32); + } + return core::reshape_tensor( + ctx, x, + core::TensorShape::from_dims({1, prompt_steps, config.hidden_size})); +} + +TextDecoderWeights load_weights(const FunAsrNanoAssets &assets, + ggml_backend_t backend, + core::BackendType backend_type, + size_t weight_context_bytes, + assets::TensorStorageType storage_type) { + const auto &config = assets.config.text; + const auto &source = *assets.model_weights; + TextDecoderWeights weights; + weights.store = std::make_shared( + backend, backend_type, "fun_asr_nano.decoder.weights", + weight_context_bytes); + weights.token_embedding = weights.store->load_tensor( + source, "model.language_model.embed_tokens.weight", storage_type, + {config.vocab_size, config.hidden_size}); + weights.layers.reserve(static_cast(config.layers)); + const int64_t dim = head_dim(config); + for (int64_t layer = 0; layer < config.layers; ++layer) { + const std::string prefix = + "model.language_model.layers." + std::to_string(layer); + TextLayerWeights w; + w.input_norm = weights.store->load_f32_tensor( + source, prefix + ".input_layernorm.weight", {config.hidden_size}); + w.q_proj = weights.store->load_tensor( + source, prefix + ".self_attn.q_proj.weight", storage_type, + {config.attention_heads * dim, config.hidden_size}); + w.k_proj = weights.store->load_tensor( + source, prefix + ".self_attn.k_proj.weight", storage_type, + {config.key_value_heads * dim, config.hidden_size}); + w.v_proj = weights.store->load_tensor( + source, prefix + ".self_attn.v_proj.weight", storage_type, + {config.key_value_heads * dim, config.hidden_size}); + w.o_proj = weights.store->load_tensor( + source, prefix + ".self_attn.o_proj.weight", storage_type, + {config.hidden_size, config.attention_heads * dim}); + w.q_norm = weights.store->load_f32_tensor( + source, prefix + ".self_attn.q_norm.weight", {dim}); + w.k_norm = weights.store->load_f32_tensor( + source, prefix + ".self_attn.k_norm.weight", {dim}); + w.post_norm = weights.store->load_f32_tensor( + source, prefix + ".post_attention_layernorm.weight", + {config.hidden_size}); + w.gate_proj = weights.store->load_tensor( + source, prefix + ".mlp.gate_proj.weight", storage_type, + {config.intermediate_size, config.hidden_size}); + w.up_proj = weights.store->load_tensor( + source, prefix + ".mlp.up_proj.weight", storage_type, + {config.intermediate_size, config.hidden_size}); + w.down_proj = weights.store->load_tensor( + source, prefix + ".mlp.down_proj.weight", storage_type, + {config.hidden_size, config.intermediate_size}); + weights.layers.push_back(std::move(w)); + } + weights.norm = weights.store->load_f32_tensor( + source, "model.language_model.norm.weight", {config.hidden_size}); + weights.lm_head = weights.token_embedding; + weights.store->upload(); + return weights; +} + +int32_t argmax_index(const std::vector &values) { + if (values.empty()) { + throw std::runtime_error( + "Fun-ASR-Nano decoder cannot select from empty logits"); + } + size_t best = 0; + for (size_t i = 1; i < values.size(); ++i) { + if (values[i] > values[best]) { + best = i; + } + } + return static_cast(best); +} + +bool is_eos(const FunAsrNanoTextConfig &config, int32_t token) { + return token == config.eos_token_id; +} + +std::shared_ptr +require_assets(std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("Fun-ASR-Nano decoder requires assets"); + } + return assets; +} + +class TextDecoderWeightsRuntime { +public: + TextDecoderWeightsRuntime(std::shared_ptr assets, + core::ExecutionContext &execution, + size_t weight_context_bytes, + assets::TensorStorageType storage_type) + : assets_(require_assets(std::move(assets))), + backend_(execution.backend()), backend_type_(execution.backend_type()), + threads_(std::max(1, execution.config().threads)), + weights_(std::make_shared( + load_weights(*assets_, backend_, backend_type_, + weight_context_bytes, storage_type))) { + if (backend_ == nullptr) { + throw std::runtime_error( + "Fun-ASR-Nano decoder backend is not initialized"); + } + } + + const FunAsrNanoAssets &assets() const noexcept { return *assets_; } + + const TextDecoderWeights &weights() const noexcept { return *weights_; } + + ggml_backend_t backend() const noexcept { return backend_; } + + core::BackendType backend_type() const noexcept { return backend_type_; } + + int threads() const noexcept { return threads_; } + +private: + std::shared_ptr assets_; + ggml_backend_t backend_ = nullptr; + core::BackendType backend_type_ = core::BackendType::Cpu; + int threads_ = 1; + std::shared_ptr weights_; +}; + +class PrefillGraph { +public: + PrefillGraph(std::shared_ptr runtime, + int64_t prompt_steps, int64_t audio_tokens, + size_t graph_arena_bytes) + : runtime_(std::move(runtime)), prompt_steps_(prompt_steps), + audio_tokens_(audio_tokens) { + if (prompt_steps_ <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano decoder prefill requires positive prompt length"); + } + if (audio_tokens_ < 0 || audio_tokens_ > prompt_steps_) { + throw std::runtime_error( + "Fun-ASR-Nano decoder prefill audio token count is invalid"); + } + const auto build_start = Clock::now(); + ggml_init_params params{graph_arena_bytes, nullptr, true}; + ctx_.reset(ggml_init(params)); + if (ctx_ == nullptr) { + throw std::runtime_error( + "failed to initialize Fun-ASR-Nano decoder prefill graph context"); + } + const auto &config = runtime_->assets().config.text; + const auto &weights = runtime_->weights(); + core::ModuleBuildContext ctx{ctx_.get(), "fun_asr_nano.decoder.prefill", + runtime_->backend_type()}; + token_ids_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, prompt_steps_); + audio_embeddings_ = + ggml_new_tensor_2d(ctx_.get(), GGML_TYPE_F32, config.hidden_size, + std::max(audio_tokens_, 1)); + audio_positions_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I64, + std::max(audio_tokens_, 1)); + auto x = + prompt_embeddings(ctx, weights, config, token_ids_, audio_embeddings_, + audio_positions_, prompt_steps_, audio_tokens_); + positions_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, prompt_steps_); + auto positions = core::wrap_tensor( + positions_, core::TensorShape::from_dims({prompt_steps_}), + GGML_TYPE_I32); + + auto decoder_out = + modules::QwenCausalDecoderModule(make_qwen_decoder_config(config)) + .build(ctx, x, positions, make_qwen_decoder_weights(weights)); + for (const auto &layer : decoder_out.state.layers) { + if (!layer.key.has_value() || !layer.value.has_value()) { + throw std::runtime_error( + "Fun-ASR-Nano decoder prefill decoder did not return K/V state"); + } + auto *key_readback = + ggml_cpy(ctx_.get(), layer.key->tensor, + ggml_dup_tensor(ctx_.get(), layer.key->tensor)); + auto *value_readback = + ggml_cpy(ctx_.get(), layer.value->tensor, + ggml_dup_tensor(ctx_.get(), layer.value->tensor)); + ggml_set_output(key_readback); + ggml_set_output(value_readback); + keys_.push_back(key_readback); + values_.push_back(value_readback); + } + logits_ = decoder_out.logits.tensor; + ggml_set_output(logits_); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); + for (auto *key : keys_) { + ggml_build_forward_expand(graph_, key); + } + for (auto *value : values_) { + ggml_build_forward_expand(graph_, value); + } + ggml_build_forward_expand(graph_, logits_); + gallocr_ = ggml_gallocr_new( + ggml_backend_get_default_buffer_type(runtime_->backend())); + if (gallocr_ == nullptr || !ggml_gallocr_reserve(gallocr_, graph_) || + !ggml_gallocr_alloc_graph(gallocr_, graph_)) { + throw std::runtime_error( + "failed to allocate Fun-ASR-Nano decoder prefill graph"); + } + const auto pos = modules::qwen_position_ids(prompt_steps_); + ggml_backend_tensor_set(positions_, pos.data(), 0, + pos.size() * sizeof(int32_t)); + debug::timing_log_scalar( + "fun_asr_nano.decoder.prefill.graph.build_ms", + engine::debug::elapsed_ms(build_start, Clock::now())); + debug::trace_log_scalar("fun_asr_nano.decoder.prefill_prompt_steps", + prompt_steps_); + } + + ~PrefillGraph() { + engine::core::release_backend_graph_resources(runtime_->backend(), graph_); + if (gallocr_ != nullptr) { + ggml_gallocr_free(gallocr_); + } + } + + bool matches(const TextDecoderWeightsRuntime &runtime, int64_t prompt_steps, + int64_t audio_tokens) const { + return runtime_.get() == &runtime && prompt_steps_ == prompt_steps && + audio_tokens_ == audio_tokens; + } + + PrefillOutput run(const std::vector &token_ids, + const std::vector &audio_embeddings, + const std::vector &audio_positions) { + const auto &config = runtime_->assets().config.text; + if (static_cast(token_ids.size()) != prompt_steps_) { + throw std::runtime_error( + "Fun-ASR-Nano decoder prefill token id count mismatch"); + } + if (static_cast(audio_embeddings.size()) != + audio_tokens_ * config.hidden_size) { + throw std::runtime_error( + "Fun-ASR-Nano decoder prefill audio embedding size mismatch"); + } + if (static_cast(audio_positions.size()) != audio_tokens_) { + throw std::runtime_error( + "Fun-ASR-Nano decoder prefill audio position count mismatch"); + } + auto timing_start = Clock::now(); + ggml_backend_tensor_set(token_ids_, token_ids.data(), 0, + token_ids.size() * sizeof(int32_t)); + if (audio_tokens_ > 0) { + std::vector positions(audio_positions.begin(), + audio_positions.end()); + ggml_backend_tensor_set(audio_embeddings_, audio_embeddings.data(), 0, + audio_embeddings.size() * sizeof(float)); + ggml_backend_tensor_set(audio_positions_, positions.data(), 0, + positions.size() * sizeof(int64_t)); + } + debug::timing_log_scalar( + "fun_asr_nano.decoder.prefill_input_upload_ms", + engine::debug::elapsed_ms(timing_start, Clock::now())); + core::set_backend_threads(runtime_->backend(), runtime_->threads()); + timing_start = Clock::now(); + const ggml_status status = + engine::core::compute_backend_graph(runtime_->backend(), graph_); + ggml_backend_synchronize(runtime_->backend()); + debug::timing_log_scalar( + "fun_asr_nano.decoder.prefill.graph.compute_ms", + engine::debug::elapsed_ms(timing_start, Clock::now())); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error( + "Fun-ASR-Nano decoder prefill graph compute failed"); + } + PrefillOutput out; + out.logits.resize(static_cast(config.vocab_size)); + timing_start = Clock::now(); + ggml_backend_tensor_get(logits_, out.logits.data(), 0, + out.logits.size() * sizeof(float)); + out.kv_state.current_end = prompt_steps_; + out.kv_state.layers.resize(keys_.size()); + const size_t layer_values = static_cast( + prompt_steps_ * config.key_value_heads * head_dim(config)); + for (size_t layer = 0; layer < keys_.size(); ++layer) { + auto &state = out.kv_state.layers[layer]; + state.valid_steps = prompt_steps_; + state.key = core::read_tensor_f32(keys_[layer]); + state.value = core::read_tensor_f32(values_[layer]); + if (state.key.size() != layer_values || + state.value.size() != layer_values) { + throw std::runtime_error( + "Fun-ASR-Nano decoder prefill K/V export size mismatch"); + } + } + debug::timing_log_scalar( + "fun_asr_nano.decoder.prefill_output_read_ms", + engine::debug::elapsed_ms(timing_start, Clock::now())); + return out; + } + +private: + std::shared_ptr runtime_; + int64_t prompt_steps_ = 0; + int64_t audio_tokens_ = 0; + std::unique_ptr ctx_; + ggml_tensor *token_ids_ = nullptr; + ggml_tensor *audio_embeddings_ = nullptr; + ggml_tensor *audio_positions_ = nullptr; + ggml_tensor *positions_ = nullptr; + ggml_tensor *logits_ = nullptr; + std::vector keys_; + std::vector values_; + ggml_cgraph *graph_ = nullptr; + ggml_gallocr_t gallocr_ = nullptr; +}; + +class DecodeGraph { +public: + DecodeGraph(std::shared_ptr runtime, + int64_t cache_steps, size_t graph_arena_bytes) + : runtime_(std::move(runtime)), real_cache_steps_(cache_steps), + cache_steps_(cache_steps) { + if (real_cache_steps_ <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano decoder decode requires positive cache length"); + } + const auto build_start = Clock::now(); + ggml_init_params params{graph_arena_bytes, nullptr, true}; + ctx_.reset(ggml_init(params)); + if (ctx_ == nullptr) { + throw std::runtime_error( + "failed to initialize Fun-ASR-Nano decoder decode graph context"); + } + const auto &config = runtime_->assets().config.text; + const auto &weights = runtime_->weights(); + core::ModuleBuildContext ctx{ctx_.get(), "fun_asr_nano.decoder.decode", + runtime_->backend_type()}; + token_id_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, 1); + ggml_set_input(token_id_); + auto token_id = core::wrap_tensor( + token_id_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + auto x = modules::EmbeddingModule({config.vocab_size, config.hidden_size}) + .build(ctx, token_id, weights.token_embedding); + x = core::reshape_tensor( + ctx, x, core::TensorShape::from_dims({1, 1, config.hidden_size})); + positions_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, 1); + ggml_set_input(positions_); + auto positions = core::wrap_tensor( + positions_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + cache_slot_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, 1); + ggml_set_input(cache_slot_); + auto cache_slot = core::wrap_tensor( + cache_slot_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + attention_mask_ = + ggml_new_tensor_4d(ctx_.get(), GGML_TYPE_F16, cache_steps_, 1, 1, 1); + ggml_set_input(attention_mask_); + auto attention_mask = core::wrap_tensor( + attention_mask_, core::TensorShape::from_dims({1, 1, 1, cache_steps_}), + GGML_TYPE_F16); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); + auto decoder_out = + modules::QwenCausalDecoderModule(make_qwen_decoder_config(config)) + .build_static_cache_tail(ctx, graph_, x, positions, + make_qwen_decoder_weights(weights), + cache_steps_, attention_mask, cache_slot); + step_cache_ = std::move(decoder_out.cache); + logits_ = decoder_out.logits.tensor; + ggml_set_output(logits_); + ggml_build_forward_expand(graph_, logits_); + buffer_ = ggml_backend_alloc_ctx_tensors(ctx_.get(), runtime_->backend()); + if (buffer_ == nullptr) { + throw std::runtime_error( + "failed to allocate Fun-ASR-Nano decoder decode graph"); + } + attention_mask_values_.assign(static_cast(cache_steps_), + ggml_fp32_to_fp16(-INFINITY)); + debug::timing_log_scalar( + "fun_asr_nano.decoder.decode.graph.build_ms", + engine::debug::elapsed_ms(build_start, Clock::now())); + debug::trace_log_scalar("fun_asr_nano.decoder.decode_cache_steps", + real_cache_steps_); + } + + ~DecodeGraph() { + engine::core::release_backend_graph_resources(runtime_->backend(), graph_); + if (buffer_ != nullptr) { + ggml_backend_buffer_free(buffer_); + } + } + + bool can_run(const TextDecoderWeightsRuntime &runtime, + int64_t required_steps) const { + return runtime_.get() == &runtime && real_cache_steps_ >= required_steps; + } + + void import_state(const runtime::TransformerKVState &state) { + step_cache_.import_state(state); + } + + std::vector run_step(int32_t token) { + const auto &config = runtime_->assets().config.text; + if (step_cache_.valid_steps() >= real_cache_steps_) { + throw std::runtime_error("Fun-ASR-Nano decoder decode cache exhausted"); + } + ggml_backend_tensor_set(token_id_, &token, 0, sizeof(int32_t)); + const int32_t position = static_cast(step_cache_.current_end()); + ggml_backend_tensor_set(positions_, &position, 0, sizeof(int32_t)); + const int32_t cache_slot = static_cast(step_cache_.valid_steps()); + ggml_backend_tensor_set(cache_slot_, &cache_slot, 0, sizeof(int32_t)); + modules::write_qwen_cached_step_mask( + attention_mask_, attention_mask_values_, cache_steps_, + step_cache_.valid_steps(), step_cache_.valid_steps()); + core::set_backend_threads(runtime_->backend(), runtime_->threads()); + const ggml_status status = + engine::core::compute_backend_graph(runtime_->backend(), graph_); + ggml_backend_synchronize(runtime_->backend()); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error( + "Fun-ASR-Nano decoder decode graph compute failed"); + } + std::vector logits(static_cast(config.vocab_size)); + ggml_backend_tensor_get(logits_, logits.data(), 0, + logits.size() * sizeof(float)); + step_cache_.advance_after_direct_append(1); + return logits; + } + +private: + std::shared_ptr runtime_; + int64_t real_cache_steps_ = 0; + int64_t cache_steps_ = 0; + std::unique_ptr ctx_; + ggml_tensor *token_id_ = nullptr; + ggml_tensor *positions_ = nullptr; + ggml_tensor *cache_slot_ = nullptr; + ggml_tensor *attention_mask_ = nullptr; + ggml_tensor *logits_ = nullptr; + std::vector attention_mask_values_; + runtime::TransformerKVCache step_cache_; + ggml_cgraph *graph_ = nullptr; + ggml_backend_buffer_t buffer_ = nullptr; +}; + +} // namespace + +struct FunAsrNanoDecoderRuntime::Impl { + Impl(std::shared_ptr assets, + core::ExecutionContext &execution, size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, size_t weight_context_bytes, + assets::TensorStorageType storage_type) + : weights(std::make_shared( + std::move(assets), execution, weight_context_bytes, storage_type)), + prefill_graph_arena_bytes(prefill_graph_arena_bytes), + decode_graph_arena_bytes(decode_graph_arena_bytes) {} + + void validate_prompt_audio( + const FunAsrNanoPrompt &prompt, + const FunAsrNanoAdaptorEmbeddings &audio_embeddings) const { + const auto &config = weights->assets().config.text; + if (audio_embeddings.hidden_size != config.hidden_size) { + throw std::runtime_error( + "Fun-ASR-Nano audio embedding hidden size mismatch"); + } + if (audio_embeddings.tokens != + static_cast(prompt.audio_token_positions.size())) { + throw std::runtime_error("Fun-ASR-Nano audio embedding token count does " + "not match prompt placeholders"); + } + if (static_cast(audio_embeddings.values.size()) != + audio_embeddings.tokens * config.hidden_size) { + throw std::runtime_error( + "Fun-ASR-Nano audio embedding value count mismatch"); + } + std::vector expected_audio_positions; + for (size_t index = 0; index < prompt.input_ids.size(); ++index) { + if (prompt.input_ids[index] == weights->assets().config.audio_token_id) { + expected_audio_positions.push_back(static_cast(index)); + } + } + if (prompt.audio_token_positions != expected_audio_positions) { + throw std::runtime_error( + "Fun-ASR-Nano audio positions must cover every audio placeholder in " + "prompt order"); + } + if (prompt.attention_mask.size() != prompt.input_ids.size() || + std::any_of(prompt.attention_mask.begin(), prompt.attention_mask.end(), + [](int32_t value) { return value != 1; })) { + throw std::runtime_error( + "Fun-ASR-Nano decoder requires an unpadded prompt mask"); + } + } + + FunAsrNanoGeneratedTokens + generate(const FunAsrNanoPrompt &prompt, + const FunAsrNanoAdaptorEmbeddings &audio_embeddings, + const FunAsrNanoGenerationOptions &options, + const FunAsrNanoTokenCallback &token_callback) { + const auto &config = weights->assets().config.text; + if (prompt.input_ids.empty()) { + throw std::runtime_error("Fun-ASR-Nano decoder prompt is empty"); + } + if (options.max_new_tokens <= 0) { + throw std::runtime_error("Fun-ASR-Nano max_new_tokens must be positive"); + } + const int64_t prompt_steps = static_cast(prompt.input_ids.size()); + if (prompt_steps > config.max_position_embeddings || + options.max_new_tokens > + config.max_position_embeddings - prompt_steps) { + throw std::runtime_error( + "Fun-ASR-Nano decoder request exceeds max_position_embeddings"); + } + auto timing_start = Clock::now(); + validate_prompt_audio(prompt, audio_embeddings); + debug::timing_log_scalar( + "fun_asr_nano.decoder.prompt_prepare_ms", + engine::debug::elapsed_ms(timing_start, Clock::now())); + if (prefill_graph == nullptr || + !prefill_graph->matches(*weights, prompt_steps, + audio_embeddings.tokens)) { + prefill_graph = std::make_unique(weights, prompt_steps, + audio_embeddings.tokens, + prefill_graph_arena_bytes); + } else { + debug::timing_log_scalar("fun_asr_nano.decoder.prefill.graph.build_ms", + 0.0); + debug::trace_log_scalar("fun_asr_nano.decoder.prefill_prompt_steps", + prompt_steps); + } + timing_start = Clock::now(); + auto prefill = prefill_graph->run(prompt.input_ids, audio_embeddings.values, + prompt.audio_token_positions); + debug::timing_log_scalar( + "fun_asr_nano.decoder.prefill_total_ms", + engine::debug::elapsed_ms(timing_start, Clock::now())); + const int64_t required_cache_steps = + prompt_steps + std::max(options.max_new_tokens - 1, 0); + if (required_cache_steps > prompt_steps) { + if (decode_graph == nullptr || + !decode_graph->can_run(*weights, required_cache_steps)) { + decode_graph = std::make_unique( + weights, required_cache_steps, decode_graph_arena_bytes); + } else { + debug::timing_log_scalar("fun_asr_nano.decoder.decode.graph.build_ms", + 0.0); + debug::trace_log_scalar("fun_asr_nano.decoder.decode_cache_steps", + required_cache_steps); + } + decode_graph->import_state(prefill.kv_state); + } + + FunAsrNanoGeneratedTokens out; + std::vector logits = std::move(prefill.logits); + timing_start = Clock::now(); + for (int64_t step = 0; step < options.max_new_tokens; ++step) { + if (options.capture_logits) { + out.step_logits.push_back(logits); + } + const int32_t token = argmax_index(logits); + if (is_eos(config, token)) { + break; + } + out.token_ids.push_back(token); + if (token_callback) { + token_callback(out); + } + if (step + 1 >= options.max_new_tokens) { + break; + } + if (decode_graph == nullptr) { + throw std::runtime_error( + "Fun-ASR-Nano decoder decode graph is not initialized"); + } + logits = decode_graph->run_step(token); + } + debug::timing_log_scalar( + "fun_asr_nano.decoder.decode_total_ms", + engine::debug::elapsed_ms(timing_start, Clock::now())); + return out; + } + + std::shared_ptr weights; + size_t prefill_graph_arena_bytes = 0; + size_t decode_graph_arena_bytes = 0; + std::unique_ptr prefill_graph; + std::unique_ptr decode_graph; +}; + +FunAsrNanoDecoderRuntime::FunAsrNanoDecoderRuntime( + std::shared_ptr assets, + core::ExecutionContext &execution, size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, size_t weight_context_bytes, + assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique(std::move(assets), execution, + prefill_graph_arena_bytes, + decode_graph_arena_bytes, + weight_context_bytes, weight_storage_type)) { +} + +FunAsrNanoDecoderRuntime::~FunAsrNanoDecoderRuntime() = default; + +FunAsrNanoGeneratedTokens FunAsrNanoDecoderRuntime::generate( + const FunAsrNanoPrompt &prompt, + const FunAsrNanoAdaptorEmbeddings &audio_embeddings, + const FunAsrNanoGenerationOptions &options, + const FunAsrNanoTokenCallback &token_callback) { + return impl_->generate(prompt, audio_embeddings, options, token_callback); +} + +} // namespace engine::models::fun_asr_nano diff --git a/src/models/fun_asr_nano/encoder.cpp b/src/models/fun_asr_nano/encoder.cpp new file mode 100644 index 00000000..9075877a --- /dev/null +++ b/src/models/fun_asr_nano/encoder.cpp @@ -0,0 +1,401 @@ +#include "engine/models/fun_asr_nano/encoder.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/speech_encoders/sanm.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace engine::models::fun_asr_nano { +namespace { + +using Clock = std::chrono::steady_clock; + +constexpr size_t kEncoderGraphNodes = 131072; +constexpr size_t kWeightContextBytes = 32 * 1024 * 1024; +constexpr float kLayerNormEpsilon = 1.0e-5F; + +engine::modules::LinearWeights +load_linear(engine::core::BackendWeightStore &store, + const engine::assets::TensorSource &source, + const std::string &prefix, int64_t input_size, int64_t output_size, + engine::assets::TensorStorageType storage_type) { + return { + store.load_tensor(source, prefix + ".weight", storage_type, + {output_size, input_size}), + store.load_f32_tensor(source, prefix + ".bias", {output_size}), + }; +} + +engine::modules::NormWeights +load_norm(engine::core::BackendWeightStore &store, + const engine::assets::TensorSource &source, const std::string &prefix, + int64_t size) { + return { + store.load_f32_tensor(source, prefix + ".weight", {size}), + store.load_f32_tensor(source, prefix + ".bias", {size}), + }; +} + +engine::modules::SanmBlockWeightsView +load_sanm_block(engine::core::BackendWeightStore &store, + const engine::assets::TensorSource &source, + const std::string &prefix, int64_t input_size, + const FunAsrNanoEncoderConfig &config, + engine::assets::TensorStorageType storage_type) { + return { + load_norm(store, source, prefix + ".self_attn_layer_norm", input_size), + load_linear(store, source, prefix + ".self_attn.q_proj", input_size, + config.d_model, storage_type), + load_linear(store, source, prefix + ".self_attn.k_proj", input_size, + config.d_model, storage_type), + load_linear(store, source, prefix + ".self_attn.v_proj", input_size, + config.d_model, storage_type), + load_linear(store, source, prefix + ".self_attn.out_proj", config.d_model, + config.d_model, storage_type), + store.load_f32_tensor(source, prefix + ".fsmn.conv.weight", + {config.d_model, 1, config.kernel_size}), + load_norm(store, source, prefix + ".final_layer_norm", config.d_model), + load_linear(store, source, prefix + ".fc1", config.d_model, + config.ffn_dim, storage_type), + load_linear(store, source, prefix + ".fc2", config.ffn_dim, + config.d_model, storage_type), + }; +} + +struct EncoderWeights { + std::unique_ptr store; + engine::modules::SanmBlockWeightsView stem; + std::vector main_layers; + engine::modules::NormWeights main_norm; + std::vector timestamp_layers; + engine::modules::NormWeights timestamp_norm; +}; + +std::unique_ptr +load_encoder_weights(const FunAsrNanoAssets &assets, + engine::core::ExecutionContext &execution_context, + engine::assets::TensorStorageType storage_type) { + auto weights = std::make_unique(); + weights->store = std::make_unique( + execution_context.backend(), execution_context.backend_type(), + "Fun-ASR-Nano encoder weights", kWeightContextBytes); + const auto &source = *assets.model_weights; + const auto &config = assets.config.encoder; + const std::string root = "model.audio_tower."; + + weights->stem = load_sanm_block(*weights->store, source, root + "stem", + config.input_size, config, storage_type); + weights->main_layers.reserve(static_cast(config.layers - 1)); + for (int64_t index = 0; index < config.layers - 1; ++index) { + weights->main_layers.push_back(load_sanm_block( + *weights->store, source, root + "layers." + std::to_string(index), + config.d_model, config, storage_type)); + } + weights->main_norm = + load_norm(*weights->store, source, root + "layer_norm", config.d_model); + weights->timestamp_layers.reserve( + static_cast(config.timestamp_prediction_layers)); + for (int64_t index = 0; index < config.timestamp_prediction_layers; ++index) { + weights->timestamp_layers.push_back(load_sanm_block( + *weights->store, source, + root + "timestamp_prediction_layers." + std::to_string(index), + config.d_model, config, storage_type)); + } + weights->timestamp_norm = + load_norm(*weights->store, source, + root + "timestamp_prediction_layer_norm", config.d_model); + weights->store->upload(); + return weights; +} + +std::vector make_sinusoidal_positions(int64_t frames, int64_t channels) { + if (frames <= 0 || channels <= 2 || channels % 2 != 0) { + throw std::runtime_error( + "Fun-ASR-Nano sinusoidal position shape is invalid"); + } + const int64_t half = channels / 2; + const float increment = std::log(10000.0F) / static_cast(half - 1); + std::vector values(static_cast(frames * channels), 0.0F); + for (int64_t frame = 0; frame < frames; ++frame) { + const float position = static_cast(frame + 1); + for (int64_t index = 0; index < half; ++index) { + const float inverse_timescale = + std::exp(-increment * static_cast(index)); + const float phase = position * inverse_timescale; + const size_t base = static_cast(frame * channels + index); + values[base] = std::sin(phase); + values[base + static_cast(half)] = std::cos(phase); + } + } + return values; +} + +engine::modules::SanmBlockConfig +block_config(const FunAsrNanoEncoderConfig &config, int64_t input_size) { + engine::modules::SanmBlockConfig result; + result.input_size = input_size; + result.model_size = config.d_model; + result.num_heads = config.attention_heads; + result.ffn_size = config.ffn_dim; + result.fsmn_kernel_size = config.kernel_size; + result.layer_norm_eps = kLayerNormEpsilon; + result.attention_lowering = + engine::modules::ScaledDotProductAttentionLowering::Explicit; + return result; +} + +} // namespace + +struct FunAsrNanoEncoderRuntime::Impl { + struct Graph { + int64_t frames = 0; + ggml_backend_t backend = nullptr; + ggml_context *ggml = nullptr; + ggml_gallocr_t allocator = nullptr; + ggml_cgraph *graph = nullptr; + engine::core::HostGraphPlan host_plan; + engine::core::TensorValue input; + engine::core::TensorValue positions; + engine::core::TensorValue output; + std::vector> checkpoints; + + ~Graph() { + host_plan.reset(); + if (backend != nullptr && graph != nullptr) { + engine::core::release_backend_graph_resources(backend, graph); + } + if (allocator != nullptr) { + ggml_gallocr_free(allocator); + } + if (ggml != nullptr) { + ggml_free(ggml); + } + } + }; + + Impl(std::shared_ptr assets_value, + engine::core::ExecutionContext &execution_context_value, + size_t graph_arena_bytes_value, + engine::assets::TensorStorageType weight_storage) + : assets(std::move(assets_value)), + execution_context(&execution_context_value), + graph_arena_bytes(graph_arena_bytes_value) { + if (assets == nullptr || assets->model_weights == nullptr) { + throw std::runtime_error( + "Fun-ASR-Nano encoder requires model assets and weights"); + } + if (graph_arena_bytes == 0) { + throw std::runtime_error( + "Fun-ASR-Nano encoder graph arena must be non-zero"); + } + weights = load_encoder_weights(*assets, *execution_context, weight_storage); + } + + void ensure_graph(int64_t frames) { + if (frames <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano encoder graph requires positive frames"); + } + const auto &config = assets->config.encoder; + if (frames >= config.max_position_embeddings) { + throw std::runtime_error( + "Fun-ASR-Nano encoder input exceeds positional capacity"); + } + if (cached_graph != nullptr && cached_graph->frames == frames && + cached_graph->backend == execution_context->backend()) { + engine::debug::trace_log_scalar("fun_asr_nano.encoder.graph_cache_hit", + true); + return; + } + + const auto build_start = Clock::now(); + auto next = std::make_unique(); + next->frames = frames; + next->backend = execution_context->backend(); + ggml_init_params params{}; + params.mem_size = graph_arena_bytes; + params.mem_buffer = nullptr; + params.no_alloc = true; + next->ggml = ggml_init(params); + if (next->ggml == nullptr) { + throw std::runtime_error( + "failed to initialize Fun-ASR-Nano encoder graph context"); + } + + engine::core::ModuleBuildContext context{next->ggml, "fun_asr_nano.encoder", + execution_context->backend_type()}; + const auto input_shape = + engine::core::TensorShape::from_dims({1, frames, config.input_size}); + next->input = + engine::core::make_tensor(context, GGML_TYPE_F32, input_shape); + ggml_set_input(next->input.tensor); + ggml_set_output(next->input.tensor); + next->positions = + engine::core::make_tensor(context, GGML_TYPE_F32, input_shape); + ggml_set_input(next->positions.tensor); + ggml_set_output(next->positions.tensor); + + auto hidden = engine::core::wrap_tensor( + ggml_scale(context.ggml, next->input.tensor, + std::sqrt(static_cast(config.d_model))), + input_shape, GGML_TYPE_F32); + hidden = + engine::modules::AddModule{}.build(context, hidden, next->positions); + hidden = engine::modules::sanm_projection_block( + context, hidden, weights->stem, + block_config(config, config.input_size)); + next->checkpoints.push_back({"stem", hidden}); + + const auto residual_config = block_config(config, config.d_model); + for (size_t index = 0; index < weights->main_layers.size(); ++index) { + hidden = engine::modules::sanm_residual_block( + context, hidden, weights->main_layers[index], residual_config); + if (index == 0 || index == 24 || index == 48) { + next->checkpoints.push_back( + {"main_layer_" + std::to_string(index), hidden}); + } + } + hidden = engine::modules::sanm_layer_norm( + context, hidden, weights->main_norm, kLayerNormEpsilon); + next->checkpoints.push_back({"main_layer_norm", hidden}); + + for (size_t index = 0; index < weights->timestamp_layers.size(); ++index) { + hidden = engine::modules::sanm_residual_block( + context, hidden, weights->timestamp_layers[index], residual_config); + if (index == 0 || index == 10 || index == 19) { + next->checkpoints.push_back( + {"timestamp_layer_" + std::to_string(index), hidden}); + } + } + next->output = engine::modules::sanm_layer_norm( + context, hidden, weights->timestamp_norm, kLayerNormEpsilon); + next->checkpoints.push_back({"final", next->output}); + for (const auto &checkpoint : next->checkpoints) { + ggml_set_output(checkpoint.second.tensor); + } + + next->graph = ggml_new_graph_custom(next->ggml, kEncoderGraphNodes, false); + ggml_build_forward_expand(next->graph, next->output.tensor); + engine::core::validate_backend_graph_supported(next->backend, next->graph, + "Fun-ASR-Nano encoder"); + next->allocator = + ggml_gallocr_new(ggml_backend_get_default_buffer_type(next->backend)); + if (next->allocator == nullptr || + !ggml_gallocr_reserve(next->allocator, next->graph) || + !ggml_gallocr_alloc_graph(next->allocator, next->graph)) { + throw std::runtime_error( + "failed to allocate Fun-ASR-Nano encoder graph tensors"); + } + engine::core::write_tensor_f32( + next->positions, make_sinusoidal_positions(frames, config.input_size)); + engine::core::prepare_host_graph_plan(*execution_context, next->graph, + next->host_plan); + + cached_graph = std::move(next); + engine::debug::timing_log_scalar( + "fun_asr_nano.encoder.graph_build_ms", + engine::debug::elapsed_ms(build_start, Clock::now())); + engine::debug::trace_log_scalar("fun_asr_nano.encoder.graph_cache_hit", + false); + engine::debug::trace_log_scalar("fun_asr_nano.encoder.graph_frames", + frames); + } + + FunAsrNanoEncoderEmbeddings encode(const FunAsrNanoAudioFeatures &features, + bool capture_stages) { + const auto &config = assets->config.encoder; + if (features.frames <= 0 || features.feature_dim != config.input_size) { + throw std::runtime_error("Fun-ASR-Nano encoder input shape is invalid"); + } + if (features.frames >= config.max_position_embeddings) { + throw std::runtime_error( + "Fun-ASR-Nano encoder input exceeds positional capacity"); + } + if (features.valid_frames != features.frames) { + throw std::runtime_error( + "Fun-ASR-Nano encoder currently requires unpadded features"); + } + if (static_cast(features.values.size()) != + features.frames * features.feature_dim) { + throw std::runtime_error( + "Fun-ASR-Nano encoder input value count mismatch"); + } + + const auto encode_start = Clock::now(); + ensure_graph(features.frames); + engine::core::write_tensor_f32(cached_graph->input, features.values); + const auto status = engine::core::compute_graph( + *execution_context, cached_graph->graph, cached_graph->host_plan, + "Fun-ASR-Nano encoder"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Fun-ASR-Nano encoder graph execution failed"); + } + + FunAsrNanoEncoderEmbeddings output; + output.values = engine::core::read_tensor_f32(cached_graph->output.tensor); + output.frames = features.frames; + output.valid_frames = features.valid_frames; + output.hidden_size = config.d_model; + if (capture_stages) { + output.stages.reserve(cached_graph->checkpoints.size()); + for (const auto &checkpoint : cached_graph->checkpoints) { + output.stages.push_back( + {checkpoint.first, + engine::core::read_tensor_f32(checkpoint.second.tensor)}); + } + } + engine::debug::timing_log_scalar( + "fun_asr_nano.encoder_ms", + engine::debug::elapsed_ms(encode_start, Clock::now())); + return output; + } + + std::shared_ptr assets; + engine::core::ExecutionContext *execution_context = nullptr; + size_t graph_arena_bytes = 0; + std::unique_ptr weights; + std::unique_ptr cached_graph; +}; + +FunAsrNanoEncoderRuntime::FunAsrNanoEncoderRuntime( + std::shared_ptr assets, + engine::core::ExecutionContext &execution_context, size_t graph_arena_bytes, + engine::assets::TensorStorageType weight_storage) + : impl_(std::make_unique(std::move(assets), execution_context, + graph_arena_bytes, weight_storage)) {} + +FunAsrNanoEncoderRuntime::~FunAsrNanoEncoderRuntime() = default; +FunAsrNanoEncoderRuntime::FunAsrNanoEncoderRuntime( + FunAsrNanoEncoderRuntime &&) noexcept = default; +FunAsrNanoEncoderRuntime &FunAsrNanoEncoderRuntime::operator=( + FunAsrNanoEncoderRuntime &&) noexcept = default; + +void FunAsrNanoEncoderRuntime::prepare_capacity(int64_t frames) { + if (impl_ == nullptr) { + throw std::runtime_error("Fun-ASR-Nano encoder runtime is moved from"); + } + impl_->ensure_graph(frames); +} + +FunAsrNanoEncoderEmbeddings +FunAsrNanoEncoderRuntime::encode(const FunAsrNanoAudioFeatures &features, + bool capture_stages) { + if (impl_ == nullptr) { + throw std::runtime_error("Fun-ASR-Nano encoder runtime is moved from"); + } + return impl_->encode(features, capture_stages); +} + +} // namespace engine::models::fun_asr_nano diff --git a/src/models/fun_asr_nano/frontend.cpp b/src/models/fun_asr_nano/frontend.cpp new file mode 100644 index 00000000..649bb97c --- /dev/null +++ b/src/models/fun_asr_nano/frontend.cpp @@ -0,0 +1,45 @@ +#include "engine/models/fun_asr_nano/frontend.h" + +#include "engine/framework/audio/kaldi_fbank.h" + +#include +#include + +namespace engine::models::fun_asr_nano { + +FunAsrNanoFrontend::FunAsrNanoFrontend(FunAsrNanoFrontendConfig config) + : config_(std::move(config)) { + if (config_.sample_rate <= 0 || config_.feature_size <= 0 || + config_.frame_length_ms <= 0 || config_.frame_shift_ms <= 0 || + config_.lfr_m <= 0 || config_.lfr_n <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano frontend configuration must be positive"); + } +} + +FunAsrNanoAudioFeatures +FunAsrNanoFrontend::extract(const std::vector &audio, + int sample_rate) const { + if (sample_rate != config_.sample_rate) { + throw std::runtime_error( + "Fun-ASR-Nano frontend requires 16 kHz input audio"); + } + engine::audio::KaldiFbankOptions options; + options.sample_rate = config_.sample_rate; + options.num_mels = static_cast(config_.feature_size); + options.frame_length_ms = static_cast(config_.frame_length_ms); + options.frame_shift_ms = static_cast(config_.frame_shift_ms); + options.lfr_m = static_cast(config_.lfr_m); + options.lfr_n = static_cast(config_.lfr_n); + options.preemphasis = config_.preemphasis; + const auto features = engine::audio::extract_kaldi_fbank(audio, options); + + FunAsrNanoAudioFeatures output; + output.values = features.values; + output.frames = features.frames; + output.feature_dim = features.feature_dim; + output.valid_frames = features.frames; + return output; +} + +} // namespace engine::models::fun_asr_nano diff --git a/src/models/fun_asr_nano/loader.cpp b/src/models/fun_asr_nano/loader.cpp new file mode 100644 index 00000000..d72185a9 --- /dev/null +++ b/src/models/fun_asr_nano/loader.cpp @@ -0,0 +1,162 @@ +#include "engine/models/fun_asr_nano/loader.h" + +#include "engine/framework/model_spec/package.h" +#include "engine/models/fun_asr_nano/session.h" + +#include +#include + +namespace engine::models::fun_asr_nano { +namespace { + +runtime::ModelMetadata metadata(const FunAsrNanoAssets &assets) { + runtime::ModelMetadata result; + result.family = "fun_asr_nano"; + result.variant = assets.config.model_type; + result.description = + "Fun-ASR-Nano-2512 offline multilingual speech recognition."; + return result; +} + +runtime::CapabilitySet capabilities(const FunAsrNanoAssets &assets) { + runtime::CapabilitySet result; + result.supported_tasks = { + {runtime::VoiceTaskKind::Asr, {runtime::RunMode::Offline}}}; + result.languages = assets.config.supported_languages; + result.languages.insert(result.languages.begin(), "auto"); + result.supports_timestamps = false; + return result; +} + +runtime::ModelCliInterface cli() { + runtime::ModelCliInterface result; + result.request_options = { + {"language", "auto|zh|en|ja", "Recognition language.", false, "auto"}, + {"enable_itn", "true|false", "Enable inverse text normalization.", false, + "true"}, + {"max_tokens", "n", "Maximum generated transcript tokens.", false, "512", + "1"}, + {"audio_chunk_mode", "auto|fixed|none", "Audio chunking mode.", false, + "auto"}, + {"audio_chunk_seconds", "seconds", "Fixed audio chunk duration.", false, + "30", "0"}, + }; + result.session_options = { + {"fun_asr_nano.weight_type", "native|f32|f16|bf16|q8_0", + "Shared model weight storage preference."}, + {"fun_asr_nano.encoder_weight_type", "native|f32|f16|bf16|q8_0", + "Encoder matmul weight storage type."}, + {"fun_asr_nano.adaptor_weight_type", "native|f32|f16|bf16|q8_0", + "Adaptor matmul weight storage type."}, + {"fun_asr_nano.decoder_weight_type", "native|f32|f16|bf16|q8_0", + "Decoder matmul weight storage type; CUDA promotes unsafe shared " + "native/F16/Q8 requests to BF16 unless this option is explicit."}, + {"fun_asr_nano.encoder_graph_arena_mb", "mb", + "Encoder graph arena size."}, + {"fun_asr_nano.adaptor_graph_arena_mb", "mb", + "Adaptor graph arena size."}, + {"fun_asr_nano.decoder_prefill_graph_arena_mb", "mb", + "Decoder prefill graph arena size."}, + {"fun_asr_nano.decoder_decode_graph_arena_mb", "mb", + "Decoder cached-step graph arena size."}, + {"fun_asr_nano.decoder_weight_context_mb", "mb", + "Decoder weight context arena size."}, + }; + return result; +} + +class FunAsrNanoLoader final : public runtime::IVoiceModelLoader { +public: + std::string family() const override { return "fun_asr_nano"; } + + runtime::CapabilitySet advertised_capabilities() const override { + runtime::CapabilitySet result; + result.supported_tasks = { + {runtime::VoiceTaskKind::Asr, {runtime::RunMode::Offline}}}; + result.languages = {"auto", "zh", "en", "ja"}; + result.supports_timestamps = false; + return result; + } + + bool can_load(const runtime::ModelLoadRequest &request) const override { + try { + if (request.family_hint.has_value() && *request.family_hint != family()) { + return false; + } + (void)engine::model_spec::load_resource_bundle( + request.model_path, engine::model_spec::default_spec_path(family())); + return true; + } catch (...) { + return false; + } + } + + runtime::ModelInspection + inspect(const runtime::ModelLoadRequest &request) const override { + const auto assets = load_fun_asr_nano_assets(request.model_path); + const auto package_spec = engine::model_spec::default_spec_path(family()); + runtime::ModelInspection result; + result.model_root = assets->resources.model_root(); + result.metadata = metadata(*assets); + result.capabilities = capabilities(*assets); + result.discovered_configs = + runtime::discover_named_assets_from_package_spec( + request.model_path, package_spec, + engine::model_spec::ResourceKind::Files); + result.discovered_weights = + runtime::discover_named_assets_from_package_spec( + request.model_path, package_spec, + engine::model_spec::ResourceKind::Tensors); + result.cli = cli(); + return result; + } + + std::unique_ptr + load(const runtime::ModelLoadRequest &request) const override { + return load_fun_asr_nano_model(request.model_path); + } +}; + +} // namespace + +FunAsrNanoLoadedModel::FunAsrNanoLoadedModel( + runtime::ModelMetadata metadata, runtime::CapabilitySet capabilities, + std::shared_ptr assets) + : metadata_(std::move(metadata)), capabilities_(std::move(capabilities)), + assets_(std::move(assets)) {} + +const runtime::ModelMetadata &FunAsrNanoLoadedModel::metadata() const noexcept { + return metadata_; +} + +const runtime::CapabilitySet & +FunAsrNanoLoadedModel::capabilities() const noexcept { + return capabilities_; +} + +std::unique_ptr +FunAsrNanoLoadedModel::create_task_session( + const runtime::TaskSpec &task, + const runtime::SessionOptions &options) const { + if (task.task != runtime::VoiceTaskKind::Asr) { + throw std::runtime_error("Fun-ASR-Nano only supports the Asr task"); + } + if (task.mode != runtime::RunMode::Offline) { + throw std::runtime_error( + "Fun-ASR-Nano currently supports offline sessions"); + } + return std::make_unique(task, options, assets_); +} + +std::unique_ptr +load_fun_asr_nano_model(const std::filesystem::path &model_path) { + auto assets = load_fun_asr_nano_assets(model_path); + return std::make_unique( + metadata(*assets), capabilities(*assets), std::move(assets)); +} + +std::shared_ptr make_fun_asr_nano_loader() { + return std::make_shared(); +} + +} // namespace engine::models::fun_asr_nano diff --git a/src/models/fun_asr_nano/prompt.cpp b/src/models/fun_asr_nano/prompt.cpp new file mode 100644 index 00000000..55cc2db7 --- /dev/null +++ b/src/models/fun_asr_nano/prompt.cpp @@ -0,0 +1,94 @@ +#include "engine/models/fun_asr_nano/prompt.h" + +#include +#include +#include +#include + +namespace engine::models::fun_asr_nano { +namespace { + +std::string language_name(const std::string &language) { + if (language.empty() || language == "auto") { + return {}; + } + if (language == "zh") { + return "中文"; + } + if (language == "en") { + return "英文"; + } + if (language == "ja") { + return "日文"; + } + throw std::runtime_error("unsupported Fun-ASR-Nano prompt language: " + + language); +} + +} // namespace + +FunAsrNanoPromptBuilder::FunAsrNanoPromptBuilder( + std::shared_ptr assets) + : assets_(std::move(assets)), + tokenizer_(std::make_shared(assets_)) { + if (assets_ == nullptr) { + throw std::runtime_error("Fun-ASR-Nano prompt builder requires assets"); + } +} + +std::string FunAsrNanoPromptBuilder::prompt_text( + const FunAsrNanoPromptRequest &request) const { + if (!request.prompt.empty()) { + return request.prompt; + } + std::string prompt = "语音转写"; + const std::string language = language_name(request.language); + if (!language.empty()) { + prompt += "成" + language; + } + if (!request.enable_itn) { + prompt += "，不进行文本规整"; + } + return prompt + "："; +} + +FunAsrNanoPrompt +FunAsrNanoPromptBuilder::build(const FunAsrNanoPromptRequest &request, + int64_t audio_tokens) const { + if (audio_tokens <= 0 || + audio_tokens > + static_cast(std::numeric_limits::max())) { + throw std::runtime_error( + "Fun-ASR-Nano prompt requires a valid positive audio token count"); + } + const std::string chat = + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n" + + prompt_text(request) + + "<|object_ref_start|><|im_end|>\n<|im_start|>assistant\n"; + const auto ids = tokenizer_->encode(chat); + const int32_t audio_token = + static_cast(assets_->config.audio_token_id); + if (std::count(ids.begin(), ids.end(), audio_token) != 1) { + throw std::runtime_error( + "Fun-ASR-Nano prompt must contain exactly one audio placeholder"); + } + + FunAsrNanoPrompt result; + result.input_ids.reserve(ids.size() + static_cast(audio_tokens - 1)); + for (const int32_t token_id : ids) { + if (token_id == audio_token) { + for (int64_t index = 0; index < audio_tokens; ++index) { + result.audio_token_positions.push_back( + static_cast(result.input_ids.size())); + result.input_ids.push_back(token_id); + } + } else { + result.input_ids.push_back(token_id); + } + } + result.attention_mask.assign(result.input_ids.size(), 1); + return result; +} + +} // namespace engine::models::fun_asr_nano diff --git a/src/models/fun_asr_nano/session.cpp b/src/models/fun_asr_nano/session.cpp new file mode 100644 index 00000000..d578076e --- /dev/null +++ b/src/models/fun_asr_nano/session.cpp @@ -0,0 +1,431 @@ +#include "engine/models/fun_asr_nano/session.h" + +#include "engine/framework/audio/chunking.h" +#include "engine/framework/audio/conversion.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/io/text.h" +#include "engine/framework/runtime/options.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::fun_asr_nano { +namespace { + +using Clock = std::chrono::steady_clock; +constexpr int64_t kDefaultMaxNewTokens = 512; +constexpr float kDefaultChunkSeconds = 30.0F; + +std::shared_ptr +require_assets(std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("Fun-ASR-Nano session requires assets"); + } + return assets; +} + +void validate_weight_storage(assets::TensorStorageType storage, + const std::string &option) { + if (storage == assets::TensorStorageType::Native || + storage == assets::TensorStorageType::F32 || + storage == assets::TensorStorageType::F16 || + storage == assets::TensorStorageType::BF16 || + storage == assets::TensorStorageType::Q8_0) { + return; + } + throw std::runtime_error(option + + " supports only native, f32, f16, bf16, and q8_0"); +} + +assets::TensorStorageType +option_weight_type(const runtime::SessionOptions &options, const char *key, + assets::TensorStorageType fallback) { + const auto value = options.options.find(key); + return value == options.options.end() + ? fallback + : assets::parse_tensor_storage_type(value->second); +} + +assets::TensorStorageType +decoder_weight_type(const runtime::SessionOptions &options, + core::BackendType backend_type) { + const auto decoder = options.options.find("fun_asr_nano.decoder_weight_type"); + if (decoder != options.options.end()) { + return assets::parse_tensor_storage_type(decoder->second); + } + auto shared = assets::TensorStorageType::Native; + const auto shared_option = options.options.find("fun_asr_nano.weight_type"); + if (shared_option != options.options.end()) { + shared = assets::parse_tensor_storage_type(shared_option->second); + } + // Native F16/Q8 decoder weights currently produce invalid logits with the + // CUDA graph path. BF16 preserves the compact GPU representation and exact + // transcript parity while encoder/adaptor weights keep the shared type. + if (backend_type == core::BackendType::Cuda && + (shared == assets::TensorStorageType::Native || + shared == assets::TensorStorageType::F16 || + shared == assets::TensorStorageType::Q8_0)) { + return assets::TensorStorageType::BF16; + } + return shared; +} + +runtime::SessionOptions +validate_session_setup(const runtime::TaskSpec &task, + runtime::SessionOptions options) { + if (task.task != runtime::VoiceTaskKind::Asr) { + throw std::runtime_error("Fun-ASR-Nano only supports VoiceTaskKind::Asr"); + } + if (task.mode != runtime::RunMode::Offline) { + throw std::runtime_error( + "Fun-ASR-Nano currently supports offline sessions"); + } + const auto shared = option_weight_type(options, "fun_asr_nano.weight_type", + assets::TensorStorageType::Native); + validate_weight_storage( + option_weight_type(options, "fun_asr_nano.encoder_weight_type", shared), + "fun_asr_nano.encoder_weight_type"); + validate_weight_storage( + option_weight_type(options, "fun_asr_nano.adaptor_weight_type", shared), + "fun_asr_nano.adaptor_weight_type"); + validate_weight_storage(decoder_weight_type(options, options.backend.type), + "fun_asr_nano.decoder_weight_type"); + for (const auto &[key, value] : options.options) { + (void)value; + if (key.rfind("fun_asr_nano.", 0) == 0 && + key != "fun_asr_nano.weight_type" && + key != "fun_asr_nano.encoder_weight_type" && + key != "fun_asr_nano.adaptor_weight_type" && + key != "fun_asr_nano.decoder_weight_type" && + key != "fun_asr_nano.encoder_graph_arena_mb" && + key != "fun_asr_nano.adaptor_graph_arena_mb" && + key != "fun_asr_nano.decoder_prefill_graph_arena_mb" && + key != "fun_asr_nano.decoder_decode_graph_arena_mb" && + key != "fun_asr_nano.decoder_weight_context_mb") { + throw std::runtime_error("unknown Fun-ASR-Nano session option: " + key); + } + } + return options; +} + +int64_t audio_frame_count(const runtime::AudioBuffer &audio) { + if (audio.sample_rate <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano audio requires a positive sample rate"); + } + if (audio.channels <= 0) { + throw std::runtime_error("Fun-ASR-Nano audio requires positive channels"); + } + if (audio.samples.empty()) { + throw std::runtime_error("Fun-ASR-Nano audio must not be empty"); + } + if (audio.samples.size() % static_cast(audio.channels) != 0) { + throw std::runtime_error( + "Fun-ASR-Nano audio samples must be divisible by channel count"); + } + return static_cast(audio.samples.size() / + static_cast(audio.channels)); +} + +bool ascii_word_boundary(const std::string &text, bool front) { + if (text.empty()) { + return false; + } + const unsigned char value = + static_cast(front ? text.front() : text.back()); + return (value >= '0' && value <= '9') || (value >= 'A' && value <= 'Z') || + (value >= 'a' && value <= 'z'); +} + +void append_chunk_text(std::string &merged, std::string chunk) { + chunk = engine::io::trim_ascii_whitespace(std::move(chunk)); + if (chunk.empty()) { + return; + } + if (!merged.empty() && ascii_word_boundary(merged, false) && + ascii_word_boundary(chunk, true)) { + merged.push_back(' '); + } + merged += chunk; +} + +void validate_request_options( + const std::unordered_map &options) { + for (const auto &[key, value] : options) { + (void)value; + if (key != "language" && key != "enable_itn" && key != "max_tokens" && + key != "audio_chunk_mode" && key != "audio_chunk_seconds" && + key != "audio_chunk_duration_seconds" && + key != "audio_chunk_duration") { + throw std::runtime_error("unknown Fun-ASR-Nano request option: " + key); + } + } +} + +} // namespace + +FunAsrNanoSession::FunAsrNanoSession( + runtime::TaskSpec task, runtime::SessionOptions options, + std::shared_ptr assets) + : RuntimeSessionBase(validate_session_setup(task, options)), task_(task), + assets_(require_assets(std::move(assets))), + encoder_graph_arena_bytes_(runtime::parse_size_mb_option( + options.options, {"fun_asr_nano.encoder_graph_arena_mb"}, + 512ull * 1024ull * 1024ull)), + adaptor_graph_arena_bytes_(runtime::parse_size_mb_option( + options.options, {"fun_asr_nano.adaptor_graph_arena_mb"}, + 128ull * 1024ull * 1024ull)), + decoder_prefill_graph_arena_bytes_(runtime::parse_size_mb_option( + options.options, {"fun_asr_nano.decoder_prefill_graph_arena_mb"}, + 256ull * 1024ull * 1024ull)), + decoder_decode_graph_arena_bytes_(runtime::parse_size_mb_option( + options.options, {"fun_asr_nano.decoder_decode_graph_arena_mb"}, + 128ull * 1024ull * 1024ull)), + decoder_weight_context_bytes_(runtime::parse_size_mb_option( + options.options, {"fun_asr_nano.decoder_weight_context_mb"}, + 32ull * 1024ull * 1024ull)), + encoder_weight_storage_type_(option_weight_type( + options, "fun_asr_nano.encoder_weight_type", + option_weight_type(options, "fun_asr_nano.weight_type", + assets::TensorStorageType::Native))), + adaptor_weight_storage_type_(option_weight_type( + options, "fun_asr_nano.adaptor_weight_type", + option_weight_type(options, "fun_asr_nano.weight_type", + assets::TensorStorageType::Native))), + decoder_weight_storage_type_( + decoder_weight_type(options, execution_context().backend_type())), + tokenizer_(assets_), frontend_(assets_->config.frontend), + encoder_(assets_, execution_context(), encoder_graph_arena_bytes_, + encoder_weight_storage_type_), + adaptor_(assets_, execution_context(), adaptor_graph_arena_bytes_, + adaptor_weight_storage_type_), + prompt_builder_(assets_), + decoder_(assets_, execution_context(), decoder_prefill_graph_arena_bytes_, + decoder_decode_graph_arena_bytes_, decoder_weight_context_bytes_, + decoder_weight_storage_type_) {} + +FunAsrNanoSession::~FunAsrNanoSession() = default; + +std::string FunAsrNanoSession::family() const { return "fun_asr_nano"; } + +runtime::VoiceTaskKind FunAsrNanoSession::task_kind() const { + return task_.task; +} + +runtime::RunMode FunAsrNanoSession::run_mode() const { return task_.mode; } + +void FunAsrNanoSession::prepare( + const runtime::SessionPreparationRequest &request) { + const auto started = Clock::now(); + if (!request.audio.has_value()) { + throw std::runtime_error( + "Fun-ASR-Nano prepare() requires an audio contract"); + } + mark_prepared(); + debug::timing_log_scalar("fun_asr_nano.prepare_ms", + debug::elapsed_ms(started)); + debug::trace_log_scalar("fun_asr_nano.prepare.max_input_samples", + request.audio->max_input_samples); +} + +runtime::TaskResult +FunAsrNanoSession::run(const runtime::TaskRequest &request) { + require_prepared("Fun-ASR-Nano run()"); + validate_request_options(request.options); + const auto chunks = audio_chunk_plan(request); + if (chunks.empty()) { + return run_single(make_request(request)); + } + const auto &audio = *request.audio_input; + if (chunks.size() == 1) { + auto item = request; + item.audio_input = + engine::audio::slice_audio_buffer(audio, chunks.front().source_span); + return run_single(make_request(item)); + } + + runtime::TaskResult merged; + std::string text; + for (const auto &chunk : chunks) { + auto item = request; + item.audio_input = + engine::audio::slice_audio_buffer(audio, chunk.source_span); + const auto result = run_single(make_request(item)); + if (result.text_output.has_value()) { + append_chunk_text(text, result.text_output->text); + if (!merged.text_output.has_value()) { + merged.text_output = + runtime::Transcript{"", result.text_output->language}; + } + } + } + if (!merged.text_output.has_value()) { + merged.text_output = runtime::Transcript{}; + } + merged.text_output->text = std::move(text); + return merged; +} + +FunAsrNanoSession::AsrRequest +FunAsrNanoSession::make_request(const runtime::TaskRequest &request) const { + if (!request.audio_input.has_value()) { + throw std::runtime_error("Fun-ASR-Nano run() requires audio_input"); + } + (void)audio_frame_count(*request.audio_input); + + AsrRequest out; + out.audio = *request.audio_input; + out.prompt.language = "auto"; + out.generation.max_new_tokens = kDefaultMaxNewTokens; + if (request.text_input.has_value()) { + out.prompt.prompt = request.text_input->text; + if (!request.text_input->language.empty()) { + out.prompt.language = request.text_input->language; + } + } + if (const auto language = + runtime::find_option(request.options, {"language"})) { + out.prompt.language = *language; + } + if (const auto value = + runtime::find_option(request.options, {"enable_itn"})) { + out.prompt.enable_itn = runtime::parse_bool_option(*value, "enable_itn"); + } + if (const auto value = + runtime::parse_int_option(request.options, {"max_tokens"})) { + if (*value <= 0) { + throw std::runtime_error("Fun-ASR-Nano max_tokens must be positive"); + } + out.generation.max_new_tokens = *value; + } + return out; +} + +std::vector +FunAsrNanoSession::audio_chunk_plan(const runtime::TaskRequest &request) const { + if (!request.audio_input.has_value()) { + return {}; + } + const auto mode = engine::audio::parse_audio_chunk_mode(request.options); + if (mode == engine::audio::AudioChunkMode::None) { + return {}; + } + if (mode == engine::audio::AudioChunkMode::Vad || + mode == engine::audio::AudioChunkMode::QuietEnergy) { + throw std::runtime_error( + "Fun-ASR-Nano supports audio_chunk_mode=auto, fixed, or none"); + } + + const auto &audio = *request.audio_input; + const int64_t frames = audio_frame_count(audio); + const float seconds = + engine::audio::parse_audio_chunk_seconds_override(request.options) + .value_or(kDefaultChunkSeconds); + if (!std::isfinite(seconds) || !(seconds > 0.0F)) { + throw std::runtime_error( + "Fun-ASR-Nano audio_chunk_seconds must be positive"); + } + const double sample_count = + static_cast(seconds) * static_cast(audio.sample_rate); + if (sample_count >= + static_cast(std::numeric_limits::max())) { + throw std::runtime_error("Fun-ASR-Nano audio_chunk_seconds is too large"); + } + const int64_t samples = static_cast(std::llround(sample_count)); + if (samples <= 0) { + throw std::runtime_error( + "Fun-ASR-Nano audio_chunk_seconds produced an empty chunk"); + } + const auto chunks = engine::audio::plan_audio_chunks( + frames, {samples, samples, engine::audio::AudioChunkPadMode::Zero, + engine::audio::AudioChunkTailAlignment::Start, 0}); + std::vector plan; + plan.reserve(chunks.size()); + for (const auto &chunk : chunks) { + plan.push_back({{chunk.output_start_sample, + chunk.output_start_sample + chunk.valid_samples}}); + } + return plan; +} + +runtime::TaskResult FunAsrNanoSession::run_single(const AsrRequest &request) { + const auto wall_start = Clock::now(); + + const auto resample_start = Clock::now(); + const auto mono = + engine::audio::convert_interleaved_audio_to_mono_linear_resampled( + request.audio.samples, request.audio.sample_rate, + request.audio.channels, assets_->config.frontend.sample_rate); + const auto resample_end = Clock::now(); + + const auto frontend_start = Clock::now(); + const auto features = + frontend_.extract(mono, assets_->config.frontend.sample_rate); + const auto frontend_end = Clock::now(); + + const auto encoder_start = Clock::now(); + const auto encoded = encoder_.encode(features); + const auto encoder_end = Clock::now(); + + std::vector mask(static_cast(encoded.frames), 0); + std::fill_n(mask.begin(), static_cast(encoded.valid_frames), 1); + const auto adaptor_start = Clock::now(); + const auto audio_embeddings = adaptor_.adapt(encoded, mask); + const auto adaptor_end = Clock::now(); + + const auto prompt_start = Clock::now(); + const auto prompt = + prompt_builder_.build(request.prompt, audio_embeddings.tokens); + const auto prompt_end = Clock::now(); + + const auto decoder_start = Clock::now(); + const auto tokens = + decoder_.generate(prompt, audio_embeddings, request.generation); + const auto decoder_end = Clock::now(); + + const auto decode_start = Clock::now(); + auto text = + engine::io::trim_ascii_whitespace(tokenizer_.decode(tokens.token_ids)); + const auto decode_end = Clock::now(); + + runtime::TaskResult result; + result.text_output = + runtime::Transcript{std::move(text), request.prompt.language == "auto" + ? std::string{} + : request.prompt.language}; + + debug::timing_log_scalar("fun_asr_nano.resample_ms", + debug::elapsed_ms(resample_start, resample_end)); + debug::timing_log_scalar("fun_asr_nano.frontend_ms", + debug::elapsed_ms(frontend_start, frontend_end)); + debug::timing_log_scalar("fun_asr_nano.encoder_ms", + debug::elapsed_ms(encoder_start, encoder_end)); + debug::timing_log_scalar("fun_asr_nano.adaptor_ms", + debug::elapsed_ms(adaptor_start, adaptor_end)); + debug::timing_log_scalar("fun_asr_nano.prompt_ms", + debug::elapsed_ms(prompt_start, prompt_end)); + debug::timing_log_scalar("fun_asr_nano.decoder_ms", + debug::elapsed_ms(decoder_start, decoder_end)); + debug::timing_log_scalar("fun_asr_nano.decode_ms", + debug::elapsed_ms(decode_start, decode_end)); + debug::timing_log_scalar("session.wall_ms", debug::elapsed_ms(wall_start)); + debug::trace_log_scalar("fun_asr_nano.audio_input_frames", + audio_frame_count(request.audio)); + debug::trace_log_scalar("fun_asr_nano.frontend_frames", features.frames); + debug::trace_log_scalar("fun_asr_nano.encoder_valid_frames", + encoded.valid_frames); + debug::trace_log_scalar("fun_asr_nano.audio_tokens", audio_embeddings.tokens); + debug::trace_log_scalar("fun_asr_nano.prompt_tokens", + prompt.input_ids.size()); + debug::trace_log_scalar("fun_asr_nano.generated_tokens", + tokens.token_ids.size()); + return result; +} + +} // namespace engine::models::fun_asr_nano diff --git a/src/models/fun_asr_nano/tokenizer_text.cpp b/src/models/fun_asr_nano/tokenizer_text.cpp new file mode 100644 index 00000000..eee10d4b --- /dev/null +++ b/src/models/fun_asr_nano/tokenizer_text.cpp @@ -0,0 +1,81 @@ +#include "engine/models/fun_asr_nano/tokenizer_text.h" + +#include "engine/framework/tokenizers/llama_bpe.h" + +#include +#include + +namespace engine::models::fun_asr_nano { + +struct FunAsrNanoTextTokenizer::Impl { + std::shared_ptr tokenizer; +}; + +namespace { + +std::shared_ptr +load_impl(const FunAsrNanoAssets &assets) { + engine::tokenizers::LlamaBpeTokenizerSpec spec; + spec.tokenizer_config_path = + assets.resources.require_file("tokenizer_config"); + if (const auto *path = assets.resources.find_file("vocab")) { + spec.vocab_path = *path; + } + if (const auto *path = assets.resources.find_file("merges")) { + spec.merges_path = *path; + } + spec.tokenizer_json_path = assets.resources.require_file("tokenizer_json"); + spec.pre_type = engine::tokenizers::LlamaBpePreTokenizer::Qwen2; + + auto impl = std::make_shared(); + impl->tokenizer = engine::tokenizers::load_llama_bpe_tokenizer(spec); + return impl; +} + +} // namespace + +FunAsrNanoTextTokenizer::FunAsrNanoTextTokenizer( + std::shared_ptr assets) + : assets_(std::move(assets)) { + if (assets_ == nullptr) { + throw std::runtime_error("Fun-ASR-Nano tokenizer requires assets"); + } + impl_ = load_impl(*assets_); + if (require_token_id("<|object_ref_start|>") != + assets_->config.audio_token_id || + require_token_id("<|im_end|>") != assets_->config.text.eos_token_id) { + throw std::runtime_error( + "Fun-ASR-Nano tokenizer special token ids do not match config"); + } +} + +std::vector +FunAsrNanoTextTokenizer::encode(const std::string &text) const { + return impl_->tokenizer->encode(text); +} + +std::string +FunAsrNanoTextTokenizer::decode(const std::vector &token_ids) const { + std::vector filtered; + filtered.reserve(token_ids.size()); + for (const int32_t token_id : token_ids) { + if (token_id == assets_->config.text.eos_token_id || + impl_->tokenizer->is_control_token_id(token_id)) { + continue; + } + filtered.push_back(token_id); + } + return impl_->tokenizer->decode(filtered); +} + +int32_t +FunAsrNanoTextTokenizer::require_token_id(const std::string &token) const { + const auto token_id = impl_->tokenizer->find_token_id(token); + if (!token_id.has_value()) { + throw std::runtime_error("Fun-ASR-Nano tokenizer is missing token: " + + token); + } + return *token_id; +} + +} // namespace engine::models::fun_asr_nano diff --git a/tests/fun_asr_nano/adaptor_reference.bin b/tests/fun_asr_nano/adaptor_reference.bin new file mode 100644 index 00000000..b43dd611 Binary files /dev/null and b/tests/fun_asr_nano/adaptor_reference.bin differ diff --git a/tests/fun_asr_nano/adaptor_reference.json b/tests/fun_asr_nano/adaptor_reference.json new file mode 100644 index 00000000..8710398f --- /dev/null +++ b/tests/fun_asr_nano/adaptor_reference.json @@ -0,0 +1,73 @@ +{ + "schema_version": 1, + "transformers_commit": "f9966442ac24fff57060774ce22e1884760f4a3b", + "model_revision": "854d88f94205cd17d2afdb24332130d86fbe654a", + "modeling_sha256": "70f8230fd5a75e8a119cea1c8dc919c3f51d8aa7c5d8f1c23c5402b856ebebfb", + "model_config_sha256": "c7c4a30316929631ac5fabc5fb3c0dd3278dcc9809670720c5920186285d004a", + "model_safetensors_sha256": "335ca3e74917f1156690400e2c344350112950165789cf78ce3d0a367affd821", + "weight_catalog_count": 36, + "weight_catalog_sha256": "7580a687f064aad5d235c36cbfbe8bfdfc742677c2f8a022f924cfd8d5953913", + "torch_version": "2.11.0+cu130", + "data_file": "adaptor_reference.bin", + "data_format": "little-endian-float32", + "input": { + "shape": [ + 2, + 4, + 512 + ], + "offset_f32": 0, + "count": 4096 + }, + "mask": { + "shape": [ + 2, + 4 + ], + "offset_f32": 4096, + "count": 8 + }, + "checkpoints": { + "linear_1": { + "shape": [ + 6, + 2048 + ], + "offset_f32": 4104, + "count": 12288 + }, + "linear_2": { + "shape": [ + 6, + 1024 + ], + "offset_f32": 16392, + "count": 6144 + }, + "block_0": { + "shape": [ + 6, + 1024 + ], + "offset_f32": 22536, + "count": 6144 + }, + "block_1": { + "shape": [ + 6, + 1024 + ], + "offset_f32": 28680, + "count": 6144 + }, + "packed_valid": { + "shape": [ + 6, + 1024 + ], + "offset_f32": 34824, + "count": 6144 + } + }, + "data_sha256": "340ebd8f080c205fa42d1646600a35661ff64d1f17a21cdd12102ca6012ef70d" +} diff --git a/tests/fun_asr_nano/decoder_reference.bin b/tests/fun_asr_nano/decoder_reference.bin new file mode 100644 index 00000000..d6c4ff66 Binary files /dev/null and b/tests/fun_asr_nano/decoder_reference.bin differ diff --git a/tests/fun_asr_nano/decoder_reference.json b/tests/fun_asr_nano/decoder_reference.json new file mode 100644 index 00000000..16162e82 --- /dev/null +++ b/tests/fun_asr_nano/decoder_reference.json @@ -0,0 +1,137 @@ +{ + "schema_version": 1, + "transformers_commit": "f9966442ac24fff57060774ce22e1884760f4a3b", + "transformers_source_sha256": { + "src/transformers/models/fun_asr_nano/configuration_fun_asr_nano.py": "174abb3aad5b38f7208c7eae140531be900eafb3a5d7f9f5562c300c93240e2d", + "src/transformers/models/fun_asr_nano/modeling_fun_asr_nano.py": "70f8230fd5a75e8a119cea1c8dc919c3f51d8aa7c5d8f1c23c5402b856ebebfb", + "src/transformers/models/fun_asr_nano/processing_fun_asr_nano.py": "bc1601a59f6acef6e5be743e511891fad688afa192d10e722fdea2dc42b5f72a", + "src/transformers/models/qwen3/modeling_qwen3.py": "966626c5f2c0f729dd6b61b99c000038602805f7d7ad73957507a63f8c2de563" + }, + "model_revision": "854d88f94205cd17d2afdb24332130d86fbe654a", + "model_config_sha256": "c7c4a30316929631ac5fabc5fb3c0dd3278dcc9809670720c5920186285d004a", + "model_safetensors_sha256": "335ca3e74917f1156690400e2c344350112950165789cf78ce3d0a367affd821", + "tokenizer_json_sha256": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506", + "tokenizer_config_sha256": "f8bd91da51857bc4674f92676d8b4979065c9f1d0b4ab2f9764fc58148cb5c62", + "text_weight_catalog_count": 310, + "text_weight_catalog_sha256": "5c75f016a25f2b0cb8abe959633ccb2fc25a8c50654b05962b6ca24493ccedd1", + "torch_version": "2.11.0+cu130", + "device": "cpu", + "data_file": "decoder_reference.bin", + "data_format": "little-endian-float32", + "audio_token_id": 151646, + "im_start_token_id": 151644, + "im_end_token_id": 151645, + "eos_token_id": 151645, + "itn": { + "prompt": "语音转写：", + "input_ids": [ + 151644, + 8948, + 198, + 2610, + 525, + 264, + 10950, + 17847, + 13, + 151645, + 198, + 151644, + 872, + 198, + 105761, + 46670, + 61443, + 5122, + 151646, + 151646, + 151645, + 198, + 151644, + 77091, + 198 + ], + "audio_token_positions": [ + 18, + 19 + ] + }, + "no_itn": { + "prompt": "语音转写，不进行文本规整：", + "input_ids": [ + 151644, + 8948, + 198, + 2610, + 525, + 264, + 10950, + 17847, + 13, + 151645, + 198, + 151644, + 872, + 198, + 105761, + 46670, + 61443, + 3837, + 16530, + 71817, + 108704, + 74386, + 63431, + 5122, + 151646, + 151646, + 151645, + 198, + 151644, + 77091, + 198 + ], + "audio_token_positions": [ + 24, + 25 + ] + }, + "audio_embeddings": { + "offset_f32": 0, + "count": 2048, + "shape": [ + 2, + 1024 + ] + }, + "greedy_token_ids": [ + 56568, + 1773, + 151645 + ], + "generated_text": "你。", + "logits": [ + { + "offset_f32": 2048, + "count": 151936, + "shape": [ + 151936 + ] + }, + { + "offset_f32": 153984, + "count": 151936, + "shape": [ + 151936 + ] + }, + { + "offset_f32": 305920, + "count": 151936, + "shape": [ + 151936 + ] + } + ], + "data_sha256": "37ee1e15e2017a9839f3c1fae08fbc1ef6ade55c9a0ed2f7890bcfc1d7cb7894" +} diff --git a/tests/fun_asr_nano/encoder_reference.bin b/tests/fun_asr_nano/encoder_reference.bin new file mode 100644 index 00000000..3ba164ac Binary files /dev/null and b/tests/fun_asr_nano/encoder_reference.bin differ diff --git a/tests/fun_asr_nano/encoder_reference.json b/tests/fun_asr_nano/encoder_reference.json new file mode 100644 index 00000000..15e73380 --- /dev/null +++ b/tests/fun_asr_nano/encoder_reference.json @@ -0,0 +1,107 @@ +{ + "schema_version": 1, + "transformers_commit": "48e7f65fb274172e15aa88875d780c67c37606c7", + "model_revision": "854d88f94205cd17d2afdb24332130d86fbe654a", + "modeling_sha256": "70f8230fd5a75e8a119cea1c8dc919c3f51d8aa7c5d8f1c23c5402b856ebebfb", + "model_config_sha256": "c7c4a30316929631ac5fabc5fb3c0dd3278dcc9809670720c5920186285d004a", + "model_safetensors_sha256": "335ca3e74917f1156690400e2c344350112950165789cf78ce3d0a367affd821", + "weight_catalog_count": 1194, + "weight_catalog_sha256": "33322fb22cad98346dd0ad0c60a18c4c25a1c4a6c4abe4aec94452d289f44dbe", + "torch_version": "2.11.0+cu130", + "data_file": "encoder_reference.bin", + "data_format": "little-endian-float32", + "data_sha256": "782f206c4d9d8484c1e48f2ad8d06e67da077cc07547cb52da8cedc345285264", + "input": { + "shape": [ + 1, + 3, + 560 + ], + "offset_f32": 0, + "count": 1680 + }, + "valid_frames": 3, + "checkpoints": { + "stem": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 1680, + "count": 1536 + }, + "main_layer_0": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 3216, + "count": 1536 + }, + "main_layer_24": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 4752, + "count": 1536 + }, + "main_layer_48": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 6288, + "count": 1536 + }, + "main_layer_norm": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 7824, + "count": 1536 + }, + "timestamp_layer_0": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 9360, + "count": 1536 + }, + "timestamp_layer_10": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 10896, + "count": 1536 + }, + "timestamp_layer_19": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 12432, + "count": 1536 + }, + "final": { + "shape": [ + 1, + 3, + 512 + ], + "offset_f32": 13968, + "count": 1536 + } + } +} diff --git a/tests/fun_asr_nano/frontend_reference.bin b/tests/fun_asr_nano/frontend_reference.bin new file mode 100644 index 00000000..6c8bcbf4 Binary files /dev/null and b/tests/fun_asr_nano/frontend_reference.bin differ diff --git a/tests/fun_asr_nano/frontend_reference.json b/tests/fun_asr_nano/frontend_reference.json new file mode 100644 index 00000000..8e392e89 --- /dev/null +++ b/tests/fun_asr_nano/frontend_reference.json @@ -0,0 +1,357 @@ +{ + "schema_version": 2, + "reference": "transformers.FunAsrNanoFeatureExtractor", + "transformers_commit": "48e7f65fb274172e15aa88875d780c67c37606c7", + "feature_extractor_sha256": "cb0e31712b264a62e0aa10f7ff23ef531bca02dbee1f378b8947efbdf3b85ee2", + "torch_version": "2.11.0+cu130", + "torchaudio_version": "2.11.0+cu130", + "sample_wav": "sample_16k.wav", + "sample_wav_sha256": "a8096a0c2ac136bcaf70143dc80ba576ade8dfbf7f9f30ecdec29b920ab3e3cb", + "data_file": "frontend_reference.bin", + "data_format": "little-endian-float32", + "data_sha256": "12a42c35f0ca863e5ff17c56394468ba91cb2fe9a4de67a59001a77cf9e0363e", + "sample_rate": 16000, + "feature_size": 80, + "frame_length_ms": 25, + "frame_shift_ms": 10, + "lfr_m": 7, + "lfr_n": 6, + "fixtures": { + "silence": { + "samples": 16000, + "frames": 17, + "width": 560, + "min": -15.942384719848633, + "max": -15.942384719848633, + "mean": -15.942384719848633, + "l2": 1555.5062464965847, + "first32": [ + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633 + ], + "last32": [ + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633 + ], + "waveform": { + "offset_f32": 0, + "count": 16000 + }, + "features": { + "offset_f32": 16000, + "count": 9520 + } + }, + "impulse": { + "samples": 16000, + "frames": 17, + "width": 560, + "min": -15.942384719848633, + "max": 3.2924442291259766, + "mean": -15.599400718495673, + "l2": 1536.8208434990697, + "first32": [ + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633 + ], + "last32": [ + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633, + -15.942384719848633 + ], + "waveform": { + "offset_f32": 25520, + "count": 16000 + }, + "features": { + "offset_f32": 41520, + "count": 9520 + } + }, + "sine_440hz": { + "samples": 16000, + "frames": 17, + "width": 560, + "min": -10.94229793548584, + "max": 4.429123401641846, + "mean": -7.612086516202373, + "l2": 796.5784973954068, + "first32": [ + -8.902217864990234, + -7.893123626708984, + -7.683259963989258, + -8.373307228088379, + -9.063929557800293, + -8.556164741516113, + -7.122377395629883, + -6.354111194610596, + -6.597945213317871, + -9.190327644348145, + -8.430506706237793, + -5.84726619720459, + 0.3718216121196747, + 3.6082963943481445, + 4.427730560302734, + 3.3434324264526367, + -0.4013015031814575, + -6.9655890464782715, + -5.87312650680542, + -4.906649589538574, + -5.516244411468506, + -6.746131896972656, + -5.554996013641357, + -5.721648693084717, + -7.326694011688232, + -5.929301738739014, + -6.312660217285156, + -7.367210865020752, + -6.188997745513916, + -7.128389358520508, + -6.936490058898926, + -6.658333778381348 + ], + "last32": [ + -10.55267333984375, + -10.470776557922363, + -10.453227043151855, + -10.47970962524414, + -10.690077781677246, + -10.710362434387207, + -10.666672706604004, + -10.772344589233398, + -10.792530059814453, + -10.789617538452148, + -10.870523452758789, + -10.887447357177734, + -10.884880065917969, + -10.852453231811523, + -10.761302947998047, + -10.912212371826172, + -10.925711631774902, + -10.91910171508789, + -10.913459777832031, + -10.814454078674316, + -10.699851036071777, + -10.876879692077637, + -10.893570899963379, + -10.742762565612793, + -10.122283935546875, + -10.580676078796387, + -10.839366912841797, + -10.707252502441406, + -10.054222106933594, + -10.470293998718262, + -10.615829467773438, + -10.527753829956055 + ], + "waveform": { + "offset_f32": 51040, + "count": 16000 + }, + "features": { + "offset_f32": 67040, + "count": 9520 + } + }, + "sample_16k": { + "samples": 225151, + "frames": 235, + "width": 560, + "min": -15.942384719848633, + "max": 6.347103118896484, + "mean": -5.632492880177623, + "l2": 2549.3456314076384, + "first32": [ + -8.025833129882812, + -7.580291748046875, + -8.279046058654785, + -8.900419235229492, + -9.515005111694336, + -10.775506019592285, + -12.052884101867676, + -11.757328987121582, + -13.11245346069336, + -15.366243362426758, + -14.615835189819336, + -13.44446849822998, + -12.274049758911133, + -12.53348159790039, + -13.522049903869629, + -13.94956111907959, + -13.105130195617676, + -12.761933326721191, + -13.164908409118652, + -13.763184547424316, + -14.2503023147583, + -12.552626609802246, + -12.877581596374512, + -12.61111068725586, + -12.65695858001709, + -13.439993858337402, + -12.562033653259277, + -12.736815452575684, + -13.652838706970215, + -14.115615844726562, + -12.287749290466309, + -11.618743896484375 + ], + "last32": [ + -10.266275405883789, + -9.827715873718262, + -9.91140365600586, + -10.755105018615723, + -11.691581726074219, + -11.801953315734863, + -11.057150840759277, + -11.519274711608887, + -11.85002613067627, + -11.05031967163086, + -10.317023277282715, + -10.019948959350586, + -10.43407154083252, + -10.095555305480957, + -9.02872085571289, + -10.206125259399414, + -9.60042667388916, + -10.06050968170166, + -9.961405754089355, + -10.69766902923584, + -9.176285743713379, + -9.457172393798828, + -9.47586441040039, + -9.176664352416992, + -9.69524097442627, + -9.745139122009277, + -9.755215644836426, + -10.710921287536621, + -10.568866729736328, + -10.585000991821289, + -11.155245780944824, + -11.610282897949219 + ], + "features": { + "offset_f32": 76560, + "count": 131600 + } + } + } +} diff --git a/tests/fun_asr_nano/fun_asr_nano_adaptor_probe.cpp b/tests/fun_asr_nano/fun_asr_nano_adaptor_probe.cpp new file mode 100644 index 00000000..84e0a3cc --- /dev/null +++ b/tests/fun_asr_nano/fun_asr_nano_adaptor_probe.cpp @@ -0,0 +1,263 @@ +#include "engine/framework/core/execution_context.h" +#include "engine/framework/io/json.h" +#include "engine/models/fun_asr_nano/adaptor.h" +#include "engine/models/fun_asr_nano/assets.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t kGraphArenaBytes = 64 * 1024 * 1024; +constexpr float kAbsoluteTolerance = 2.0e-3F; +constexpr float kRelativeTolerance = 2.0e-3F; +constexpr const char *kTransformersCommit = + "f9966442ac24fff57060774ce22e1884760f4a3b"; +constexpr const char *kModelRevision = + "854d88f94205cd17d2afdb24332130d86fbe654a"; + +struct Arguments { + engine::core::BackendType backend = engine::core::BackendType::Cpu; + std::filesystem::path model; + std::filesystem::path reference; + std::filesystem::path data; +}; + +Arguments parse_arguments(int argc, char **argv) { + Arguments result; + for (int index = 1; index < argc; ++index) { + const std::string option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + option); + } + const std::string value(argv[++index]); + if (option == "--backend") { + if (value == "cpu") { + result.backend = engine::core::BackendType::Cpu; + } else if (value == "cuda") { + result.backend = engine::core::BackendType::Cuda; + } else { + throw std::runtime_error("unsupported adaptor probe backend: " + value); + } + } else if (option == "--model") { + result.model = value; + } else if (option == "--reference") { + result.reference = value; + } else if (option == "--data") { + result.data = value; + } else { + throw std::runtime_error("unknown adaptor probe option: " + option); + } + } + if (result.model.empty() || result.reference.empty() || result.data.empty()) { + throw std::runtime_error("--model, --reference, and --data are required"); + } + return result; +} + +std::vector load_f32_slice(const std::filesystem::path &data_path, + const engine::io::json::Value &descriptor) { + const int64_t signed_offset = descriptor.require("offset_f32").as_i64(); + const int64_t signed_count = descriptor.require("count").as_i64(); + if (signed_offset < 0 || signed_count < 0 || + static_cast(signed_count) > + std::numeric_limits::max()) { + throw std::runtime_error("invalid adaptor reference data slice"); + } + std::ifstream input(data_path, std::ios::binary); + input.seekg(static_cast(signed_offset * 4)); + std::vector values(static_cast(signed_count)); + for (float &value : values) { + uint8_t bytes[4]{}; + input.read(reinterpret_cast(bytes), 4); + if (!input) { + throw std::runtime_error("truncated adaptor reference data"); + } + const uint32_t bits = static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); + std::memcpy(&value, &bits, sizeof(value)); + } + return values; +} + +void require_close(const std::string &label, const std::vector &actual, + const std::vector &expected) { + if (actual.size() != expected.size()) { + throw std::runtime_error(label + " output size mismatch"); + } + float maximum_absolute_error = 0.0F; + float maximum_relative_error = 0.0F; + size_t worst_index = 0; + for (size_t index = 0; index < actual.size(); ++index) { + const float absolute_error = std::abs(actual[index] - expected[index]); + const float relative_error = + absolute_error / std::max(std::abs(expected[index]), 1.0e-12F); + if (absolute_error > maximum_absolute_error) { + maximum_absolute_error = absolute_error; + worst_index = index; + } + maximum_relative_error = std::max(maximum_relative_error, relative_error); + const float limit = + kAbsoluteTolerance + kRelativeTolerance * std::abs(expected[index]); + if (absolute_error > limit) { + throw std::runtime_error( + label + " mismatch at " + std::to_string(index) + ": expected " + + std::to_string(expected[index]) + ", got " + + std::to_string(actual[index]) + ", absolute error " + + std::to_string(absolute_error) + ", limit " + std::to_string(limit)); + } + } + std::cout << label << ": count=" << actual.size() + << ", max_abs=" << maximum_absolute_error + << ", max_rel=" << maximum_relative_error + << ", worst_index=" << worst_index << '\n'; +} + +void require_runtime_error(const std::string &label, const std::string &message, + const std::function &operation) { + try { + operation(); + } catch (const std::runtime_error &error) { + if (std::string(error.what()).find(message) != std::string::npos) { + return; + } + throw std::runtime_error(label + + " returned an unexpected error: " + error.what()); + } + throw std::runtime_error(label + " did not reject invalid input"); +} + +} // namespace + +int main(int argc, char **argv) try { + const auto arguments = parse_arguments(argc, argv); + const auto reference = engine::io::json::parse_file(arguments.reference); + if (reference.require("schema_version").as_i64() != 1 || + reference.require("transformers_commit").as_string() != + kTransformersCommit || + reference.require("model_revision").as_string() != kModelRevision || + reference.require("weight_catalog_count").as_i64() != 36 || + reference.require("data_format").as_string() != "little-endian-float32") { + throw std::runtime_error("unexpected adaptor reference metadata"); + } + + const auto input_shape = + engine::io::json::require_i64_array(reference.require("input"), "shape"); + const auto mask_shape = + engine::io::json::require_i64_array(reference.require("mask"), "shape"); + if (input_shape.size() != 3 || input_shape[0] != 2 || input_shape[1] != 4 || + input_shape[2] != 512 || mask_shape.size() != 2 || mask_shape[0] != 2 || + mask_shape[1] != 4) { + throw std::runtime_error("unexpected adaptor reference input shape"); + } + const auto input_values = + load_f32_slice(arguments.data, reference.require("input")); + const auto mask_values = + load_f32_slice(arguments.data, reference.require("mask")); + + auto assets = + engine::models::fun_asr_nano::load_fun_asr_nano_assets(arguments.model); + engine::core::ExecutionContext execution_context({arguments.backend, 0, 4}); + engine::models::fun_asr_nano::FunAsrNanoAdaptorRuntime runtime( + assets, execution_context, kGraphArenaBytes); + + std::unordered_map> actual_stages; + std::vector actual_packed; + std::vector + per_utterance; + for (int64_t batch = 0; batch < input_shape[0]; ++batch) { + engine::models::fun_asr_nano::FunAsrNanoEncoderEmbeddings embeddings; + const size_t row_offset = static_cast(batch * 4 * 512); + embeddings.values.assign(input_values.begin() + row_offset, + input_values.begin() + row_offset + 4 * 512); + embeddings.frames = 4; + embeddings.hidden_size = 512; + std::vector mask(4); + for (int64_t frame = 0; frame < 4; ++frame) { + mask[static_cast(frame)] = static_cast( + mask_values[static_cast(batch * 4 + frame)]); + } + embeddings.valid_frames = + std::count(mask.begin(), mask.end(), static_cast(1)); + auto output = runtime.adapt(embeddings, mask, true); + if (output.tokens != embeddings.valid_frames || + output.hidden_size != 1024 || output.stages.size() != 5) { + throw std::runtime_error("unexpected adaptor output contract"); + } + actual_packed.insert(actual_packed.end(), output.values.begin(), + output.values.end()); + for (const auto &stage : output.stages) { + auto &values = actual_stages[stage.name]; + values.insert(values.end(), stage.values.begin(), stage.values.end()); + } + per_utterance.push_back(std::move(output)); + } + + const auto &expected_stages = reference.require("checkpoints"); + for (const char *name : + {"packed_valid", "linear_1", "linear_2", "block_0", "block_1"}) { + const auto found = actual_stages.find(name); + if (found == actual_stages.end()) { + throw std::runtime_error(std::string("missing adaptor stage: ") + name); + } + require_close( + name, found->second, + load_f32_slice(arguments.data, expected_stages.require(name))); + } + require_close( + "output", actual_packed, + load_f32_slice(arguments.data, expected_stages.require("packed_valid"))); + + engine::models::fun_asr_nano::FunAsrNanoEncoderEmbeddings padded; + padded.values.assign(input_values.begin() + 4 * 512, + input_values.begin() + 8 * 512); + padded.frames = 4; + padded.valid_frames = 2; + padded.hidden_size = 512; + std::fill(padded.values.begin() + 2 * 512, padded.values.end(), -9999.0F); + const std::vector padded_mask = {1, 1, 0, 0}; + const auto changed_padding = runtime.adapt(padded, padded_mask, false); + if (changed_padding.values != per_utterance[1].values) { + throw std::runtime_error("adaptor output depends on padded frame values"); + } + const auto cached = runtime.adapt(padded, padded_mask, false); + if (cached.values != changed_padding.values) { + throw std::runtime_error("cached adaptor graph changed its output"); + } + + require_runtime_error("non-contiguous mask", "right-padded", [&] { + static_cast(runtime.adapt(padded, {1, 0, 1, 0}, false)); + }); + require_runtime_error("mask size", "mask size", [&] { + static_cast(runtime.adapt(padded, {1, 1}, false)); + }); + auto wrong_hidden = padded; + wrong_hidden.hidden_size = 256; + require_runtime_error("hidden size", "shape", [&] { + static_cast(runtime.adapt(wrong_hidden, padded_mask, false)); + }); + auto wrong_values = padded; + wrong_values.values.pop_back(); + require_runtime_error("value count", "value count", [&] { + static_cast(runtime.adapt(wrong_values, padded_mask, false)); + }); + + std::cout << "Fun-ASR-Nano adaptor parity passed\n"; + return 0; +} catch (const std::exception &error) { + std::cerr << "fun_asr_nano_adaptor_probe: " << error.what() << '\n'; + return 1; +} diff --git a/tests/fun_asr_nano/fun_asr_nano_decoder_probe.cpp b/tests/fun_asr_nano/fun_asr_nano_decoder_probe.cpp new file mode 100644 index 00000000..954e152b --- /dev/null +++ b/tests/fun_asr_nano/fun_asr_nano_decoder_probe.cpp @@ -0,0 +1,304 @@ +#include "engine/framework/core/execution_context.h" +#include "engine/framework/io/json.h" +#include "engine/models/fun_asr_nano/assets.h" +#include "engine/models/fun_asr_nano/decoder.h" +#include "engine/models/fun_asr_nano/prompt.h" +#include "engine/models/fun_asr_nano/tokenizer_text.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t kPrefillGraphArenaBytes = 256 * 1024 * 1024; +constexpr size_t kDecodeGraphArenaBytes = 128 * 1024 * 1024; +constexpr size_t kWeightContextBytes = 32 * 1024 * 1024; +constexpr float kAbsoluteTolerance = 3.0e-3F; +constexpr float kRelativeTolerance = 3.0e-3F; +constexpr const char *kTransformersCommit = + "f9966442ac24fff57060774ce22e1884760f4a3b"; +constexpr const char *kModelRevision = + "854d88f94205cd17d2afdb24332130d86fbe654a"; + +struct Arguments { + engine::core::BackendType backend = engine::core::BackendType::Cpu; + std::filesystem::path model; + std::filesystem::path reference; + std::filesystem::path data; +}; + +Arguments parse_arguments(int argc, char **argv) { + Arguments result; + for (int index = 1; index < argc; ++index) { + const std::string option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + option); + } + const std::string value(argv[++index]); + if (option == "--backend") { + if (value == "cpu") { + result.backend = engine::core::BackendType::Cpu; + } else if (value == "cuda") { + result.backend = engine::core::BackendType::Cuda; + } else { + throw std::runtime_error("unsupported decoder probe backend: " + value); + } + } else if (option == "--model") { + result.model = value; + } else if (option == "--reference") { + result.reference = value; + } else if (option == "--data") { + result.data = value; + } else { + throw std::runtime_error("unknown decoder probe option: " + option); + } + } + if (result.model.empty() || result.reference.empty() || result.data.empty()) { + throw std::runtime_error("--model, --reference, and --data are required"); + } + return result; +} + +std::vector load_f32_slice(const std::filesystem::path &data_path, + const engine::io::json::Value &descriptor) { + const int64_t signed_offset = descriptor.require("offset_f32").as_i64(); + const int64_t signed_count = descriptor.require("count").as_i64(); + if (signed_offset < 0 || signed_count < 0 || + static_cast(signed_count) > + std::numeric_limits::max()) { + throw std::runtime_error("invalid decoder reference data slice"); + } + std::ifstream input(data_path, std::ios::binary); + input.seekg(static_cast(signed_offset * 4)); + std::vector values(static_cast(signed_count)); + for (float &value : values) { + uint8_t bytes[4]{}; + input.read(reinterpret_cast(bytes), 4); + if (!input) { + throw std::runtime_error("truncated decoder reference data"); + } + const uint32_t bits = static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); + std::memcpy(&value, &bits, sizeof(value)); + } + return values; +} + +void require_equal(const std::string &label, const std::vector &actual, + const std::vector &expected) { + if (actual.size() != expected.size()) { + throw std::runtime_error(label + " size mismatch"); + } + for (size_t index = 0; index < actual.size(); ++index) { + if (actual[index] != expected[index]) { + throw std::runtime_error(label + " mismatch at " + std::to_string(index)); + } + } +} + +int32_t argmax_index(const std::vector &values) { + if (values.empty()) { + throw std::runtime_error("cannot select from empty decoder logits"); + } + return static_cast(std::distance( + values.begin(), std::max_element(values.begin(), values.end()))); +} + +void require_close(const std::string &label, const std::vector &actual, + const std::vector &expected) { + if (actual.size() != expected.size()) { + throw std::runtime_error(label + " output size mismatch"); + } + float maximum_absolute_error = 0.0F; + float maximum_relative_error = 0.0F; + size_t worst_index = 0; + for (size_t index = 0; index < actual.size(); ++index) { + const float absolute_error = std::abs(actual[index] - expected[index]); + const float relative_error = + absolute_error / std::max(std::abs(expected[index]), 1.0e-12F); + if (absolute_error > maximum_absolute_error) { + maximum_absolute_error = absolute_error; + worst_index = index; + } + maximum_relative_error = std::max(maximum_relative_error, relative_error); + const float limit = + kAbsoluteTolerance + kRelativeTolerance * std::abs(expected[index]); + if (absolute_error > limit) { + throw std::runtime_error( + label + " mismatch at " + std::to_string(index) + ": expected " + + std::to_string(expected[index]) + ", got " + + std::to_string(actual[index]) + ", absolute error " + + std::to_string(absolute_error) + ", limit " + std::to_string(limit)); + } + } + std::cout << label << ": count=" << actual.size() + << ", max_abs=" << maximum_absolute_error + << ", max_rel=" << maximum_relative_error + << ", worst_index=" << worst_index << '\n'; +} + +void require_runtime_error(const std::string &label, const std::string &message, + const std::function &operation) { + try { + operation(); + } catch (const std::runtime_error &error) { + if (std::string(error.what()).find(message) != std::string::npos) { + return; + } + throw std::runtime_error(label + + " returned an unexpected error: " + error.what()); + } + throw std::runtime_error(label + " did not reject invalid input"); +} + +} // namespace + +int main(int argc, char **argv) try { + const auto arguments = parse_arguments(argc, argv); + const auto reference = engine::io::json::parse_file(arguments.reference); + if (reference.require("schema_version").as_i64() != 1 || + reference.require("transformers_commit").as_string() != + kTransformersCommit || + reference.require("model_revision").as_string() != kModelRevision || + reference.require("text_weight_catalog_count").as_i64() != 310 || + reference.require("data_format").as_string() != "little-endian-float32") { + throw std::runtime_error("unexpected decoder reference metadata"); + } + + auto assets = + engine::models::fun_asr_nano::load_fun_asr_nano_assets(arguments.model); + engine::models::fun_asr_nano::FunAsrNanoTextTokenizer tokenizer(assets); + if (tokenizer.require_token_id("<|im_start|>") != + reference.require("im_start_token_id").as_i64() || + tokenizer.require_token_id("<|im_end|>") != + reference.require("im_end_token_id").as_i64() || + tokenizer.require_token_id("<|object_ref_start|>") != + reference.require("audio_token_id").as_i64()) { + throw std::runtime_error( + "Fun-ASR-Nano special token ids differ from reference"); + } + + engine::models::fun_asr_nano::FunAsrNanoPromptBuilder prompt_builder(assets); + engine::models::fun_asr_nano::FunAsrNanoPromptRequest prompt_request; + auto prompt = prompt_builder.build(prompt_request, 2); + const auto &itn_reference = reference.require("itn"); + require_equal( + "ITN prompt ids", prompt.input_ids, + engine::io::json::require_i64_array(itn_reference, "input_ids")); + require_equal("ITN audio positions", prompt.audio_token_positions, + engine::io::json::require_i64_array(itn_reference, + "audio_token_positions")); + + prompt_request.enable_itn = false; + const auto no_itn_prompt = prompt_builder.build(prompt_request, 2); + const auto &no_itn_reference = reference.require("no_itn"); + require_equal( + "non-ITN prompt ids", no_itn_prompt.input_ids, + engine::io::json::require_i64_array(no_itn_reference, "input_ids")); + require_equal("non-ITN audio positions", no_itn_prompt.audio_token_positions, + engine::io::json::require_i64_array(no_itn_reference, + "audio_token_positions")); + + engine::models::fun_asr_nano::FunAsrNanoAdaptorEmbeddings audio_embeddings; + audio_embeddings.values = + load_f32_slice(arguments.data, reference.require("audio_embeddings")); + audio_embeddings.tokens = 2; + audio_embeddings.hidden_size = 1024; + + engine::core::ExecutionContext execution_context({arguments.backend, 0, 4}); + engine::models::fun_asr_nano::FunAsrNanoDecoderRuntime decoder( + assets, execution_context, kPrefillGraphArenaBytes, + kDecodeGraphArenaBytes, kWeightContextBytes, + engine::assets::TensorStorageType::F32); + engine::models::fun_asr_nano::FunAsrNanoGenerationOptions options; + options.max_new_tokens = 3; + options.capture_logits = true; + const auto output = decoder.generate(prompt, audio_embeddings, options); + const auto greedy_ids = + engine::io::json::require_i64_array(reference, "greedy_token_ids"); + if (output.step_logits.size() != greedy_ids.size()) { + throw std::runtime_error("decoder trace step count mismatch"); + } + const auto &logit_references = reference.require("logits").as_array(); + if (logit_references.size() != output.step_logits.size()) { + throw std::runtime_error("decoder reference logit count mismatch"); + } + for (size_t step = 0; step < output.step_logits.size(); ++step) { + require_close("decoder logits " + std::to_string(step), + output.step_logits[step], + load_f32_slice(arguments.data, logit_references[step])); + if (argmax_index(output.step_logits[step]) != greedy_ids[step]) { + throw std::runtime_error("decoder greedy token mismatch at step " + + std::to_string(step)); + } + } + std::vector expected_output(greedy_ids.begin(), + greedy_ids.end() - 1); + require_equal("generated non-EOS ids", output.token_ids, expected_output); + if (tokenizer.decode(output.token_ids) != + reference.require("generated_text").as_string()) { + throw std::runtime_error("decoder text differs from reference"); + } + + const auto cached = decoder.generate(prompt, audio_embeddings, options); + if (cached.token_ids != output.token_ids || + cached.step_logits != output.step_logits) { + throw std::runtime_error("cached decoder graphs changed their output"); + } + + auto wrong_audio = audio_embeddings; + wrong_audio.tokens = 1; + require_runtime_error("audio token count", "token count", [&] { + static_cast(decoder.generate(prompt, wrong_audio, options)); + }); + auto wrong_position = prompt; + wrong_position.audio_token_positions[0] = 0; + require_runtime_error("audio position", "cover every", [&] { + static_cast( + decoder.generate(wrong_position, audio_embeddings, options)); + }); + auto duplicate_position = prompt; + duplicate_position.audio_token_positions[1] = + duplicate_position.audio_token_positions[0]; + require_runtime_error("duplicate audio position", "cover every", [&] { + static_cast( + decoder.generate(duplicate_position, audio_embeddings, options)); + }); + auto wrong_mask = prompt; + wrong_mask.attention_mask.pop_back(); + require_runtime_error("prompt mask", "unpadded prompt mask", [&] { + static_cast(decoder.generate(wrong_mask, audio_embeddings, options)); + }); + auto wrong_options = options; + wrong_options.max_new_tokens = 0; + require_runtime_error("generation length", "must be positive", [&] { + static_cast( + decoder.generate(prompt, audio_embeddings, wrong_options)); + }); + wrong_options.max_new_tokens = std::numeric_limits::max(); + require_runtime_error("generation overflow", "max_position_embeddings", [&] { + static_cast( + decoder.generate(prompt, audio_embeddings, wrong_options)); + }); + require_runtime_error("audio token length", "positive audio token", [&] { + static_cast(prompt_builder.build(prompt_request, 0)); + }); + + std::cout << "Fun-ASR-Nano decoder parity passed\n"; + return 0; +} catch (const std::exception &error) { + std::cerr << "fun_asr_nano_decoder_probe: " << error.what() << '\n'; + return 1; +} diff --git a/tests/fun_asr_nano/fun_asr_nano_encoder_probe.cpp b/tests/fun_asr_nano/fun_asr_nano_encoder_probe.cpp new file mode 100644 index 00000000..077696df --- /dev/null +++ b/tests/fun_asr_nano/fun_asr_nano_encoder_probe.cpp @@ -0,0 +1,255 @@ +#include "engine/framework/core/execution_context.h" +#include "engine/framework/io/json.h" +#include "engine/models/fun_asr_nano/assets.h" +#include "engine/models/fun_asr_nano/encoder.h" +#include "engine/models/fun_asr_nano/frontend.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t kGraphArenaBytes = 128 * 1024 * 1024; +constexpr float kAbsoluteTolerance = 2.0e-3F; +constexpr float kRelativeTolerance = 2.0e-3F; +constexpr const char *kTransformersCommit = + "48e7f65fb274172e15aa88875d780c67c37606c7"; +constexpr const char *kModelRevision = + "854d88f94205cd17d2afdb24332130d86fbe654a"; + +struct Arguments { + engine::core::BackendType backend = engine::core::BackendType::Cpu; + std::filesystem::path model; + std::filesystem::path reference; + std::filesystem::path data; +}; + +Arguments parse_arguments(int argc, char **argv) { + Arguments arguments; + for (int index = 1; index < argc; ++index) { + const std::string option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + option); + } + const std::string value(argv[++index]); + if (option == "--backend") { + if (value == "cpu") { + arguments.backend = engine::core::BackendType::Cpu; + } else if (value == "cuda") { + arguments.backend = engine::core::BackendType::Cuda; + } else { + throw std::runtime_error("unsupported encoder probe backend: " + value); + } + } else if (option == "--model") { + arguments.model = value; + } else if (option == "--reference") { + arguments.reference = value; + } else if (option == "--data") { + arguments.data = value; + } else { + throw std::runtime_error("unknown encoder probe option: " + option); + } + } + if (arguments.model.empty() || arguments.reference.empty() || + arguments.data.empty()) { + throw std::runtime_error("--model, --reference, and --data are required"); + } + return arguments; +} + +std::vector load_f32_slice(const std::filesystem::path &data_path, + const engine::io::json::Value &descriptor) { + const int64_t signed_offset = descriptor.require("offset_f32").as_i64(); + const int64_t signed_count = descriptor.require("count").as_i64(); + if (signed_offset < 0 || signed_count < 0 || + static_cast(signed_count) > + std::numeric_limits::max()) { + throw std::runtime_error("invalid encoder reference data slice"); + } + const uint64_t offset = static_cast(signed_offset); + const size_t count = static_cast(signed_count); + std::ifstream input(data_path, std::ios::binary); + input.seekg(static_cast(offset * sizeof(float))); + if (!input) { + throw std::runtime_error("could not seek in encoder reference data"); + } + std::vector values(count); + for (float &value : values) { + uint8_t bytes[4]{}; + input.read(reinterpret_cast(bytes), 4); + if (!input) { + throw std::runtime_error("truncated encoder reference data"); + } + const uint32_t bits = static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); + std::memcpy(&value, &bits, sizeof(value)); + } + return values; +} + +void require_close(const std::string &label, const std::vector &actual, + const std::vector &expected) { + if (actual.size() != expected.size()) { + throw std::runtime_error(label + " output size mismatch"); + } + float maximum_absolute_error = 0.0F; + float maximum_relative_error = 0.0F; + size_t worst_index = 0; + for (size_t index = 0; index < actual.size(); ++index) { + const float absolute_error = std::abs(actual[index] - expected[index]); + const float relative_error = + absolute_error / std::max(std::abs(expected[index]), 1.0e-12F); + if (absolute_error > maximum_absolute_error) { + maximum_absolute_error = absolute_error; + worst_index = index; + } + maximum_relative_error = std::max(maximum_relative_error, relative_error); + const float limit = + kAbsoluteTolerance + kRelativeTolerance * std::abs(expected[index]); + if (absolute_error > limit) { + throw std::runtime_error( + label + " mismatch at " + std::to_string(index) + ": expected " + + std::to_string(expected[index]) + ", got " + + std::to_string(actual[index]) + ", absolute error " + + std::to_string(absolute_error) + ", limit " + std::to_string(limit)); + } + } + std::cout << label << ": count=" << actual.size() + << ", max_abs=" << maximum_absolute_error + << ", max_rel=" << maximum_relative_error + << ", worst_index=" << worst_index << '\n'; +} + +void require_runtime_error(const std::string &label, const std::string &message, + const std::function &operation) { + try { + operation(); + } catch (const std::runtime_error &error) { + if (std::string(error.what()).find(message) != std::string::npos) { + return; + } + throw std::runtime_error(label + + " returned an unexpected error: " + error.what()); + } + throw std::runtime_error(label + " did not reject invalid input"); +} + +} // namespace + +int main(int argc, char **argv) try { + const auto arguments = parse_arguments(argc, argv); + const auto reference = engine::io::json::parse_file(arguments.reference); + if (reference.require("schema_version").as_i64() != 1 || + reference.require("transformers_commit").as_string() != + kTransformersCommit || + reference.require("model_revision").as_string() != kModelRevision || + reference.require("weight_catalog_count").as_i64() != 1194 || + reference.require("data_format").as_string() != "little-endian-float32") { + throw std::runtime_error("unexpected encoder reference metadata"); + } + + const auto input_shape = + engine::io::json::require_i64_array(reference.require("input"), "shape"); + if (input_shape.size() != 3 || input_shape[0] != 1 || input_shape[1] <= 0 || + input_shape[2] != 560) { + throw std::runtime_error("unexpected encoder reference input shape"); + } + engine::models::fun_asr_nano::FunAsrNanoAudioFeatures features; + features.values = load_f32_slice(arguments.data, reference.require("input")); + features.frames = input_shape[1]; + features.feature_dim = input_shape[2]; + features.valid_frames = reference.require("valid_frames").as_i64(); + + auto assets = + engine::models::fun_asr_nano::load_fun_asr_nano_assets(arguments.model); + engine::models::fun_asr_nano::FunAsrNanoFrontend frontend( + assets->config.frontend); + const auto frontend_output = + frontend.extract(std::vector(800, 0.0F), 16000); + if (frontend_output.frames <= 0 || frontend_output.feature_dim != 560 || + frontend_output.valid_frames != frontend_output.frames || + static_cast(frontend_output.values.size()) != + frontend_output.frames * frontend_output.feature_dim) { + throw std::runtime_error("unexpected encoder frontend contract"); + } + bool rejected_sample_rate = false; + try { + static_cast(frontend.extract(std::vector(800, 0.0F), 8000)); + } catch (const std::runtime_error &) { + rejected_sample_rate = true; + } + if (!rejected_sample_rate) { + throw std::runtime_error( + "Fun-ASR-Nano frontend accepted a non-16-kHz sample rate"); + } + engine::core::ExecutionContext execution_context({arguments.backend, 0, 4}); + engine::models::fun_asr_nano::FunAsrNanoEncoderRuntime runtime( + std::move(assets), execution_context, kGraphArenaBytes); + const auto output = runtime.encode(features, true); + if (output.frames != features.frames || + output.valid_frames != features.valid_frames || + output.hidden_size != 512 || output.stages.size() != 9) { + throw std::runtime_error("unexpected encoder output contract"); + } + + std::unordered_map> actual_stages; + for (const auto &stage : output.stages) { + if (!actual_stages.emplace(stage.name, stage.values).second) { + throw std::runtime_error("duplicate encoder stage: " + stage.name); + } + } + const auto &expected_stages = reference.require("checkpoints"); + for (const char *name : + {"stem", "main_layer_0", "main_layer_24", "main_layer_48", + "main_layer_norm", "timestamp_layer_0", "timestamp_layer_10", + "timestamp_layer_19", "final"}) { + const auto found = actual_stages.find(name); + if (found == actual_stages.end()) { + throw std::runtime_error("missing encoder stage: " + std::string(name)); + } + require_close( + name, found->second, + load_f32_slice(arguments.data, expected_stages.require(name))); + } + require_close( + "final_output", output.values, + load_f32_slice(arguments.data, expected_stages.require("final"))); + + const auto cached_output = runtime.encode(features, false); + if (!cached_output.stages.empty()) { + throw std::runtime_error( + "encoder graph cache returned unrequested stage outputs"); + } + require_close("cached_output", cached_output.values, output.values); + + auto padded_features = features; + --padded_features.valid_frames; + require_runtime_error("encoder padding contract", "unpadded", [&] { + static_cast(runtime.encode(padded_features)); + }); + auto malformed_features = features; + malformed_features.values.pop_back(); + require_runtime_error("encoder value count", "value count", [&] { + static_cast(runtime.encode(malformed_features)); + }); + require_runtime_error("encoder position capacity", "positional capacity", + [&] { runtime.prepare_capacity(2049); }); + return 0; +} catch (const std::exception &error) { + std::cerr << "fun_asr_nano_encoder_probe failed: " << error.what() << '\n'; + return 1; +} diff --git a/tests/fun_asr_nano/fun_asr_nano_frontend_probe.cpp b/tests/fun_asr_nano/fun_asr_nano_frontend_probe.cpp new file mode 100644 index 00000000..6cef4239 --- /dev/null +++ b/tests/fun_asr_nano/fun_asr_nano_frontend_probe.cpp @@ -0,0 +1,268 @@ +#include "engine/framework/audio/kaldi_fbank.h" +#include "engine/framework/audio/wav_reader.h" +#include "engine/framework/io/json.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr int kSampleRate = 16000; +constexpr float kAbsoluteTolerance = 2.0e-4F; +constexpr float kRelativeTolerance = 2.0e-4F; +constexpr const char *kTransformersCommit = + "48e7f65fb274172e15aa88875d780c67c37606c7"; + +template +void expect_runtime_error(Callable &&callable, const std::string &context) { + try { + callable(); + } catch (const std::runtime_error &) { + return; + } + throw std::runtime_error(context + " did not reject invalid options"); +} + +std::vector load_f32_slice(const std::filesystem::path &data_path, + const engine::io::json::Value &descriptor) { + const int64_t signed_offset = descriptor.require("offset_f32").as_i64(); + const int64_t signed_count = descriptor.require("count").as_i64(); + if (signed_offset < 0 || signed_count < 0 || + static_cast(signed_count) > + std::numeric_limits::max()) { + throw std::runtime_error("invalid frontend reference data slice"); + } + const auto offset = static_cast(signed_offset); + const auto count = static_cast(signed_count); + std::ifstream input(data_path, std::ios::binary); + input.seekg(static_cast(offset * 4)); + if (!input) { + throw std::runtime_error("could not seek in frontend reference data"); + } + + std::vector values(static_cast(count)); + for (float &value : values) { + uint8_t bytes[4]{}; + input.read(reinterpret_cast(bytes), 4); + if (!input) { + throw std::runtime_error("truncated frontend reference data"); + } + const uint32_t bits = static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); + static_assert(sizeof(value) == sizeof(bits)); + std::memcpy(&value, &bits, sizeof(value)); + } + return values; +} + +std::vector downmix(const engine::audio::WavData &wav) { + if (wav.sample_rate != kSampleRate || wav.channels <= 0) { + throw std::runtime_error( + "sample WAV must be 16 kHz with at least one channel"); + } + if (wav.samples.size() % static_cast(wav.channels) != 0) { + throw std::runtime_error("sample WAV has an incomplete interleaved frame"); + } + const size_t frames = wav.samples.size() / static_cast(wav.channels); + std::vector mono(frames, 0.0F); + for (size_t frame = 0; frame < frames; ++frame) { + double sum = 0.0; + for (int channel = 0; channel < wav.channels; ++channel) { + sum += wav.samples[frame * static_cast(wav.channels) + + static_cast(channel)]; + } + mono[frame] = static_cast(sum / static_cast(wav.channels)); + } + return mono; +} + +std::vector make_fixture(const std::string &name, + const engine::io::json::Value &expected, + const std::filesystem::path &data_path, + const std::filesystem::path &sample_wav) { + if (name == "sample_16k") { + return downmix(engine::audio::read_wav_f32(sample_wav)); + } + if (const auto *waveform = expected.find("waveform")) { + return load_f32_slice(data_path, *waveform); + } + throw std::runtime_error( + "synthetic frontend fixture is missing its reference waveform: " + name); +} + +void compare_fixture(const std::string &name, + const engine::io::json::Value &expected, + const std::filesystem::path &data_path, + const std::filesystem::path &sample_wav) { + engine::audio::KaldiFbankOptions options; + options.sample_rate = kSampleRate; + options.num_mels = 80; + options.frame_length_ms = 25.0F; + options.frame_shift_ms = 10.0F; + options.lfr_m = 7; + options.lfr_n = 6; + options.upscale_samples = false; + options.apply_cmvn = false; + + const auto audio = make_fixture(name, expected, data_path, sample_wav); + const auto actual = engine::audio::extract_kaldi_fbank(audio, options); + const int expected_frames = + static_cast(expected.require("frames").as_i64()); + const int expected_width = + static_cast(expected.require("width").as_i64()); + const auto expected_values = + load_f32_slice(data_path, expected.require("features")); + + if (actual.frames != expected_frames || + actual.feature_dim != expected_width) { + throw std::runtime_error(name + " shape mismatch: expected " + + std::to_string(expected_frames) + "x" + + std::to_string(expected_width) + ", got " + + std::to_string(actual.frames) + "x" + + std::to_string(actual.feature_dim)); + } + if (actual.values.size() != expected_values.size()) { + throw std::runtime_error(name + " flattened feature count mismatch"); + } + + float max_absolute_error = 0.0F; + float max_relative_error = 0.0F; + size_t worst_index = 0; + for (size_t index = 0; index < actual.values.size(); ++index) { + const float absolute_error = + std::abs(actual.values[index] - expected_values[index]); + const float relative_error = + absolute_error / std::max(std::abs(expected_values[index]), 1.0e-12F); + if (absolute_error > max_absolute_error) { + max_absolute_error = absolute_error; + worst_index = index; + } + max_relative_error = std::max(max_relative_error, relative_error); + const float limit = kAbsoluteTolerance + + kRelativeTolerance * std::abs(expected_values[index]); + if (absolute_error > limit) { + throw std::runtime_error( + name + " parity mismatch at index " + std::to_string(index) + + ": expected " + std::to_string(expected_values[index]) + ", got " + + std::to_string(actual.values[index]) + ", absolute error " + + std::to_string(absolute_error) + ", limit " + std::to_string(limit)); + } + } + + std::cout << name << ": " << actual.frames << "x" << actual.feature_dim + << ", max_abs=" << max_absolute_error + << ", max_rel=" << max_relative_error + << ", worst_index=" << worst_index << '\n'; +} + +void check_frontend_contracts() { + engine::audio::KaldiFbankOptions options; + const auto short_result = engine::audio::extract_kaldi_fbank( + std::vector(399, 0.0F), options); + if (short_result.frames != 0 || short_result.feature_dim != 560 || + !short_result.values.empty()) { + throw std::runtime_error( + "short audio must return an empty, dimensioned feature result"); + } + + auto invalid = options; + invalid.lfr_n = 0; + expect_runtime_error( + [&] { + static_cast(engine::audio::extract_kaldi_fbank({}, invalid)); + }, + "zero LFR stride"); + + invalid = options; + invalid.low_frequency = 9000.0F; + expect_runtime_error( + [&] { + static_cast(engine::audio::extract_kaldi_fbank({}, invalid)); + }, + "frequency bounds"); + + invalid = options; + invalid.preemphasis = std::numeric_limits::quiet_NaN(); + expect_runtime_error( + [&] { + static_cast(engine::audio::extract_kaldi_fbank({}, invalid)); + }, + "non-finite preemphasis"); + + invalid = options; + invalid.low_frequency = std::numeric_limits::quiet_NaN(); + expect_runtime_error( + [&] { + static_cast(engine::audio::extract_kaldi_fbank({}, invalid)); + }, + "non-finite frequency bounds"); + + std::vector impulse(400, 0.0F); + impulse[200] = 1.0F; + const auto baseline = engine::audio::extract_kaldi_fbank(impulse, options); + options.apply_cmvn = true; + options.cmvn_shift.assign(560, 1.0F); + options.cmvn_scale.assign(560, 2.0F); + const auto normalized = engine::audio::extract_kaldi_fbank(impulse, options); + if (normalized.frames != baseline.frames || + normalized.values.size() != baseline.values.size()) { + throw std::runtime_error("CMVN changed the frontend shape"); + } + for (size_t index = 0; index < baseline.values.size(); ++index) { + const float expected = (baseline.values[index] + 1.0F) * 2.0F; + if (std::abs(normalized.values[index] - expected) > 1.0e-6F) { + throw std::runtime_error("CMVN did not apply (feature + shift) * scale"); + } + } +} + +} // namespace + +int main(int argc, char **argv) try { + if (argc != 4) { + std::cerr << "usage: fun_asr_nano_frontend_probe " + " \n"; + return 2; + } + + const auto reference = + engine::io::json::parse_file(std::filesystem::path(argv[1])); + if (reference.require("schema_version").as_i64() != 2 || + reference.require("reference").as_string() != + "transformers.FunAsrNanoFeatureExtractor" || + reference.require("transformers_commit").as_string() != + kTransformersCommit || + reference.require("data_format").as_string() != "little-endian-float32" || + reference.require("sample_rate").as_i64() != kSampleRate || + reference.require("feature_size").as_i64() != 80 || + reference.require("lfr_m").as_i64() != 7 || + reference.require("lfr_n").as_i64() != 6) { + throw std::runtime_error("unexpected frontend reference metadata"); + } + + const auto &fixtures = reference.require("fixtures"); + const std::filesystem::path data_path(argv[2]); + const std::filesystem::path sample_wav(argv[3]); + check_frontend_contracts(); + for (const std::string name : + {"silence", "impulse", "sine_440hz", "sample_16k"}) { + compare_fixture(name, fixtures.require(name), data_path, sample_wav); + } + return 0; +} catch (const std::exception &error) { + std::cerr << "fun_asr_nano_frontend_probe failed: " << error.what() << '\n'; + return 1; +} diff --git a/tests/fun_asr_nano/fun_asr_nano_session_probe.cpp b/tests/fun_asr_nano/fun_asr_nano_session_probe.cpp new file mode 100644 index 00000000..74bfd829 --- /dev/null +++ b/tests/fun_asr_nano/fun_asr_nano_session_probe.cpp @@ -0,0 +1,208 @@ +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/wav_reader.h" +#include "engine/framework/runtime/registry.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Arguments { + engine::core::BackendType backend = engine::core::BackendType::Cpu; + std::filesystem::path model; + std::filesystem::path audio; +}; + +Arguments parse_arguments(int argc, char **argv) { + Arguments result; + for (int index = 1; index < argc; ++index) { + if (index + 1 >= argc) { + throw std::runtime_error("missing probe argument value"); + } + const std::string option(argv[index]); + const std::string value(argv[++index]); + if (option == "--backend") { + if (value == "cpu") { + result.backend = engine::core::BackendType::Cpu; + } else if (value == "cuda") { + result.backend = engine::core::BackendType::Cuda; + } else { + throw std::runtime_error("unsupported probe backend: " + value); + } + } else if (option == "--model") { + result.model = value; + } else if (option == "--audio") { + result.audio = value; + } else { + throw std::runtime_error("unknown probe argument: " + option); + } + } + if (result.model.empty() || result.audio.empty()) { + throw std::runtime_error("--model and --audio are required"); + } + return result; +} + +void require(bool condition, const std::string &message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +void require_runtime_error(const std::string &needle, + const std::function &operation) { + try { + operation(); + } catch (const std::runtime_error &error) { + if (std::string(error.what()).find(needle) != std::string::npos) { + return; + } + throw std::runtime_error("unexpected error: " + std::string(error.what())); + } + throw std::runtime_error("operation did not reject invalid input: " + needle); +} + +bool has_option(const std::vector &options, + const std::string &name) { + return std::any_of(options.begin(), options.end(), + [&](const auto &option) { return option.name == name; }); +} + +engine::runtime::AudioBuffer read_audio(const std::filesystem::path &path) { + const auto wav = engine::audio::read_wav_f32(path); + return {wav.sample_rate, wav.channels, wav.samples}; +} + +engine::runtime::AudioBuffer +make_resampled_stereo(const engine::runtime::AudioBuffer &source) { + auto mono = engine::audio::convert_interleaved_audio_to_mono_linear_resampled( + source.samples, source.sample_rate, source.channels, 8000); + std::vector stereo; + stereo.reserve(mono.size() * 2); + for (const float sample : mono) { + stereo.push_back(sample); + stereo.push_back(sample); + } + return {8000, 2, std::move(stereo)}; +} + +} // namespace + +int main(int argc, char **argv) try { + const auto arguments = parse_arguments(argc, argv); + auto registry = engine::runtime::make_default_registry(); + engine::runtime::ModelLoadRequest load_request; + load_request.model_path = arguments.model; + load_request.family_hint = "fun_asr_nano"; + + const auto inspection = registry.inspect(load_request); + require(inspection.metadata.family == "fun_asr_nano", + "loader family mismatch"); + require(inspection.capabilities.supported_tasks.size() == 1, + "loader must advertise exactly one task"); + require(inspection.capabilities.supported_tasks.front().task == + engine::runtime::VoiceTaskKind::Asr, + "loader must advertise ASR"); + require(inspection.capabilities.supported_tasks.front().modes == + std::vector{ + engine::runtime::RunMode::Offline}, + "loader must advertise offline mode only"); + require(!inspection.capabilities.supports_timestamps, + "loader must not advertise timestamps"); + for (const char *name : {"language", "enable_itn", "max_tokens", + "audio_chunk_mode", "audio_chunk_seconds"}) { + require(has_option(inspection.cli.request_options, name), + std::string("missing request option: ") + name); + } + + auto model = registry.load(load_request); + require_runtime_error("offline", [&] { + (void)model->create_task_session({engine::runtime::VoiceTaskKind::Asr, + engine::runtime::RunMode::Streaming}, + {}); + }); + + engine::runtime::SessionOptions session_options; + session_options.backend = {arguments.backend, 0, 4}; + session_options.options["fun_asr_nano.weight_type"] = "f32"; + auto invalid_session_options = session_options; + invalid_session_options.options["fun_asr_nano.unknown"] = "1"; + require_runtime_error("unknown Fun-ASR-Nano session option", [&] { + (void)model->create_task_session({engine::runtime::VoiceTaskKind::Asr, + engine::runtime::RunMode::Offline}, + invalid_session_options); + }); + auto session_base = model->create_task_session( + {engine::runtime::VoiceTaskKind::Asr, engine::runtime::RunMode::Offline}, + session_options); + auto *session = dynamic_cast( + session_base.get()); + require(session != nullptr, "loader did not create an offline session"); + + const auto audio = read_audio(arguments.audio); + engine::runtime::TaskRequest request; + request.audio_input = audio; + request.options["language"] = "auto"; + request.options["enable_itn"] = "true"; + request.options["max_tokens"] = "1"; + request.options["audio_chunk_mode"] = "none"; + require_runtime_error("prepare", [&] { (void)session->run(request); }); + + session->prepare(engine::runtime::build_preparation_request(request)); + const auto first = session->run(request); + require(first.text_output.has_value(), "session omitted text output"); + require(first.word_timestamps.empty(), "session emitted timestamps"); + + request.audio_input = make_resampled_stereo(audio); + const auto resampled = session->run(request); + require(resampled.text_output.has_value(), + "resampled stereo request omitted text output"); + + request.options["max_tokens"] = "0"; + require_runtime_error("max_tokens", [&] { (void)session->run(request); }); + request.options["max_tokens"] = "1"; + request.options["enable_itn"] = "maybe"; + require_runtime_error("enable_itn", [&] { (void)session->run(request); }); + request.options["enable_itn"] = "true"; + request.options["audio_chunk_mode"] = "vad"; + require_runtime_error("audio_chunk_mode", + [&] { (void)session->run(request); }); + request.options["audio_chunk_mode"] = "fixed"; + request.options["audio_chunk_seconds"] = "inf"; + require_runtime_error("audio_chunk_seconds", + [&] { (void)session->run(request); }); + const int saved_sample_rate = request.audio_input->sample_rate; + request.audio_input->sample_rate = 1; + request.options["audio_chunk_seconds"] = "9223372036854775808"; + require_runtime_error("audio_chunk_seconds is too large", + [&] { (void)session->run(request); }); + request.audio_input->sample_rate = saved_sample_rate; + request.options.erase("audio_chunk_seconds"); + request.options["return_timestamps"] = "true"; + require_runtime_error("unknown Fun-ASR-Nano request option", + [&] { (void)session->run(request); }); + request.options.erase("return_timestamps"); + request.options["langauge"] = "en"; + require_runtime_error("unknown Fun-ASR-Nano request option", + [&] { (void)session->run(request); }); + + session_base.reset(); + auto second_session = model->create_task_session( + {engine::runtime::VoiceTaskKind::Asr, engine::runtime::RunMode::Offline}, + session_options); + require(dynamic_cast( + second_session.get()) != nullptr, + "loaded model could not create a second offline session"); + + std::cout << "fun_asr_nano_session_probe passed\n"; + return 0; +} catch (const std::exception &error) { + std::cerr << "fun_asr_nano_session_probe failed: " << error.what() << '\n'; + return 1; +} diff --git a/tests/fun_asr_nano/fun_asr_nano_warm_bench.cpp b/tests/fun_asr_nano/fun_asr_nano_warm_bench.cpp new file mode 100644 index 00000000..1315a300 --- /dev/null +++ b/tests/fun_asr_nano/fun_asr_nano_warm_bench.cpp @@ -0,0 +1,9 @@ +#include "../core/audio_task_warm_bench.h" + +int main(int argc, char **argv) { + return engine::tools::run_audio_task_warm_bench( + argc, argv, + {"fun_asr_nano", "models/Fun-ASR-Nano-2512-hf", + engine::runtime::VoiceTaskKind::Asr, + engine::tools::AudioTaskOutputKind::Asr}); +} diff --git a/tests/fun_asr_nano/reference_adaptor.py b/tests/fun_asr_nano/reference_adaptor.py new file mode 100644 index 00000000..8293b93e --- /dev/null +++ b/tests/fun_asr_nano/reference_adaptor.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Generate deterministic Fun-ASR-Nano projector/adaptor checkpoints.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +TRANSFORMERS_COMMIT = "f9966442ac24fff57060774ce22e1884760f4a3b" +MODEL_REVISION = "854d88f94205cd17d2afdb24332130d86fbe654a" +MODEL_CONFIG_SHA256 = "c7c4a30316929631ac5fabc5fb3c0dd3278dcc9809670720c5920186285d004a" +MODEL_SAFETENSORS_SHA256 = ( + "335ca3e74917f1156690400e2c344350112950165789cf78ce3d0a367affd821" +) +TORCH_VERSION = "2.11.0" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--transformers-src", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + args = parse_args() + transformers_src = args.transformers_src.resolve() + model_dir = args.model_dir.resolve() + sys.path.insert(0, str(transformers_src)) + + import torch + import transformers + from safetensors import safe_open + from transformers import FunAsrNanoConfig + from transformers.models.fun_asr_nano import modeling_fun_asr_nano as modeling + + imported_module = Path(transformers.__file__).resolve() + if transformers_src not in imported_module.parents: + raise RuntimeError( + f"expected Transformers from {transformers_src}, imported {imported_module}" + ) + transformers_root = transformers_src.parent + actual_commit = subprocess.check_output( + ["git", "-C", str(transformers_root), "rev-parse", "HEAD"], text=True + ).strip() + if actual_commit != TRANSFORMERS_COMMIT: + raise RuntimeError( + f"expected Transformers {TRANSFORMERS_COMMIT}, found {actual_commit}" + ) + modeling_path = ( + transformers_src / "transformers/models/fun_asr_nano/modeling_fun_asr_nano.py" + ) + modeling_relative = modeling_path.relative_to(transformers_root) + modeling_status = subprocess.check_output( + [ + "git", + "-C", + str(transformers_root), + "status", + "--porcelain", + "--", + str(modeling_relative), + ], + text=True, + ).strip() + if modeling_status: + raise RuntimeError( + f"Fun-ASR-Nano modeling source has uncommitted changes: {modeling_status}" + ) + if torch.__version__.split("+", maxsplit=1)[0] != TORCH_VERSION: + raise RuntimeError(f"expected torch {TORCH_VERSION}, found {torch.__version__}") + + model_path = model_dir / "model.safetensors" + config_path = model_dir / "config.json" + actual_config_sha256 = sha256_file(config_path) + actual_model_sha256 = sha256_file(model_path) + if actual_config_sha256 != MODEL_CONFIG_SHA256: + raise RuntimeError( + f"expected config SHA-256 {MODEL_CONFIG_SHA256}, found {actual_config_sha256}" + ) + if actual_model_sha256 != MODEL_SAFETENSORS_SHA256: + raise RuntimeError( + f"expected model SHA-256 {MODEL_SAFETENSORS_SHA256}, found {actual_model_sha256}" + ) + + config = FunAsrNanoConfig.from_pretrained(model_dir, local_files_only=True) + config.encoder_config._attn_implementation = "eager" + projector = modeling.FunAsrNanoMultiModalProjector(config).eval() + adaptor = modeling.FunAsrNanoAdaptor(config).eval() + projector_state = {} + adaptor_state = {} + catalog_lines = [] + with safe_open(model_path, framework="pt", device="cpu") as source: + for name in source.keys(): + if not name.startswith("model.multi_modal_projector."): + continue + tensor_slice = source.get_slice(name) + catalog_lines.append( + json.dumps( + { + "name": name, + "dtype": tensor_slice.get_dtype(), + "shape": list(tensor_slice.get_shape()), + }, + sort_keys=True, + separators=(",", ":"), + ) + ) + if ".blocks." in name: + target = name.removeprefix("model.multi_modal_projector.") + adaptor_state[target] = source.get_tensor(name).float() + else: + target = name.removeprefix("model.multi_modal_projector.") + projector_state[target] = source.get_tensor(name).float() + if len(projector_state) != 4 or len(adaptor_state) != 32: + raise RuntimeError( + f"expected 4 projector and 32 adaptor tensors, found {len(projector_state)} and {len(adaptor_state)}" + ) + projector.load_state_dict(projector_state, strict=True, assign=True) + adaptor.load_state_dict(adaptor_state, strict=True, assign=True) + + torch.set_num_threads(1) + encoder_embeddings = torch.linspace( + -0.75, 0.75, steps=2 * 4 * config.encoder_config.d_model, dtype=torch.float32 + ).reshape(2, 4, config.encoder_config.d_model) + encoder_embeddings[1, 2:] = torch.linspace( + 10.0, 20.0, steps=2 * config.encoder_config.d_model, dtype=torch.float32 + ).reshape(2, config.encoder_config.d_model) + mask = torch.tensor([[1, 1, 1, 1], [1, 1, 0, 0]], dtype=torch.int64) + + def forward( + values: torch.Tensor, keep_mask: torch.Tensor + ) -> dict[str, torch.Tensor]: + linear_1 = projector.linear_1(values) + linear_2 = projector.linear_2(projector.act(linear_1)) + attention_mask = modeling._prepare_4d_attention_mask(keep_mask, linear_2.dtype) + block_0 = adaptor.blocks[0](linear_2, attention_mask) + block_1 = adaptor.blocks[1](block_0, attention_mask) + return { + "linear_1": linear_1, + "linear_2": linear_2, + "block_0": block_0, + "block_1": block_1, + "packed_valid": block_1[keep_mask.bool()], + } + + with torch.no_grad(): + batched = forward(encoder_embeddings, mask) + valid_mask = mask.bool() + checkpoints = { + name: tensor if name == "packed_valid" else tensor[valid_mask] + for name, tensor in batched.items() + } + individual = [] + for batch_index, valid_frames in enumerate(mask.sum(dim=1).tolist()): + values = encoder_embeddings[batch_index : batch_index + 1, :valid_frames] + keep = torch.ones((1, valid_frames), dtype=torch.int64) + individual.append(forward(values, keep)["packed_valid"]) + torch.testing.assert_close( + checkpoints["packed_valid"], + torch.cat(individual, dim=0), + atol=2.0e-4, + rtol=2.0e-4, + ) + + binary_data = bytearray() + + def store_tensor(tensor: torch.Tensor) -> dict[str, object]: + array = tensor.detach().cpu().contiguous().numpy().astype(" argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--transformers-src", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--device", default="cuda") + return parser.parse_args() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def require_revision(transformers_src: Path) -> None: + actual = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=transformers_src, text=True + ).strip() + if actual != TRANSFORMERS_COMMIT: + raise RuntimeError( + f"expected Transformers {TRANSFORMERS_COMMIT}, found {actual}" + ) + dirty = subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=no"], + cwd=transformers_src, + text=True, + ).strip() + if dirty: + raise RuntimeError("Transformers reference checkout has tracked modifications") + for relative_path, expected_hash in TRANSFORMERS_SOURCE_SHA256.items(): + actual_hash = sha256(transformers_src / relative_path) + if actual_hash != expected_hash: + raise RuntimeError( + f"unexpected Transformers source hash for {relative_path}: " + f"{actual_hash}" + ) + + +def chat_text(prompt: str) -> str: + return ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + f"<|im_start|>user\n{prompt}<|object_ref_start|><|im_end|>\n" + "<|im_start|>assistant\n" + ) + + +def expanded_prompt( + tokenizer, prompt: str, audio_token_id: int +) -> tuple[list[int], list[int]]: + ids = tokenizer.encode(chat_text(prompt), add_special_tokens=False) + if ids.count(audio_token_id) != 1: + raise RuntimeError("expected one audio placeholder in Fun-ASR-Nano prompt") + expanded: list[int] = [] + positions: list[int] = [] + for token_id in ids: + if token_id == audio_token_id: + for _ in range(AUDIO_TOKENS): + positions.append(len(expanded)) + expanded.append(token_id) + else: + expanded.append(token_id) + return expanded, positions + + +def deterministic_audio(torch): + values = [ + ((index % 97) - 48) / 97.0 + math.sin(index * 0.013) * 0.1 + for index in range(AUDIO_TOKENS * 1024) + ] + return torch.tensor(values, dtype=torch.float32).reshape(AUDIO_TOKENS, 1024) + + +def append_f32(blob: array, values) -> dict[str, int]: + flat = values.detach().to(device="cpu", dtype=values.new_zeros(()).float().dtype) + flat = flat.contiguous().view(-1).tolist() + offset = len(blob) + blob.extend(flat) + return {"offset_f32": offset, "count": len(flat)} + + +def main() -> None: + args = parse_args() + require_revision(args.transformers_src) + sys.path.insert(0, str(args.transformers_src / "src")) + + import torch + from safetensors import safe_open + from transformers import AutoProcessor, FunAsrNanoForConditionalGeneration + + torch.manual_seed(0) + torch.set_grad_enabled(False) + torch.set_float32_matmul_precision("highest") + + config_path = args.model_dir / "config.json" + model_path = args.model_dir / "model.safetensors" + if sha256(config_path) != MODEL_CONFIG_SHA256: + raise RuntimeError("unexpected Fun-ASR-Nano config hash") + if sha256(model_path) != MODEL_SAFETENSORS_SHA256: + raise RuntimeError("unexpected Fun-ASR-Nano model hash") + + processor = AutoProcessor.from_pretrained(args.model_dir, local_files_only=True) + model = FunAsrNanoForConditionalGeneration.from_pretrained( + args.model_dir, + local_files_only=True, + torch_dtype=torch.float32, + attn_implementation="eager", + ).to(args.device) + model.eval() + + audio_token_id = model.config.audio_token_id + itn_ids, itn_positions = expanded_prompt( + processor.tokenizer, ITN_PROMPT, audio_token_id + ) + no_itn_ids, no_itn_positions = expanded_prompt( + processor.tokenizer, NO_ITN_PROMPT, audio_token_id + ) + if len(itn_positions) != AUDIO_TOKENS or len(no_itn_positions) != AUDIO_TOKENS: + raise RuntimeError("expanded audio token count mismatch") + + input_ids = torch.tensor([itn_ids], dtype=torch.long, device=args.device) + attention_mask = torch.ones_like(input_ids) + audio = deterministic_audio(torch).to(args.device) + inputs_embeds = model.get_input_embeddings()(input_ids) + inputs_embeds[0, itn_positions] = audio + + outputs = model( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + use_cache=True, + logits_to_keep=1, + return_dict=True, + ) + logits = [outputs.logits[0, -1].float()] + greedy_tokens = [int(torch.argmax(logits[0]).item())] + past_key_values = outputs.past_key_values + + for _ in range(2): + step_id = torch.tensor( + [[greedy_tokens[-1]]], dtype=torch.long, device=args.device + ) + attention_mask = torch.ones( + (1, attention_mask.shape[1] + 1), dtype=torch.long, device=args.device + ) + outputs = model( + input_ids=step_id, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + logits_to_keep=1, + return_dict=True, + ) + past_key_values = outputs.past_key_values + logits.append(outputs.logits[0, -1].float()) + greedy_tokens.append(int(torch.argmax(logits[-1]).item())) + + with safe_open(model_path, framework="pt", device="cpu") as handle: + text_names = sorted( + name for name in handle.keys() if name.startswith("model.language_model.") + ) + catalog_hash = hashlib.sha256("\n".join(text_names).encode()).hexdigest() + + blob = array("f") + audio_descriptor = append_f32(blob, audio) + audio_descriptor["shape"] = [AUDIO_TOKENS, 1024] + logit_descriptors = [] + for values in logits: + descriptor = append_f32(blob, values) + descriptor["shape"] = [151936] + logit_descriptors.append(descriptor) + if sys.byteorder != "little": + blob.byteswap() + + args.output.parent.mkdir(parents=True, exist_ok=True) + data_path = args.output.with_suffix(".bin") + with data_path.open("wb") as handle: + blob.tofile(handle) + + metadata = { + "schema_version": 1, + "transformers_commit": TRANSFORMERS_COMMIT, + "transformers_source_sha256": TRANSFORMERS_SOURCE_SHA256, + "model_revision": MODEL_REVISION, + "model_config_sha256": MODEL_CONFIG_SHA256, + "model_safetensors_sha256": MODEL_SAFETENSORS_SHA256, + "tokenizer_json_sha256": sha256(args.model_dir / "tokenizer.json"), + "tokenizer_config_sha256": sha256(args.model_dir / "tokenizer_config.json"), + "text_weight_catalog_count": len(text_names), + "text_weight_catalog_sha256": catalog_hash, + "torch_version": torch.__version__, + "device": str(args.device), + "data_file": data_path.name, + "data_format": "little-endian-float32", + "audio_token_id": audio_token_id, + "im_start_token_id": processor.tokenizer.convert_tokens_to_ids("<|im_start|>"), + "im_end_token_id": processor.tokenizer.convert_tokens_to_ids("<|im_end|>"), + "eos_token_id": model.config.text_config.eos_token_id, + "itn": { + "prompt": ITN_PROMPT, + "input_ids": itn_ids, + "audio_token_positions": itn_positions, + }, + "no_itn": { + "prompt": NO_ITN_PROMPT, + "input_ids": no_itn_ids, + "audio_token_positions": no_itn_positions, + }, + "audio_embeddings": audio_descriptor, + "greedy_token_ids": greedy_tokens, + "generated_text": processor.tokenizer.decode( + greedy_tokens, skip_special_tokens=True + ), + "logits": logit_descriptors, + "data_sha256": sha256(data_path), + } + args.output.write_text(json.dumps(metadata, indent=2, ensure_ascii=False) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tests/fun_asr_nano/reference_encoder.py b/tests/fun_asr_nano/reference_encoder.py new file mode 100644 index 00000000..5f2a05d7 --- /dev/null +++ b/tests/fun_asr_nano/reference_encoder.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Generate deterministic full-encoder checkpoints from Transformers #46180.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import numpy as np + + +TRANSFORMERS_COMMIT = "48e7f65fb274172e15aa88875d780c67c37606c7" +MODEL_REVISION = "854d88f94205cd17d2afdb24332130d86fbe654a" +MODEL_CONFIG_SHA256 = "c7c4a30316929631ac5fabc5fb3c0dd3278dcc9809670720c5920186285d004a" +MODEL_SAFETENSORS_SHA256 = "335ca3e74917f1156690400e2c344350112950165789cf78ce3d0a367affd821" +TORCH_VERSION = "2.11.0" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--transformers-src", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + args = parse_args() + transformers_src = args.transformers_src.resolve() + model_dir = args.model_dir.resolve() + sys.path.insert(0, str(transformers_src)) + + import torch + import transformers + from safetensors import safe_open + from transformers import FunAsrNanoConfig + from transformers.models.fun_asr_nano.modeling_fun_asr_nano import ( + FunAsrNanoEncoder, + ) + + imported_module = Path(transformers.__file__).resolve() + if transformers_src not in imported_module.parents: + raise RuntimeError( + f"expected Transformers from {transformers_src}, imported {imported_module}" + ) + transformers_root = transformers_src.parent + actual_commit = subprocess.check_output( + ["git", "-C", str(transformers_root), "rev-parse", "HEAD"], text=True + ).strip() + if actual_commit != TRANSFORMERS_COMMIT: + raise RuntimeError( + f"expected Transformers {TRANSFORMERS_COMMIT}, found {actual_commit}" + ) + modeling_path = ( + transformers_src + / "transformers/models/fun_asr_nano/modeling_fun_asr_nano.py" + ) + modeling_relative = modeling_path.relative_to(transformers_root) + modeling_status = subprocess.check_output( + [ + "git", + "-C", + str(transformers_root), + "status", + "--porcelain", + "--", + str(modeling_relative), + ], + text=True, + ).strip() + if modeling_status: + raise RuntimeError( + f"Fun-ASR-Nano modeling source has uncommitted changes: {modeling_status}" + ) + if torch.__version__.split("+", maxsplit=1)[0] != TORCH_VERSION: + raise RuntimeError(f"expected torch {TORCH_VERSION}, found {torch.__version__}") + + model_path = model_dir / "model.safetensors" + config_path = model_dir / "config.json" + if not model_path.is_file() or not config_path.is_file(): + raise RuntimeError("model directory is missing config.json or model.safetensors") + actual_config_sha256 = sha256_file(config_path) + actual_model_sha256 = sha256_file(model_path) + if actual_config_sha256 != MODEL_CONFIG_SHA256: + raise RuntimeError( + f"expected config SHA-256 {MODEL_CONFIG_SHA256}, found {actual_config_sha256}" + ) + if actual_model_sha256 != MODEL_SAFETENSORS_SHA256: + raise RuntimeError( + f"expected model SHA-256 {MODEL_SAFETENSORS_SHA256}, found {actual_model_sha256}" + ) + + config = FunAsrNanoConfig.from_pretrained(model_dir, local_files_only=True) + encoder = FunAsrNanoEncoder(config.encoder_config).eval() + state_dict = {} + catalog_lines = [] + with safe_open(model_path, framework="pt", device="cpu") as source: + for name in source.keys(): + if not name.startswith("model.audio_tower."): + continue + tensor_slice = source.get_slice(name) + shape = list(tensor_slice.get_shape()) + catalog_lines.append( + json.dumps( + {"name": name, "dtype": tensor_slice.get_dtype(), "shape": shape}, + sort_keys=True, + separators=(",", ":"), + ) + ) + target_name = name.removeprefix("model.audio_tower.").replace( + ".fsmn.", ".feedforward_sequential_memory." + ) + state_dict[target_name] = source.get_tensor(name).float() + if len(state_dict) != 1194: + raise RuntimeError( + f"expected 1194 audio-tower tensors, found {len(state_dict)}" + ) + encoder.load_state_dict(state_dict, strict=True, assign=True) + + torch.set_num_threads(1) + input_features = torch.linspace( + -1.0, 1.0, steps=3 * config.encoder_config.input_size, dtype=torch.float32 + ).reshape(1, 3, config.encoder_config.input_size) + input_mask = torch.ones((1, 3), dtype=torch.int64) + checkpoints = {} + with torch.no_grad(): + hidden = encoder.stem(input_features, input_mask) + checkpoints["stem"] = hidden + for index, layer in enumerate(encoder.layers): + hidden = layer(hidden, input_mask) + if index in (0, 24, 48): + checkpoints[f"main_layer_{index}"] = hidden + hidden = encoder.layer_norm(hidden) + checkpoints["main_layer_norm"] = hidden + for index, layer in enumerate(encoder.timestamp_prediction_layers): + hidden = layer(hidden, input_mask) + if index in (0, 10, 19): + checkpoints[f"timestamp_layer_{index}"] = hidden + hidden = encoder.timestamp_prediction_layer_norm(hidden) + checkpoints["final"] = hidden + actual = encoder(input_features, input_features_mask=input_mask).last_hidden_state + torch.testing.assert_close(actual, hidden, atol=0.0, rtol=0.0) + + binary_data = bytearray() + + def store_tensor(tensor) -> dict[str, object]: + array = tensor.detach().cpu().contiguous().numpy().astype(" argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--transformers-src", type=Path, required=True) + parser.add_argument("--sample-wav", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def load_sample(path: Path) -> np.ndarray: + audio, sample_rate = sf.read(path, dtype="float32", always_2d=True) + if sample_rate != SAMPLE_RATE: + raise ValueError(f"expected {SAMPLE_RATE} Hz sample, got {sample_rate}") + return audio.mean(axis=1, dtype=np.float32) + + +def synthetic_inputs(sample_wav: Path) -> dict[str, np.ndarray]: + samples = np.arange(SAMPLE_RATE, dtype=np.float32) + impulse = np.zeros(SAMPLE_RATE, dtype=np.float32) + impulse[SAMPLE_RATE // 2] = 1.0 + return { + "silence": np.zeros(SAMPLE_RATE, dtype=np.float32), + "impulse": impulse, + "sine_440hz": (0.5 * np.sin(2.0 * math.pi * 440.0 * samples / SAMPLE_RATE)).astype(np.float32), + "sample_16k": load_sample(sample_wav), + } + + +def summarize(features: np.ndarray, waveform: np.ndarray) -> dict[str, object]: + flat = features.reshape(-1) + values = flat.astype(np.float32) + return { + "samples": len(waveform), + "frames": int(features.shape[0]), + "width": int(features.shape[1]), + "min": float(np.min(flat)), + "max": float(np.max(flat)), + "mean": float(np.mean(flat, dtype=np.float64)), + "l2": float(np.linalg.norm(flat.astype(np.float64))), + "first32": values[:32].tolist(), + "last32": values[-32:].tolist(), + } + + +def main() -> None: + args = parse_args() + sys.path.insert(0, str(args.transformers_src.resolve())) + + import torch + import torchaudio + import transformers + from transformers import FunAsrNanoFeatureExtractor + + transformers_module = Path(transformers.__file__).resolve() + transformers_src = args.transformers_src.resolve() + if transformers_src not in transformers_module.parents: + raise RuntimeError( + f"expected Transformers from {transformers_src}, imported {transformers_module}" + ) + + transformers_root = transformers_src.parent + actual_commit = subprocess.check_output( + ["git", "-C", str(transformers_root), "rev-parse", "HEAD"], text=True + ).strip() + if actual_commit != TRANSFORMERS_COMMIT: + raise RuntimeError(f"expected Transformers {TRANSFORMERS_COMMIT}, found {actual_commit}") + feature_extractor_path = ( + transformers_src / "transformers/models/fun_asr_nano/feature_extraction_fun_asr_nano.py" + ) + feature_extractor_relative = feature_extractor_path.relative_to(transformers_root) + feature_status = subprocess.check_output( + ["git", "-C", str(transformers_root), "status", "--porcelain", "--", str(feature_extractor_relative)], + text=True, + ).strip() + if feature_status: + raise RuntimeError(f"Fun-ASR-Nano feature extractor has uncommitted changes: {feature_status}") + actual_torchaudio_version = torchaudio.__version__ + if actual_torchaudio_version.split("+", maxsplit=1)[0] != TORCHAUDIO_VERSION: + raise RuntimeError( + f"expected torchaudio {TORCHAUDIO_VERSION}, found {actual_torchaudio_version}" + ) + + torch.set_num_threads(1) + extractor = FunAsrNanoFeatureExtractor() + fixtures: dict[str, object] = {} + binary_data = bytearray() + + def store_f32(values: np.ndarray) -> dict[str, int]: + array = np.asarray(values, dtype=" argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--transformers-src", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def projection_block(module, hidden_states): + """Run the stem after the encoder-wide scale and positional embedding steps.""" + normalized = module.self_attn_layer_norm(hidden_states) + value_states = module.self_attn.v_proj(normalized) + attention_output, _ = module.self_attn(normalized) + fsmn_output = module.feedforward_sequential_memory(value_states) + hidden_states = attention_output + fsmn_output + + residual = hidden_states + hidden_states = module.final_layer_norm(hidden_states) + hidden_states = module.fc1(hidden_states) + hidden_states = module.activation_fn(hidden_states) + hidden_states = module.fc2(hidden_states) + return residual + hidden_states + + +def main() -> None: + args = parse_args() + transformers_src = args.transformers_src.resolve() + sys.path.insert(0, str(transformers_src)) + + import torch + import transformers + from transformers import FunAsrNanoEncoderConfig + from transformers.models.fun_asr_nano.modeling_fun_asr_nano import ( + FunAsrNanoEncoderLayer, + FunAsrNanoEncoderStem, + ) + + transformers_module = Path(transformers.__file__).resolve() + if transformers_src not in transformers_module.parents: + raise RuntimeError( + f"expected Transformers from {transformers_src}, imported {transformers_module}" + ) + transformers_root = transformers_src.parent + actual_commit = subprocess.check_output( + ["git", "-C", str(transformers_root), "rev-parse", "HEAD"], text=True + ).strip() + if actual_commit != TRANSFORMERS_COMMIT: + raise RuntimeError(f"expected Transformers {TRANSFORMERS_COMMIT}, found {actual_commit}") + modeling_path = transformers_src / "transformers/models/fun_asr_nano/modeling_fun_asr_nano.py" + modeling_relative = modeling_path.relative_to(transformers_root) + modeling_status = subprocess.check_output( + ["git", "-C", str(transformers_root), "status", "--porcelain", "--", str(modeling_relative)], + text=True, + ).strip() + if modeling_status: + raise RuntimeError(f"Fun-ASR-Nano modeling source has uncommitted changes: {modeling_status}") + if torch.__version__.split("+", maxsplit=1)[0] != TORCH_VERSION: + raise RuntimeError(f"expected torch {TORCH_VERSION}, found {torch.__version__}") + + torch.set_num_threads(1) + torch.manual_seed(20260729) + config = FunAsrNanoEncoderConfig( + num_mel_bins=12, + num_stacked_frames=1, + d_model=8, + encoder_attention_heads=2, + encoder_ffn_dim=16, + encoder_layers=2, + dropout=0.0, + attention_dropout=0.0, + activation_dropout=0.0, + activation_function="relu", + max_position_embeddings=32, + kernel_size=3, + ) + projection = FunAsrNanoEncoderStem(config).eval() + residual = FunAsrNanoEncoderLayer(config).eval() + + projection_input = torch.linspace(-1.1, 1.3, steps=5 * 12, dtype=torch.float32).reshape(1, 5, 12) + residual_input = torch.linspace(0.9, -0.7, steps=5 * 8, dtype=torch.float32).reshape(1, 5, 8) + with torch.no_grad(): + projection_norm = projection.self_attn_layer_norm(projection_input) + projection_output = projection_block(projection, projection_input) + positions = projection.position_embeddings(6)[1:].to(dtype=projection_input.dtype) + raw_stem_input = (projection_input - positions.unsqueeze(0)) / math.sqrt(config.d_model) + actual_stem_output = projection(raw_stem_input) + torch.testing.assert_close(actual_stem_output, projection_output, atol=2e-6, rtol=2e-6) + residual_norm = residual.self_attn_layer_norm(residual_input) + residual_output = residual(residual_input) + + binary_data = bytearray() + + def store_tensor(tensor) -> dict[str, object]: + array = tensor.detach().cpu().contiguous().numpy().astype(" dict[str, object]: + return { + "input": store_tensor(input_tensor), + "output": store_tensor(output_tensor), + "checkpoints": { + name: store_tensor(tensor) + for name, tensor in checkpoints.items() + }, + "weights": { + name: store_tensor(tensor) + for name, tensor in sorted(module.state_dict().items()) + }, + } + + blocks = { + "projection": store_block( + projection, + projection_input, + projection_output, + {"self_attn_layer_norm": projection_norm}, + ), + "residual": store_block( + residual, + residual_input, + residual_output, + {"self_attn_layer_norm": residual_norm}, + ), + } + binary_path = args.output.with_suffix(".bin") + binary_bytes = bytes(binary_data) + payload = { + "schema_version": 1, + "reference": "transformers.FunAsrNanoEncoderStem/FunAsrNanoEncoderLayer", + "transformers_commit": TRANSFORMERS_COMMIT, + "modeling_sha256": hashlib.sha256(modeling_path.read_bytes()).hexdigest(), + "torch_version": torch.__version__, + "data_file": binary_path.name, + "data_format": "little-endian-float32", + "data_sha256": hashlib.sha256(binary_bytes).hexdigest(), + "config": { + "input_size": 12, + "model_size": 8, + "num_heads": 2, + "ffn_size": 16, + "fsmn_kernel_size": 3, + "layer_norm_eps": 1e-5, + "frames": 5, + }, + "blocks": blocks, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + binary_path.write_bytes(binary_bytes) + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + print(json.dumps(payload, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/fun_asr_nano/sanm_probe.cpp b/tests/fun_asr_nano/sanm_probe.cpp new file mode 100644 index 00000000..2af30fe3 --- /dev/null +++ b/tests/fun_asr_nano/sanm_probe.cpp @@ -0,0 +1,428 @@ +#include "engine/framework/core/backend.h" +#include "engine/framework/io/json.h" +#include "engine/framework/modules/speech_encoders/sanm.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t kGraphBytes = 64 * 1024 * 1024; +constexpr size_t kGraphNodes = 4096; +constexpr float kCpuAbsoluteTolerance = 2.0e-4F; +constexpr float kCudaAbsoluteTolerance = 1.0e-3F; +constexpr float kRelativeTolerance = 2.0e-4F; +constexpr const char *kTransformersCommit = + "48e7f65fb274172e15aa88875d780c67c37606c7"; + +struct Arguments { + engine::core::BackendType backend = engine::core::BackendType::Cpu; + std::filesystem::path reference; + std::filesystem::path data; +}; + +Arguments parse_arguments(int argc, char **argv) { + Arguments arguments; + for (int index = 1; index < argc; ++index) { + const std::string option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + option); + } + const std::string value(argv[++index]); + if (option == "--backend") { + if (value == "cpu") { + arguments.backend = engine::core::BackendType::Cpu; + } else if (value == "cuda") { + arguments.backend = engine::core::BackendType::Cuda; + } else { + throw std::runtime_error("unsupported SAN-M probe backend: " + value); + } + } else if (option == "--reference") { + arguments.reference = value; + } else if (option == "--data") { + arguments.data = value; + } else { + throw std::runtime_error("unknown SAN-M probe option: " + option); + } + } + if (arguments.reference.empty() || arguments.data.empty()) { + throw std::runtime_error("--reference and --data are required"); + } + return arguments; +} + +std::vector load_f32_slice(const std::filesystem::path &data_path, + const engine::io::json::Value &descriptor) { + const int64_t signed_offset = descriptor.require("offset_f32").as_i64(); + const int64_t signed_count = descriptor.require("count").as_i64(); + if (signed_offset < 0 || signed_count < 0 || + static_cast(signed_count) > + std::numeric_limits::max()) { + throw std::runtime_error("invalid SAN-M reference data slice"); + } + const uint64_t offset = static_cast(signed_offset); + const size_t count = static_cast(signed_count); + if (offset > + static_cast(std::numeric_limits::max()) / 4) { + throw std::runtime_error("SAN-M reference data offset is too large"); + } + std::ifstream input(data_path, std::ios::binary); + input.seekg(static_cast(offset * 4)); + if (!input) { + throw std::runtime_error("could not seek in SAN-M reference data"); + } + std::vector values(count); + for (float &value : values) { + uint8_t bytes[4]{}; + input.read(reinterpret_cast(bytes), 4); + if (!input) { + throw std::runtime_error("truncated SAN-M reference data"); + } + const uint32_t bits = static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); + static_assert(sizeof(value) == sizeof(bits)); + std::memcpy(&value, &bits, sizeof(value)); + } + return values; +} + +engine::core::TensorShape +descriptor_shape(const engine::io::json::Value &descriptor) { + const auto dimensions = + engine::io::json::require_i64_array(descriptor, "shape"); + if (dimensions.empty() || dimensions.size() > engine::core::kMaxTensorRank) { + throw std::runtime_error("SAN-M reference tensor rank is invalid"); + } + engine::core::TensorShape shape; + shape.rank = dimensions.size(); + for (size_t axis = 0; axis < dimensions.size(); ++axis) { + if (dimensions[axis] <= 0) { + throw std::runtime_error( + "SAN-M reference tensor dimension must be positive"); + } + shape.dims[axis] = dimensions[axis]; + } + return shape; +} + +class GraphRunner { +public: + explicit GraphRunner(engine::core::BackendType backend_type) + : backend_(engine::core::init_backend({backend_type, 0, 4})) { + if (backend_ == nullptr) { + throw std::runtime_error("failed to initialize SAN-M probe backend"); + } + engine::core::set_backend_threads(backend_, 4); + ggml_init_params parameters{}; + parameters.mem_size = kGraphBytes; + parameters.mem_buffer = nullptr; + parameters.no_alloc = true; + ggml_ = ggml_init(parameters); + if (ggml_ == nullptr) { + throw std::runtime_error( + "failed to initialize SAN-M probe graph context"); + } + context_.ggml = ggml_; + context_.module_instance_name = "fun_asr_nano_sanm_probe"; + context_.backend_type = backend_type; + } + + ~GraphRunner() { + if (buffer_ != nullptr) { + ggml_backend_buffer_free(buffer_); + } + if (ggml_ != nullptr) { + ggml_free(ggml_); + } + if (backend_ != nullptr) { + ggml_backend_free(backend_); + } + } + + GraphRunner(const GraphRunner &) = delete; + GraphRunner &operator=(const GraphRunner &) = delete; + + engine::core::ModuleBuildContext &context() { return context_; } + + engine::core::TensorValue make_f32(const engine::core::TensorShape &shape, + std::vector values) { + if (static_cast(values.size()) != shape.num_elements()) { + throw std::runtime_error("SAN-M tensor data does not match its shape"); + } + auto tensor = engine::core::make_tensor(context_, GGML_TYPE_F32, shape); + pending_.push_back({tensor, std::move(values)}); + return tensor; + } + + std::vector run(const engine::core::TensorValue &output) { + buffer_ = ggml_backend_alloc_ctx_tensors(ggml_, backend_); + if (buffer_ == nullptr) { + throw std::runtime_error("failed to allocate SAN-M probe tensors"); + } + for (const auto &pending : pending_) { + engine::core::write_tensor_f32(pending.tensor, pending.values); + } + ggml_cgraph *graph = ggml_new_graph_custom(ggml_, kGraphNodes, false); + ggml_build_forward_expand(graph, output.tensor); + engine::core::validate_backend_graph_supported(backend_, graph, + "SAN-M probe"); + const auto status = engine::core::compute_backend_graph( + backend_, graph, nullptr, "SAN-M probe"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("SAN-M probe graph execution failed"); + } + return engine::core::read_tensor_f32(output.tensor); + } + +private: + struct PendingTensor { + engine::core::TensorValue tensor; + std::vector values; + }; + + ggml_backend_t backend_ = nullptr; + ggml_backend_buffer_t buffer_ = nullptr; + ggml_context *ggml_ = nullptr; + engine::core::ModuleBuildContext context_{}; + std::vector pending_; +}; + +class TensorLoader { +public: + TensorLoader(GraphRunner &runner, std::filesystem::path data_path) + : runner_(runner), data_path_(std::move(data_path)) {} + + engine::core::TensorValue load(const engine::io::json::Value &descriptor) { + return runner_.make_f32(descriptor_shape(descriptor), + load_f32_slice(data_path_, descriptor)); + } + + engine::core::TensorValue load_named(const engine::io::json::Value &weights, + const std::string &name) { + return load(weights.require(name)); + } + + engine::modules::LinearWeights + load_linear(const engine::io::json::Value &weights, + const std::string &prefix) { + return { + load_named(weights, prefix + ".weight"), + load_named(weights, prefix + ".bias"), + }; + } + + engine::modules::NormWeights load_norm(const engine::io::json::Value &weights, + const std::string &prefix) { + return { + load_named(weights, prefix + ".weight"), + load_named(weights, prefix + ".bias"), + }; + } + +private: + GraphRunner &runner_; + std::filesystem::path data_path_; +}; + +engine::modules::SanmBlockWeightsView +load_weights(TensorLoader &loader, const engine::io::json::Value &weights) { + return { + loader.load_norm(weights, "self_attn_layer_norm"), + loader.load_linear(weights, "self_attn.q_proj"), + loader.load_linear(weights, "self_attn.k_proj"), + loader.load_linear(weights, "self_attn.v_proj"), + loader.load_linear(weights, "self_attn.out_proj"), + loader.load_named(weights, "feedforward_sequential_memory.conv.weight"), + loader.load_norm(weights, "final_layer_norm"), + loader.load_linear(weights, "fc1"), + loader.load_linear(weights, "fc2"), + }; +} + +void require_close(const std::string &label, const std::vector &actual, + const std::vector &expected, + engine::core::BackendType backend_type); + +void run_layer_norm(const std::string &name, + const engine::io::json::Value &block, + const engine::io::json::Value &config_json, + const std::filesystem::path &data_path, + engine::core::BackendType backend_type) { + GraphRunner runner(backend_type); + TensorLoader loader(runner, data_path); + const auto input = loader.load(block.require("input")); + const auto weights = + loader.load_norm(block.require("weights"), "self_attn_layer_norm"); + const auto output = engine::modules::sanm_layer_norm( + runner.context(), input, weights, + engine::io::json::require_f32(config_json, "layer_norm_eps")); + const auto actual = runner.run(output); + const auto expected = load_f32_slice( + data_path, block.require("checkpoints").require("self_attn_layer_norm")); + require_close(name + ".self_attn_layer_norm", actual, expected, backend_type); +} + +void require_close(const std::string &label, const std::vector &actual, + const std::vector &expected, + engine::core::BackendType backend_type) { + if (actual.size() != expected.size()) { + throw std::runtime_error(label + " output size mismatch"); + } + float maximum_absolute_error = 0.0F; + float maximum_relative_error = 0.0F; + size_t worst_index = 0; + for (size_t index = 0; index < actual.size(); ++index) { + const float absolute_error = std::abs(actual[index] - expected[index]); + const float relative_error = + absolute_error / std::max(std::abs(expected[index]), 1.0e-12F); + if (absolute_error > maximum_absolute_error) { + maximum_absolute_error = absolute_error; + worst_index = index; + } + maximum_relative_error = std::max(maximum_relative_error, relative_error); + const float absolute_tolerance = + backend_type == engine::core::BackendType::Cuda ? kCudaAbsoluteTolerance + : kCpuAbsoluteTolerance; + const float limit = + absolute_tolerance + kRelativeTolerance * std::abs(expected[index]); + if (absolute_error > limit) { + throw std::runtime_error( + label + " mismatch at " + std::to_string(index) + ": expected " + + std::to_string(expected[index]) + ", got " + + std::to_string(actual[index]) + ", absolute error " + + std::to_string(absolute_error) + ", limit " + std::to_string(limit)); + } + } + std::cout << label << ": count=" << actual.size() + << ", max_abs=" << maximum_absolute_error + << ", max_rel=" << maximum_relative_error + << ", worst_index=" << worst_index << '\n'; +} + +void run_block(const std::string &name, const engine::io::json::Value &block, + const engine::io::json::Value &config_json, + const std::filesystem::path &data_path, + engine::core::BackendType backend_type) { + GraphRunner runner(backend_type); + TensorLoader loader(runner, data_path); + const auto input = loader.load(block.require("input")); + const auto weights = load_weights(loader, block.require("weights")); + engine::modules::SanmBlockConfig config; + config.input_size = + name == "projection" + ? engine::io::json::require_i64(config_json, "input_size") + : engine::io::json::require_i64(config_json, "model_size"); + config.model_size = engine::io::json::require_i64(config_json, "model_size"); + config.num_heads = engine::io::json::require_i64(config_json, "num_heads"); + config.ffn_size = engine::io::json::require_i64(config_json, "ffn_size"); + config.fsmn_kernel_size = + engine::io::json::require_i64(config_json, "fsmn_kernel_size"); + config.layer_norm_eps = + engine::io::json::require_f32(config_json, "layer_norm_eps"); + config.attention_lowering = + engine::modules::ScaledDotProductAttentionLowering::Explicit; + + const auto output = name == "projection" + ? engine::modules::sanm_projection_block( + runner.context(), input, weights, config) + : engine::modules::sanm_residual_block( + runner.context(), input, weights, config); + const auto actual = runner.run(output); + const auto expected = load_f32_slice(data_path, block.require("output")); + require_close(name, actual, expected, backend_type); +} + +void require_runtime_error(const std::string &label, + const std::string &expected_message, + const std::function &operation) { + try { + operation(); + } catch (const std::runtime_error &error) { + if (std::string(error.what()).find(expected_message) != std::string::npos) { + return; + } + throw std::runtime_error(label + + " returned an unexpected error: " + error.what()); + } + throw std::runtime_error(label + " did not reject the invalid configuration"); +} + +void run_config_contracts() { + GraphRunner runner(engine::core::BackendType::Cpu); + const auto input8 = + runner.make_f32(engine::core::TensorShape::from_dims({1, 2, 8}), + std::vector(16, 0.0F)); + const auto input7 = + runner.make_f32(engine::core::TensorShape::from_dims({1, 2, 7}), + std::vector(14, 0.0F)); + const engine::modules::SanmBlockWeightsView weights; + engine::modules::SanmBlockConfig config{ + 8, 8, 2, 16, 3, + }; + + auto bad_heads = config; + bad_heads.num_heads = 3; + require_runtime_error("SAN-M head divisibility", "divisible", [&] { + static_cast(engine::modules::sanm_projection_block( + runner.context(), input8, weights, bad_heads)); + }); + + auto even_kernel = config; + even_kernel.fsmn_kernel_size = 4; + require_runtime_error("SAN-M FSMN kernel", "odd", [&] { + static_cast(engine::modules::sanm_projection_block( + runner.context(), input8, weights, even_kernel)); + }); + + auto mismatched_residual = config; + mismatched_residual.input_size = 7; + require_runtime_error( + "SAN-M residual dimensions", "input_size == model_size", [&] { + static_cast(engine::modules::sanm_residual_block( + runner.context(), input7, weights, mismatched_residual)); + }); +} + +} // namespace + +int main(int argc, char **argv) try { + const auto arguments = parse_arguments(argc, argv); + const auto reference = engine::io::json::parse_file(arguments.reference); + if (reference.require("schema_version").as_i64() != 1 || + reference.require("transformers_commit").as_string() != + kTransformersCommit || + reference.require("data_format").as_string() != "little-endian-float32") { + throw std::runtime_error("unexpected SAN-M reference metadata"); + } + run_config_contracts(); + const auto &blocks = reference.require("blocks"); + const auto &config = reference.require("config"); + run_layer_norm("projection", blocks.require("projection"), config, + arguments.data, arguments.backend); + run_layer_norm("residual", blocks.require("residual"), config, arguments.data, + arguments.backend); + run_block("projection", blocks.require("projection"), config, arguments.data, + arguments.backend); + run_block("residual", blocks.require("residual"), config, arguments.data, + arguments.backend); + return 0; +} catch (const std::exception &error) { + std::cerr << "fun_asr_nano_sanm_probe failed: " << error.what() << '\n'; + return 1; +} diff --git a/tests/fun_asr_nano/sanm_reference.bin b/tests/fun_asr_nano/sanm_reference.bin new file mode 100644 index 00000000..2066e040 Binary files /dev/null and b/tests/fun_asr_nano/sanm_reference.bin differ diff --git a/tests/fun_asr_nano/sanm_reference.json b/tests/fun_asr_nano/sanm_reference.json new file mode 100644 index 00000000..9b9c7c29 --- /dev/null +++ b/tests/fun_asr_nano/sanm_reference.json @@ -0,0 +1,341 @@ +{ + "schema_version": 1, + "reference": "transformers.FunAsrNanoEncoderStem/FunAsrNanoEncoderLayer", + "transformers_commit": "48e7f65fb274172e15aa88875d780c67c37606c7", + "modeling_sha256": "70f8230fd5a75e8a119cea1c8dc919c3f51d8aa7c5d8f1c23c5402b856ebebfb", + "torch_version": "2.11.0+cu130", + "data_file": "sanm_reference.bin", + "data_format": "little-endian-float32", + "data_sha256": "2149540b2a724efa8e3242933071163ae97fab5d85331a35b119330de090255e", + "config": { + "input_size": 12, + "model_size": 8, + "num_heads": 2, + "ffn_size": 16, + "fsmn_kernel_size": 3, + "layer_norm_eps": 1e-05, + "frames": 5 + }, + "blocks": { + "projection": { + "input": { + "shape": [ + 1, + 5, + 12 + ], + "offset_f32": 0, + "count": 60 + }, + "output": { + "shape": [ + 1, + 5, + 8 + ], + "offset_f32": 60, + "count": 40 + }, + "checkpoints": { + "self_attn_layer_norm": { + "shape": [ + 1, + 5, + 12 + ], + "offset_f32": 100, + "count": 60 + } + }, + "weights": { + "fc1.bias": { + "shape": [ + 16 + ], + "offset_f32": 160, + "count": 16 + }, + "fc1.weight": { + "shape": [ + 16, + 8 + ], + "offset_f32": 176, + "count": 128 + }, + "fc2.bias": { + "shape": [ + 8 + ], + "offset_f32": 304, + "count": 8 + }, + "fc2.weight": { + "shape": [ + 8, + 16 + ], + "offset_f32": 312, + "count": 128 + }, + "feedforward_sequential_memory.conv.weight": { + "shape": [ + 8, + 1, + 3 + ], + "offset_f32": 440, + "count": 24 + }, + "final_layer_norm.bias": { + "shape": [ + 8 + ], + "offset_f32": 464, + "count": 8 + }, + "final_layer_norm.weight": { + "shape": [ + 8 + ], + "offset_f32": 472, + "count": 8 + }, + "self_attn.k_proj.bias": { + "shape": [ + 8 + ], + "offset_f32": 480, + "count": 8 + }, + "self_attn.k_proj.weight": { + "shape": [ + 8, + 12 + ], + "offset_f32": 488, + "count": 96 + }, + "self_attn.out_proj.bias": { + "shape": [ + 8 + ], + "offset_f32": 584, + "count": 8 + }, + "self_attn.out_proj.weight": { + "shape": [ + 8, + 8 + ], + "offset_f32": 592, + "count": 64 + }, + "self_attn.q_proj.bias": { + "shape": [ + 8 + ], + "offset_f32": 656, + "count": 8 + }, + "self_attn.q_proj.weight": { + "shape": [ + 8, + 12 + ], + "offset_f32": 664, + "count": 96 + }, + "self_attn.v_proj.bias": { + "shape": [ + 8 + ], + "offset_f32": 760, + "count": 8 + }, + "self_attn.v_proj.weight": { + "shape": [ + 8, + 12 + ], + "offset_f32": 768, + "count": 96 + }, + "self_attn_layer_norm.bias": { + "shape": [ + 12 + ], + "offset_f32": 864, + "count": 12 + }, + "self_attn_layer_norm.weight": { + "shape": [ + 12 + ], + "offset_f32": 876, + "count": 12 + } + } + }, + "residual": { + "input": { + "shape": [ + 1, + 5, + 8 + ], + "offset_f32": 888, + "count": 40 + }, + "output": { + "shape": [ + 1, + 5, + 8 + ], + "offset_f32": 928, + "count": 40 + }, + "checkpoints": { + "self_attn_layer_norm": { + "shape": [ + 1, + 5, + 8 + ], + "offset_f32": 968, + "count": 40 + } + }, + "weights": { + "fc1.bias": { + "shape": [ + 16 + ], + "offset_f32": 1008, + "count": 16 + }, + "fc1.weight": { + "shape": [ + 16, + 8 + ], + "offset_f32": 1024, + "count": 128 + }, + "fc2.bias": { + "shape": [ + 8 + ], + "offset_f32": 1152, + "count": 8 + }, + "fc2.weight": { + "shape": [ + 8, + 16 + ], + "offset_f32": 1160, + "count": 128 + }, + "feedforward_sequential_memory.conv.weight": { + "shape": [ + 8, + 1, + 3 + ], + "offset_f32": 1288, + "count": 24 + }, + "final_layer_norm.bias": { + "shape": [ + 8 + ], + "offset_f32": 1312, + "count": 8 + }, + "final_layer_norm.weight": { + "shape": [ + 8 + ], + "offset_f32": 1320, + "count": 8 + }, + "self_attn.k_proj.bias": { + "shape": [ + 8 + ], + "offset_f32": 1328, + "count": 8 + }, + "self_attn.k_proj.weight": { + "shape": [ + 8, + 8 + ], + "offset_f32": 1336, + "count": 64 + }, + "self_attn.out_proj.bias": { + "shape": [ + 8 + ], + "offset_f32": 1400, + "count": 8 + }, + "self_attn.out_proj.weight": { + "shape": [ + 8, + 8 + ], + "offset_f32": 1408, + "count": 64 + }, + "self_attn.q_proj.bias": { + "shape": [ + 8 + ], + "offset_f32": 1472, + "count": 8 + }, + "self_attn.q_proj.weight": { + "shape": [ + 8, + 8 + ], + "offset_f32": 1480, + "count": 64 + }, + "self_attn.v_proj.bias": { + "shape": [ + 8 + ], + "offset_f32": 1544, + "count": 8 + }, + "self_attn.v_proj.weight": { + "shape": [ + 8, + 8 + ], + "offset_f32": 1552, + "count": 64 + }, + "self_attn_layer_norm.bias": { + "shape": [ + 8 + ], + "offset_f32": 1616, + "count": 8 + }, + "self_attn_layer_norm.weight": { + "shape": [ + 8 + ], + "offset_f32": 1624, + "count": 8 + } + } + } + } +} diff --git a/tests/unittests/test_fun_asr_nano_assets.cpp b/tests/unittests/test_fun_asr_nano_assets.cpp new file mode 100644 index 00000000..03ff416b --- /dev/null +++ b/tests/unittests/test_fun_asr_nano_assets.cpp @@ -0,0 +1,495 @@ +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/io/safetensors.h" +#include "engine/models/fun_asr_nano/assets.h" +#include "test_assert.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct ResourceOptions { + bool include_tokenizer = true; + bool include_stem = true; + bool include_embedding = true; +}; + +std::vector float_bytes(float value) { + std::vector bytes(sizeof(value)); + std::memcpy(bytes.data(), &value, sizeof(value)); + return bytes; +} + +void write_text(const std::filesystem::path &path, const std::string &text) { + std::ofstream output(path, std::ios::binary); + output << text; +} + +std::string processor_config() { + return R"json({ + "audio_token":"<|object_ref_start|>", + "default_transcription_prompt":"Transcribe the audio:", + "feature_extractor":{ + "feature_size":80, + "sampling_rate":16000, + "frame_length":25, + "frame_shift":10, + "lfr_m":7, + "lfr_n":6, + "preemphasis":0.97, + "return_attention_mask":true + } + })json"; +} + +engine::assets::ResourceBundle make_resources(const std::filesystem::path &root, + const std::string &config_json, + ResourceOptions options = {}) { + std::filesystem::create_directories(root); + write_text(root / "config.json", config_json); + write_text(root / "generation_config.json", + R"json({"eos_token_id":151645,"bos_token_id":151643})json"); + write_text(root / "processor_config.json", processor_config()); + if (options.include_tokenizer) { + write_text(root / "tokenizer.json", + R"json({"version":"1.0","added_tokens":[]})json"); + } + std::vector tensors; + if (options.include_stem) { + tensors.push_back({ + "model.audio_tower.stem.self_attn.q_proj.weight", + "F32", + {1}, + float_bytes(1.0F), + }); + } + if (options.include_embedding) { + tensors.push_back({ + "model.language_model.embed_tokens.weight", + "F32", + {1}, + float_bytes(1.0F), + }); + } + engine::io::write_safetensors_file(root / "model.safetensors", tensors); + + engine::assets::ResourceBundle resources(root); + resources.add_file("config", root / "config.json"); + resources.add_file("generation_config", root / "generation_config.json"); + resources.add_file("processor_config", root / "processor_config.json"); + if (options.include_tokenizer) { + resources.add_file("tokenizer_json", root / "tokenizer.json"); + } + resources.add_tensor_source("weights", root / "model.safetensors"); + return resources; +} + +std::string current_config() { + return R"json({ + "model_type":"fun_asr_nano", + "audio_token_id":151646, + "projector_hidden_size":2048, + "tie_word_embeddings":true, + "encoder_config":{ + "model_type":"fun_asr_nano_encoder", + "num_mel_bins":80, + "num_stacked_frames":7, + "d_model":512, + "encoder_attention_heads":4, + "encoder_ffn_dim":2048, + "encoder_layers":50, + "num_timestamp_prediction_blocks":20, + "kernel_size":11, + "activation_function":"relu", + "max_position_embeddings":2049 + }, + "adaptor_config":{ + "d_model":1024, + "encoder_attention_heads":8, + "encoder_ffn_dim":256, + "encoder_layers":2, + "activation_function":"relu" + }, + "text_config":{ + "model_type":"qwen3", + "vocab_size":151936, + "hidden_size":1024, + "intermediate_size":3072, + "num_hidden_layers":28, + "num_attention_heads":16, + "num_key_value_heads":8, + "head_dim":128, + "max_position_embeddings":40960, + "rms_norm_eps":0.000001, + "rope_parameters":{"rope_theta":1000000}, + "tie_word_embeddings":true + } + })json"; +} + +std::string legacy_config() { + return R"json({ + "model_type":"fun_asr_nano", + "audio_token_id":151646, + "activation_function":"relu", + "adaptor_intermediate_size":2048, + "adaptor_num_attention_heads":8, + "adaptor_num_hidden_layers":2, + "tie_word_embeddings":true, + "encoder_config":{ + "model_type":"fun_asr_nano_encoder", + "num_mel_bins":80, + "num_stacked_frames":7, + "d_model":512, + "encoder_attention_heads":4, + "encoder_ffn_dim":2048, + "encoder_layers":50, + "num_timestamp_prediction_blocks":20, + "kernel_size":11, + "activation_function":"relu" + }, + "text_config":{ + "model_type":"qwen3", + "vocab_size":151936, + "hidden_size":1024, + "intermediate_size":3072, + "num_hidden_layers":28, + "num_attention_heads":16, + "num_key_value_heads":8, + "head_dim":128, + "max_position_embeddings":40960, + "rms_norm_eps":0.000001, + "rope_parameters":{"rope_theta":1000000}, + "tie_word_embeddings":true + } + })json"; +} + +std::string replace_once(std::string value, const std::string &from, + const std::string &to) { + const auto position = value.find(from); + if (position == std::string::npos) { + throw std::runtime_error("test fixture marker not found: " + from); + } + value.replace(position, from.size(), to); + return value; +} + +void require_rejected(const std::filesystem::path &root, + const std::string &expected_message, + const std::function &action) { + bool rejected = false; + try { + action(); + } catch (const std::runtime_error &error) { + rejected = + std::string(error.what()).find(expected_message) != std::string::npos; + } + engine::test::require(rejected, + "Fun-ASR did not reject invalid assets with: " + + expected_message); + std::filesystem::remove_all(root); +} + +void require_published_dimensions( + const engine::models::fun_asr_nano::FunAsrNanoConfig &config) { + engine::test::require_eq(config.model_type, std::string("fun_asr_nano"), + "Fun-ASR model type"); + engine::test::require_eq(config.encoder.input_size, int64_t{560}, + "Fun-ASR encoder input size"); + engine::test::require_eq(config.encoder.d_model, int64_t{512}, + "Fun-ASR encoder width"); + engine::test::require_eq(config.encoder.layers, int64_t{50}, + "Fun-ASR main blocks"); + engine::test::require_eq(config.encoder.timestamp_prediction_layers, + int64_t{20}, "Fun-ASR timestamp blocks"); + engine::test::require_eq(config.encoder.attention_heads, int64_t{4}, + "Fun-ASR encoder heads"); + engine::test::require_eq(config.encoder.ffn_dim, int64_t{2048}, + "Fun-ASR encoder FFN"); + engine::test::require_eq(config.encoder.kernel_size, int64_t{11}, + "Fun-ASR FSMN kernel"); + engine::test::require_eq(config.encoder.max_position_embeddings, + int64_t{2049}, "Fun-ASR encoder position limit"); + engine::test::require_eq(config.projector_hidden_size, int64_t{2048}, + "Fun-ASR projector width"); + engine::test::require_eq(config.adaptor.d_model, int64_t{1024}, + "Fun-ASR adaptor width"); + engine::test::require_eq(config.adaptor.attention_heads, int64_t{8}, + "Fun-ASR adaptor heads"); + engine::test::require_eq(config.adaptor.ffn_dim, int64_t{256}, + "Fun-ASR adaptor FFN"); + engine::test::require_eq(config.adaptor.layers, int64_t{2}, + "Fun-ASR adaptor layers"); + engine::test::require_eq(config.text.hidden_size, int64_t{1024}, + "Fun-ASR text width"); + engine::test::require_eq(config.text.intermediate_size, int64_t{3072}, + "Fun-ASR text FFN"); + engine::test::require_eq(config.text.layers, int64_t{28}, + "Fun-ASR text layers"); + engine::test::require_eq(config.text.attention_heads, int64_t{16}, + "Fun-ASR text heads"); + engine::test::require_eq(config.text.key_value_heads, int64_t{8}, + "Fun-ASR KV heads"); + engine::test::require_eq(config.text.head_dim, int64_t{128}, + "Fun-ASR text head dimension"); + engine::test::require_eq(config.text.vocab_size, int64_t{151936}, + "Fun-ASR vocabulary"); + engine::test::require_close(config.text.rope_theta, 1000000.0F, 0.0F, + "Fun-ASR RoPE theta"); + engine::test::require_eq(config.frontend.sample_rate, 16000, + "Fun-ASR sample rate"); + engine::test::require_eq(config.frontend.feature_size, int64_t{80}, + "Fun-ASR frontend feature size"); + engine::test::require_eq(config.frontend.frame_length_ms, int64_t{25}, + "Fun-ASR frontend frame length"); + engine::test::require_eq(config.frontend.frame_shift_ms, int64_t{10}, + "Fun-ASR frontend frame shift"); + engine::test::require_eq(config.frontend.lfr_m, int64_t{7}, + "Fun-ASR LFR stack"); + engine::test::require_eq(config.frontend.lfr_n, int64_t{6}, + "Fun-ASR LFR stride"); + engine::test::require_close(config.frontend.preemphasis, 0.97F, 0.0F, + "Fun-ASR frontend preemphasis"); +} + +void test_current_nested_config() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_nested_assets_test"; + std::filesystem::remove_all(root); + auto assets = engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, current_config())); + require_published_dimensions(assets->config); + std::filesystem::remove_all(root); +} + +void test_legacy_flat_adaptor_config() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_legacy_assets_test"; + std::filesystem::remove_all(root); + auto assets = engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, legacy_config())); + require_published_dimensions(assets->config); + std::filesystem::remove_all(root); +} + +void test_loads_model_directory_through_family_spec() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_model_spec_test"; + std::filesystem::remove_all(root); + (void)make_resources(root, current_config()); + auto assets = engine::models::fun_asr_nano::load_fun_asr_nano_assets(root); + require_published_dimensions(assets->config); + engine::test::require_eq(assets->resources.model_root(), root, + "Fun-ASR model root"); + std::filesystem::remove_all(root); +} + +void test_rejects_incompatible_adaptor_width() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_invalid_assets_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"d_model\":1024", "\"d_model\":768"); + require_rejected(root, "adaptor", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_nonpublished_adaptor_shape() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_adaptor_shape_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"encoder_attention_heads\":8", + "\"encoder_attention_heads\":16"); + require_rejected(root, "published adaptor architecture", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_nonpublished_encoder_shape() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_encoder_shape_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"d_model\":512", "\"d_model\":768"); + require_rejected(root, "published encoder architecture", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_nonpublished_encoder_position_limit() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_encoder_position_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"max_position_embeddings\":2049", + "\"max_position_embeddings\":1024"); + require_rejected(root, "published encoder architecture", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_nonpublished_text_shape() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_text_shape_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"intermediate_size\":3072", + "\"intermediate_size\":4096"); + require_rejected(root, "published Qwen architecture", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_zero_text_attention_heads() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_zero_text_heads_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"num_attention_heads\":16", + "\"num_attention_heads\":0"); + require_rejected(root, "text dimensions", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_nonpublished_projector_shape() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_projector_shape_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"projector_hidden_size\":2048", + "\"projector_hidden_size\":4096"); + require_rejected(root, "projector hidden size", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_nonpublished_frontend_parameters() { + const std::vector> mutations = { + {"\"frame_length\":25", "\"frame_length\":30"}, + {"\"frame_shift\":10", "\"frame_shift\":20"}, + {"\"lfr_n\":6", "\"lfr_n\":5"}, + {"\"preemphasis\":0.97", "\"preemphasis\":0.50"}, + }; + for (size_t index = 0; index < mutations.size(); ++index) { + const auto root = + std::filesystem::temp_directory_path() / + ("audiocpp_fun_asr_frontend_shape_test_" + std::to_string(index)); + std::filesystem::remove_all(root); + auto resources = make_resources(root, current_config()); + write_text(root / "processor_config.json", + replace_once(processor_config(), mutations[index].first, + mutations[index].second)); + require_rejected(root, "published frontend", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + std::move(resources)); + }); + } +} + +void test_rejects_unsupported_activation() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_activation_assets_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"activation_function\":\"relu\"", + "\"activation_function\":\"gelu\""); + require_rejected(root, "ReLU encoder", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_untied_embeddings() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_untied_assets_test"; + std::filesystem::remove_all(root); + const auto invalid = + replace_once(current_config(), "\"tie_word_embeddings\":true", + "\"tie_word_embeddings\":false"); + require_rejected(root, "tied text embeddings", [&] { + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, invalid)); + }); +} + +void test_rejects_missing_tokenizer() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_tokenizer_assets_test"; + std::filesystem::remove_all(root); + require_rejected(root, "tokenizer_json", [&] { + ResourceOptions options; + options.include_tokenizer = false; + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, current_config(), options)); + }); +} + +void test_rejects_missing_stem_tensor() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_stem_assets_test"; + std::filesystem::remove_all(root); + require_rejected(root, "audio_tower.stem", [&] { + ResourceOptions options; + options.include_stem = false; + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, current_config(), options)); + }); +} + +void test_rejects_missing_token_embedding() { + const auto root = std::filesystem::temp_directory_path() / + "audiocpp_fun_asr_embedding_assets_test"; + std::filesystem::remove_all(root); + require_rejected(root, "embed_tokens", [&] { + ResourceOptions options; + options.include_embedding = false; + (void)engine::models::fun_asr_nano::load_fun_asr_nano_assets( + make_resources(root, current_config(), options)); + }); +} + +} // namespace + +int main() { + try { + test_current_nested_config(); + test_legacy_flat_adaptor_config(); + test_loads_model_directory_through_family_spec(); + test_rejects_incompatible_adaptor_width(); + test_rejects_nonpublished_adaptor_shape(); + test_rejects_nonpublished_encoder_shape(); + test_rejects_nonpublished_encoder_position_limit(); + test_rejects_nonpublished_text_shape(); + test_rejects_zero_text_attention_heads(); + test_rejects_nonpublished_projector_shape(); + test_rejects_nonpublished_frontend_parameters(); + test_rejects_unsupported_activation(); + test_rejects_untied_embeddings(); + test_rejects_missing_tokenizer(); + test_rejects_missing_stem_tensor(); + test_rejects_missing_token_embedding(); + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; + } + std::cout << "fun_asr_nano_assets_test passed\n"; + return 0; +} diff --git a/tools/model_manager.py b/tools/model_manager.py index 49aa70c0..e54892c5 100644 --- a/tools/model_manager.py +++ b/tools/model_manager.py @@ -352,6 +352,42 @@ def package_usage_examples(package: ModelPackage) -> list[str]: "audio_tokenizer/model.safetensors", ), ), + ModelPackage( + id="fun_asr_nano_2512_q8_0", + display_name="Fun-ASR-Nano-2512 Q8_0 GGUF", + target_directory="Fun-ASR-Nano-2512-GGUF", + source=SnapshotSource( + repo_id="FunAudioLLM/Fun-ASR-Nano-2512-GGUF", + revision="ce72677f84900f0dc57f498ace253bfb3c9155b6", + include_prefixes=("fun-asr-nano-2512-q8_0.gguf",), + ), + required_files=("fun-asr-nano-2512-q8_0.gguf",), + family="fun_asr_nano", + tasks=("asr",), + modes=("offline",), + description=( + "Standalone Fun-ASR-Nano-2512 Q8_0 GGUF for offline multilingual ASR; " + "governed by the FunASR Model Open Source License Agreement v1.1." + ), + ), + ModelPackage( + id="fun_asr_nano_2512_f16", + display_name="Fun-ASR-Nano-2512 F16 GGUF", + target_directory="Fun-ASR-Nano-2512-GGUF", + source=SnapshotSource( + repo_id="FunAudioLLM/Fun-ASR-Nano-2512-GGUF", + revision="ce72677f84900f0dc57f498ace253bfb3c9155b6", + include_prefixes=("fun-asr-nano-2512-f16.gguf",), + ), + required_files=("fun-asr-nano-2512-f16.gguf",), + family="fun_asr_nano", + tasks=("asr",), + modes=("offline",), + description=( + "Standalone Fun-ASR-Nano-2512 F16 GGUF for offline multilingual ASR; " + "governed by the FunASR Model Open Source License Agreement v1.1." + ), + ), ModelPackage( id="qwen3_asr_0_6b", display_name="Qwen3 ASR 0.6B",