diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d2601b7..85d9694b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,6 +147,7 @@ add_library(engine_runtime STATIC src/framework/core/backend.cpp src/framework/core/deferred_tensor_writer.cpp src/framework/core/execution_context.cpp + src/framework/core/host_memory.cpp src/framework/runtime/registry.cpp src/framework/runtime/model.cpp src/framework/runtime/session.cpp diff --git a/include/engine/framework/core/host_memory.h b/include/engine/framework/core/host_memory.h new file mode 100644 index 00000000..554d9d73 --- /dev/null +++ b/include/engine/framework/core/host_memory.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace engine::core { + +// Physical memory the host still has free, or 0 when it cannot be determined +// (no portable query on this platform). Callers must treat 0 as "unknown" and +// fall back to whatever they would have done anyway, never as "no memory". +size_t available_host_memory_bytes(); + +} // namespace engine::core diff --git a/include/engine/models/qwen3_tts/session.h b/include/engine/models/qwen3_tts/session.h index 6c14df6a..064454f3 100644 --- a/include/engine/models/qwen3_tts/session.h +++ b/include/engine/models/qwen3_tts/session.h @@ -63,6 +63,8 @@ class Qwen3TTSSession final size_t speech_decoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t speaker_encoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; // No-alloc GGML context capacities for reusable constant tensor descriptors. + // Generous on purpose; ConstantTensorCache fits them to the host when it has + // to, so a small machine is not asked to reserve what it does not have. size_t talker_constant_context_bytes_ = 4ull * 1024ull * 1024ull * 1024ull; size_t code_predictor_constant_context_bytes_ = 1536ull * 1024ull * 1024ull; size_t speech_decoder_constant_context_bytes_ = 1536ull * 1024ull * 1024ull; diff --git a/src/framework/core/host_memory.cpp b/src/framework/core/host_memory.cpp new file mode 100644 index 00000000..377857c7 --- /dev/null +++ b/src/framework/core/host_memory.cpp @@ -0,0 +1,64 @@ +#include "engine/framework/core/host_memory.h" + +#if defined(_WIN32) +#include +#elif defined(__linux__) +#include +#include +#elif defined(__unix__) || defined(__APPLE__) +#include +#endif + +namespace engine::core { + +#if defined(__linux__) +namespace { + +// MemAvailable: what the kernel estimates a new allocation can get without +// swapping, counting reclaimable page cache. sysconf(_SC_AVPHYS_PAGES) reports +// MemFree instead, which sits near zero on any busy machine because spare RAM +// is used for cache -- reading it as memory pressure would be wrong. +size_t mem_available_bytes() { + std::FILE * meminfo = std::fopen("/proc/meminfo", "r"); + if (meminfo == nullptr) { + return 0; + } + char line[256]; + size_t bytes = 0; + while (std::fgets(line, sizeof(line), meminfo) != nullptr) { + unsigned long long kib = 0; + if (std::sscanf(line, "MemAvailable: %llu kB", &kib) == 1) { + bytes = static_cast(kib) * 1024ull; + break; + } + } + std::fclose(meminfo); + return bytes; +} + +} // namespace +#endif + +size_t available_host_memory_bytes() { +#if defined(_WIN32) + MEMORYSTATUSEX status{}; + status.dwLength = sizeof(status); + if (GlobalMemoryStatusEx(&status)) { + return static_cast(status.ullAvailPhys); + } +#elif defined(__linux__) + if (const size_t bytes = mem_available_bytes(); bytes > 0) { + return bytes; + } +#elif defined(_SC_AVPHYS_PAGES) && defined(_SC_PAGE_SIZE) + const long pages = sysconf(_SC_AVPHYS_PAGES); + const long page_size = sysconf(_SC_PAGE_SIZE); + if (pages > 0 && page_size > 0) { + return static_cast(pages) * static_cast(page_size); + } +#endif + // macOS has no _SC_AVPHYS_PAGES, and any query can fail: report unknown. + return 0; +} + +} // namespace engine::core diff --git a/src/models/common/constant_tensor_cache.h b/src/models/common/constant_tensor_cache.h index 8eeddee6..37e89753 100644 --- a/src/models/common/constant_tensor_cache.h +++ b/src/models/common/constant_tensor_cache.h @@ -1,7 +1,9 @@ #pragma once #include "engine/framework/core/backend.h" +#include "engine/framework/core/host_memory.h" #include "engine/framework/core/module.h" +#include "engine/framework/debug/trace.h" #include #include @@ -19,6 +21,51 @@ namespace engine::models::common { +namespace detail { + +// Smallest context this guard will hand out. A tensor descriptor costs roughly +// ggml_object + ggml_tensor (~400 B), so even this floor holds ~160k of them -- +// far past what any model here builds. +inline constexpr size_t kMinConstantContextBytes = 64ull * 1024ull * 1024ull; + +// Fraction of still-free host memory a single descriptor context may reserve. +inline constexpr size_t kConstantContextMemoryDivisor = 4; + +// Fit a descriptor-context reservation to what the host can actually afford. +// +// These contexts are created with no_alloc, so they only ever hold tensor +// descriptors -- but ggml_init() mallocs mem_size regardless of no_alloc (it +// calls ggml_aligned_malloc whenever mem_buffer is null). What that costs +// depends entirely on the OS: Linux never makes the untouched pages resident, +// so an oversized request is just address space, while Windows charges the +// whole reservation against the commit limit immediately and a multi-gigabyte +// context can fail outright on a small machine. +// +// So rather than shrink the defaults for everyone -- capable hosts lose nothing +// by reserving generously -- only step in when the request outweighs the host. +inline size_t fit_constant_context_bytes(size_t requested, const std::string & name) { + const size_t available = engine::core::available_host_memory_bytes(); + if (available == 0) { + return requested; // unknown: trust the caller rather than guess + } + const size_t budget = available / kConstantContextMemoryDivisor; + if (requested <= budget) { + return requested; + } + const size_t fitted = std::max(kMinConstantContextBytes, budget); + if (fitted >= requested) { + return requested; + } + engine::debug::log_message( + engine::debug::LogLevel::Warning, "constant_cache", + name + " descriptor context " + std::to_string(requested >> 20) + + " MiB exceeds the memory budget; using " + std::to_string(fitted >> 20) + + " MiB (host has " + std::to_string(available >> 20) + " MiB free)"); + return fitted; +} + +} // namespace detail + class ConstantTensorCache { public: ConstantTensorCache( @@ -32,7 +79,8 @@ class ConstantTensorCache { if (backend_ == nullptr) { throw std::runtime_error(name_ + " constant cache backend is not initialized"); } - ggml_init_params params{context_bytes, nullptr, true}; + ggml_init_params params{detail::fit_constant_context_bytes(context_bytes, name_), + nullptr, true}; ctx_.reset(ggml_init(params)); if (ctx_ == nullptr) { throw std::runtime_error("failed to initialize " + name_ + " constant tensor cache");