Fit constant tensor descriptor contexts to host memory

CMakeLists.txt

@@ -147,6 +147,7 @@ add_library(engine_runtime STATIC
     src/framework/core/backend.cpp
     src/framework/core/deferred_tensor_writer.cpp
     src/framework/core/execution_context.cpp
+    src/framework/core/host_memory.cpp
     src/framework/runtime/registry.cpp
     src/framework/runtime/model.cpp
     src/framework/runtime/session.cpp

include/engine/framework/core/host_memory.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <cstddef>
+
+namespace engine::core {
+
+// Physical memory the host still has free, or 0 when it cannot be determined
+// (no portable query on this platform). Callers must treat 0 as "unknown" and
+// fall back to whatever they would have done anyway, never as "no memory".
+size_t available_host_memory_bytes();
+
+}  // namespace engine::core

include/engine/models/qwen3_tts/session.h

@@ -63,6 +63,8 @@ class Qwen3TTSSession final
     size_t speech_decoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull;
     size_t speaker_encoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull;
     // No-alloc GGML context capacities for reusable constant tensor descriptors.
+    // Generous on purpose; ConstantTensorCache fits them to the host when it has
+    // to, so a small machine is not asked to reserve what it does not have.
     size_t talker_constant_context_bytes_ = 4ull * 1024ull * 1024ull * 1024ull;
     size_t code_predictor_constant_context_bytes_ = 1536ull * 1024ull * 1024ull;
     size_t speech_decoder_constant_context_bytes_ = 1536ull * 1024ull * 1024ull;

src/framework/core/host_memory.cpp

@@ -0,0 +1,64 @@
+#include "engine/framework/core/host_memory.h"
+
+#if defined(_WIN32)
+#include <windows.h>
+#elif defined(__linux__)
+#include <cstdio>
+#include <cstring>
+#elif defined(__unix__) || defined(__APPLE__)
+#include <unistd.h>
+#endif
+
+namespace engine::core {
+
+#if defined(__linux__)
+namespace {
+
+// MemAvailable: what the kernel estimates a new allocation can get without
+// swapping, counting reclaimable page cache. sysconf(_SC_AVPHYS_PAGES) reports
+// MemFree instead, which sits near zero on any busy machine because spare RAM
+// is used for cache -- reading it as memory pressure would be wrong.
+size_t mem_available_bytes() {
+    std::FILE * meminfo = std::fopen("/proc/meminfo", "r");
+    if (meminfo == nullptr) {
+        return 0;
+    }
+    char line[256];
+    size_t bytes = 0;
+    while (std::fgets(line, sizeof(line), meminfo) != nullptr) {
+        unsigned long long kib = 0;
+        if (std::sscanf(line, "MemAvailable: %llu kB", &kib) == 1) {
+            bytes = static_cast<size_t>(kib) * 1024ull;
+            break;
+        }
+    }
+    std::fclose(meminfo);
+    return bytes;
+}
+
+}  // namespace
+#endif
+
+size_t available_host_memory_bytes() {
+#if defined(_WIN32)
+    MEMORYSTATUSEX status{};
+    status.dwLength = sizeof(status);
+    if (GlobalMemoryStatusEx(&status)) {
+        return static_cast<size_t>(status.ullAvailPhys);
+    }
+#elif defined(__linux__)
+    if (const size_t bytes = mem_available_bytes(); bytes > 0) {
+        return bytes;
+    }
+#elif defined(_SC_AVPHYS_PAGES) && defined(_SC_PAGE_SIZE)
+    const long pages = sysconf(_SC_AVPHYS_PAGES);
+    const long page_size = sysconf(_SC_PAGE_SIZE);
+    if (pages > 0 && page_size > 0) {
+        return static_cast<size_t>(pages) * static_cast<size_t>(page_size);
+    }
+#endif
+    // macOS has no _SC_AVPHYS_PAGES, and any query can fail: report unknown.
+    return 0;
+}
+
+}  // namespace engine::core

src/models/common/constant_tensor_cache.h

@@ -1,7 +1,9 @@
 #pragma once
 
 #include "engine/framework/core/backend.h"
+#include "engine/framework/core/host_memory.h"
 #include "engine/framework/core/module.h"
+#include "engine/framework/debug/trace.h"
 
 #include <ggml-backend.h>
 #include <ggml.h>
@@ -19,6 +21,51 @@
 
 namespace engine::models::common {
 
+namespace detail {
+
+// Smallest context this guard will hand out. A tensor descriptor costs roughly
+// ggml_object + ggml_tensor (~400 B), so even this floor holds ~160k of them --
+// far past what any model here builds.
+inline constexpr size_t kMinConstantContextBytes = 64ull * 1024ull * 1024ull;
+
+// Fraction of still-free host memory a single descriptor context may reserve.
+inline constexpr size_t kConstantContextMemoryDivisor = 4;
+
+// Fit a descriptor-context reservation to what the host can actually afford.
+//
+// These contexts are created with no_alloc, so they only ever hold tensor
+// descriptors -- but ggml_init() mallocs mem_size regardless of no_alloc (it
+// calls ggml_aligned_malloc whenever mem_buffer is null). What that costs
+// depends entirely on the OS: Linux never makes the untouched pages resident,
+// so an oversized request is just address space, while Windows charges the
+// whole reservation against the commit limit immediately and a multi-gigabyte
+// context can fail outright on a small machine.
+//
+// So rather than shrink the defaults for everyone -- capable hosts lose nothing
+// by reserving generously -- only step in when the request outweighs the host.
+inline size_t fit_constant_context_bytes(size_t requested, const std::string & name) {
+    const size_t available = engine::core::available_host_memory_bytes();
+    if (available == 0) {
+        return requested;  // unknown: trust the caller rather than guess
+    }
+    const size_t budget = available / kConstantContextMemoryDivisor;
+    if (requested <= budget) {
+        return requested;
+    }
+    const size_t fitted = std::max(kMinConstantContextBytes, budget);
+    if (fitted >= requested) {
+        return requested;
+    }
+    engine::debug::log_message(
+        engine::debug::LogLevel::Warning, "constant_cache",
+        name + " descriptor context " + std::to_string(requested >> 20) +
+            " MiB exceeds the memory budget; using " + std::to_string(fitted >> 20) +
+            " MiB (host has " + std::to_string(available >> 20) + " MiB free)");
+    return fitted;
+}
+
+}  // namespace detail
+
 class ConstantTensorCache {
 public:
     ConstantTensorCache(
@@ -32,7 +79,8 @@ class ConstantTensorCache {
         if (backend_ == nullptr) {
             throw std::runtime_error(name_ + " constant cache backend is not initialized");
         }
-        ggml_init_params params{context_bytes, nullptr, true};
+        ggml_init_params params{detail::fit_constant_context_bytes(context_bytes, name_),
+                                nullptr, true};
         ctx_.reset(ggml_init(params));
         if (ctx_ == nullptr) {
             throw std::runtime_error("failed to initialize " + name_ + " constant tensor cache");