From df813305df8d3a5c1aea1e35147611811aaf41d1 Mon Sep 17 00:00:00 2001 From: "peng.li24" Date: Thu, 4 Jun 2026 08:49:24 +0000 Subject: [PATCH 1/4] fix: prevent unsigned integer underflow when usage exceeds memory limit When usage > limit, (limit - usage) wraps around to a massive unsigned value (~18 EB). Clamp both free and used so that free + used = total always holds, preventing corrupted values in monitoring tools. Fixes #200 Signed-off-by: peng.li24 Co-Authored-By: Claude Opus 4.7 Signed-off-by: peng.li24 <734991033@qq.com> --- src/nvml/hook.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/nvml/hook.c b/src/nvml/hook.c index 0bc16d31..a0511670 100644 --- a/src/nvml/hook.c +++ b/src/nvml/hook.c @@ -353,16 +353,17 @@ nvmlReturn_t _nvmlDeviceGetMemoryInfo(nvmlDevice_t device,void* memory,int versi return NVML_SUCCESS; } } else { + size_t clamped = (usage > limit) ? limit : usage; switch (version) { case 1: - ((nvmlMemory_t*)memory)->free = (limit-usage); + ((nvmlMemory_t*)memory)->free = limit - clamped; ((nvmlMemory_t*)memory)->total = limit; - ((nvmlMemory_t*)memory)->used = usage; + ((nvmlMemory_t*)memory)->used = clamped; return NVML_SUCCESS; case 2: - ((nvmlMemory_v2_t *)memory)->free = (limit-usage); + ((nvmlMemory_v2_t *)memory)->free = limit - clamped; ((nvmlMemory_v2_t *)memory)->total = limit; - ((nvmlMemory_v2_t *)memory)->used = usage; + ((nvmlMemory_v2_t *)memory)->used = clamped; return NVML_SUCCESS; } } From d6f4da869ccadc48d7cbd33bfbc1288a23845f3c Mon Sep 17 00:00:00 2001 From: miaobyte Date: Fri, 24 Jul 2026 03:53:41 +0000 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20nvml=20memory=20underflow=20?= =?UTF-8?q?=E2=80=94=20usage>limit=20=E6=97=B6=20clamp=20=E8=80=8C?= =?UTF-8?q?=E9=9D=9E=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cuMemGetInfo_v2: usage 超过 limit 时不再返回 INVALID_VALUE， 而是 clamp 到 limit 再计算 free memory。 nvmlDeviceGetMemoryInfo: 同样添加 clamp + 警告日志。 Co-Authored-By: Claude Signed-off-by: peng.li24 <734991033@qq.com> Signed-off-by: miaobyte Signed-off-by: peng.li24 <734991033@qq.com> --- src/cuda/memory.c | 9 +++++---- src/nvml/hook.c | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cuda/memory.c b/src/cuda/memory.c index 00857f30..bdfddf82 100755 --- a/src/cuda/memory.c +++ b/src/cuda/memory.c @@ -512,16 +512,17 @@ CUresult cuMemGetInfo_v2(size_t* free, size_t* total) { *free = *total - usage; LOG_INFO("after free=%ld total=%ld", *free, *total); return CUDA_SUCCESS; - } else if (limit < usage) { - LOG_WARN("limit < usage; usage=%ld, limit=%ld", usage, limit); - return CUDA_ERROR_INVALID_VALUE; } else { CUDA_OVERRIDE_CALL(cuda_library_entry,cuMemGetInfo_v2, free, total); LOG_INFO("orig free=%ld total=%ld limit=%ld usage=%ld", *free, *total, limit, usage); // Ensure total memory does not exceed the physical or imposed limit. size_t actual_limit = (limit > *total) ? *total : limit; - *free = (actual_limit > usage) ? (actual_limit - usage) : 0; + size_t clamped = (usage > limit) ? limit : usage; + if (usage > limit) { + LOG_WARN("CUDA meminfo: usage %lu exceeds limit %lu, clamping", usage, limit); + } + *free = (actual_limit > clamped) ? (actual_limit - clamped) : 0; *total = actual_limit; LOG_INFO("after free=%ld total=%ld limit=%ld usage=%ld", *free, *total, limit, usage); diff --git a/src/nvml/hook.c b/src/nvml/hook.c index a0511670..38a6c37c 100644 --- a/src/nvml/hook.c +++ b/src/nvml/hook.c @@ -354,6 +354,9 @@ nvmlReturn_t _nvmlDeviceGetMemoryInfo(nvmlDevice_t device,void* memory,int versi } } else { size_t clamped = (usage > limit) ? limit : usage; + if (usage > limit) { + LOG_WARN("NVML meminfo: usage %lu exceeds limit %lu, clamping", usage, limit); + } switch (version) { case 1: ((nvmlMemory_t*)memory)->free = limit - clamped; From fee12bea090a927e5dd797a4c78091bd9eb501d2 Mon Sep 17 00:00:00 2001 From: "peng.li24" <734991033@qq.com> Date: Tue, 28 Jul 2026 06:41:41 +0000 Subject: [PATCH 3/4] fix: clamp NVML total against physical GPU memory with min(limit, physical_total) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match the CUDA path behavior in src/cuda/memory.c:520 — if the configured limit exceeds the real GPU memory, report the physical total rather than the impossibly-high limit. Co-Authored-By: Claude Signed-off-by: peng.li24 <734991033@qq.com> --- src/nvml/hook.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/nvml/hook.c b/src/nvml/hook.c index 38a6c37c..b6bcc66b 100644 --- a/src/nvml/hook.c +++ b/src/nvml/hook.c @@ -358,17 +358,23 @@ nvmlReturn_t _nvmlDeviceGetMemoryInfo(nvmlDevice_t device,void* memory,int versi LOG_WARN("NVML meminfo: usage %lu exceeds limit %lu, clamping", usage, limit); } switch (version) { - case 1: - ((nvmlMemory_t*)memory)->free = limit - clamped; - ((nvmlMemory_t*)memory)->total = limit; - ((nvmlMemory_t*)memory)->used = clamped; + case 1: { + size_t physical_total = ((nvmlMemory_t*)memory)->total; + size_t actual_limit = (limit > physical_total) ? physical_total : limit; + ((nvmlMemory_t*)memory)->free = (actual_limit > clamped) ? (actual_limit - clamped) : 0; + ((nvmlMemory_t*)memory)->total = actual_limit; + ((nvmlMemory_t*)memory)->used = clamped; return NVML_SUCCESS; - case 2: - ((nvmlMemory_v2_t *)memory)->free = limit - clamped; - ((nvmlMemory_v2_t *)memory)->total = limit; + } + case 2: { + size_t physical_total = ((nvmlMemory_v2_t *)memory)->total; + size_t actual_limit = (limit > physical_total) ? physical_total : limit; + ((nvmlMemory_v2_t *)memory)->free = (actual_limit > clamped) ? (actual_limit - clamped) : 0; + ((nvmlMemory_v2_t *)memory)->total = actual_limit; ((nvmlMemory_v2_t *)memory)->used = clamped; return NVML_SUCCESS; - } + } + } } return NVML_SUCCESS; } From 4c6680b9c5f8d5d0469c63f03f0a89ea5536d71a Mon Sep 17 00:00:00 2001 From: "peng.li24" <734991033@qq.com> Date: Tue, 28 Jul 2026 06:48:54 +0000 Subject: [PATCH 4/4] fix: use actual_limit for NVML clamp and warning Compute physical_total and actual_limit first, then derive clamped and the warning from actual_limit instead of the raw limit. Prevents broken free+used=total invariant when limit exceeds physical GPU memory and usage falls between them. Signed-off-by: peng.li24 <734991033@qq.com> --- src/nvml/hook.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/nvml/hook.c b/src/nvml/hook.c index b6bcc66b..71b37281 100644 --- a/src/nvml/hook.c +++ b/src/nvml/hook.c @@ -353,28 +353,34 @@ nvmlReturn_t _nvmlDeviceGetMemoryInfo(nvmlDevice_t device,void* memory,int versi return NVML_SUCCESS; } } else { - size_t clamped = (usage > limit) ? limit : usage; - if (usage > limit) { - LOG_WARN("NVML meminfo: usage %lu exceeds limit %lu, clamping", usage, limit); + size_t physical_total; + switch (version) { + case 1: + physical_total = ((nvmlMemory_t*)memory)->total; + break; + case 2: + physical_total = ((nvmlMemory_v2_t *)memory)->total; + break; + default: + return NVML_ERROR_INVALID_ARGUMENT; + } + size_t actual_limit = (limit > physical_total) ? physical_total : limit; + size_t clamped = (usage > actual_limit) ? actual_limit : usage; + if (usage > actual_limit) { + LOG_WARN("NVML meminfo: usage %lu exceeds limit %lu, clamping", usage, actual_limit); } switch (version) { - case 1: { - size_t physical_total = ((nvmlMemory_t*)memory)->total; - size_t actual_limit = (limit > physical_total) ? physical_total : limit; + case 1: ((nvmlMemory_t*)memory)->free = (actual_limit > clamped) ? (actual_limit - clamped) : 0; ((nvmlMemory_t*)memory)->total = actual_limit; ((nvmlMemory_t*)memory)->used = clamped; return NVML_SUCCESS; - } - case 2: { - size_t physical_total = ((nvmlMemory_v2_t *)memory)->total; - size_t actual_limit = (limit > physical_total) ? physical_total : limit; + case 2: ((nvmlMemory_v2_t *)memory)->free = (actual_limit > clamped) ? (actual_limit - clamped) : 0; ((nvmlMemory_v2_t *)memory)->total = actual_limit; ((nvmlMemory_v2_t *)memory)->used = clamped; return NVML_SUCCESS; } - } } return NVML_SUCCESS; }