diff --git a/src/cuda/memory.c b/src/cuda/memory.c index 00857f30..bdfddf82 100755 --- a/src/cuda/memory.c +++ b/src/cuda/memory.c @@ -512,16 +512,17 @@ CUresult cuMemGetInfo_v2(size_t* free, size_t* total) { *free = *total - usage; LOG_INFO("after free=%ld total=%ld", *free, *total); return CUDA_SUCCESS; - } else if (limit < usage) { - LOG_WARN("limit < usage; usage=%ld, limit=%ld", usage, limit); - return CUDA_ERROR_INVALID_VALUE; } else { CUDA_OVERRIDE_CALL(cuda_library_entry,cuMemGetInfo_v2, free, total); LOG_INFO("orig free=%ld total=%ld limit=%ld usage=%ld", *free, *total, limit, usage); // Ensure total memory does not exceed the physical or imposed limit. size_t actual_limit = (limit > *total) ? *total : limit; - *free = (actual_limit > usage) ? (actual_limit - usage) : 0; + size_t clamped = (usage > limit) ? limit : usage; + if (usage > limit) { + LOG_WARN("CUDA meminfo: usage %lu exceeds limit %lu, clamping", usage, limit); + } + *free = (actual_limit > clamped) ? (actual_limit - clamped) : 0; *total = actual_limit; LOG_INFO("after free=%ld total=%ld limit=%ld usage=%ld", *free, *total, limit, usage); diff --git a/src/nvml/hook.c b/src/nvml/hook.c index 0bc16d31..71b37281 100644 --- a/src/nvml/hook.c +++ b/src/nvml/hook.c @@ -353,18 +353,34 @@ nvmlReturn_t _nvmlDeviceGetMemoryInfo(nvmlDevice_t device,void* memory,int versi return NVML_SUCCESS; } } else { + size_t physical_total; switch (version) { case 1: - ((nvmlMemory_t*)memory)->free = (limit-usage); - ((nvmlMemory_t*)memory)->total = limit; - ((nvmlMemory_t*)memory)->used = usage; + physical_total = ((nvmlMemory_t*)memory)->total; + break; + case 2: + physical_total = ((nvmlMemory_v2_t *)memory)->total; + break; + default: + return NVML_ERROR_INVALID_ARGUMENT; + } + size_t actual_limit = (limit > physical_total) ? physical_total : limit; + size_t clamped = (usage > actual_limit) ? actual_limit : usage; + if (usage > actual_limit) { + LOG_WARN("NVML meminfo: usage %lu exceeds limit %lu, clamping", usage, actual_limit); + } + switch (version) { + case 1: + ((nvmlMemory_t*)memory)->free = (actual_limit > clamped) ? (actual_limit - clamped) : 0; + ((nvmlMemory_t*)memory)->total = actual_limit; + ((nvmlMemory_t*)memory)->used = clamped; return NVML_SUCCESS; case 2: - ((nvmlMemory_v2_t *)memory)->free = (limit-usage); - ((nvmlMemory_v2_t *)memory)->total = limit; - ((nvmlMemory_v2_t *)memory)->used = usage; + ((nvmlMemory_v2_t *)memory)->free = (actual_limit > clamped) ? (actual_limit - clamped) : 0; + ((nvmlMemory_v2_t *)memory)->total = actual_limit; + ((nvmlMemory_v2_t *)memory)->used = clamped; return NVML_SUCCESS; - } + } } return NVML_SUCCESS; }