From 8e9a83ff190300298e8d4fd5204c24ee12d35d5c Mon Sep 17 00:00:00 2001 From: LHT129 Date: Tue, 30 Jun 2026 17:49:40 +0800 Subject: [PATCH] feat: unify memory statistics API signatures - GetMemoryUsage: int64_t -> uint64_t (memory size is never negative) - GetMemoryUsageDetail: std::string (JSON) -> std::unordered_map - Returns structured component-level memory breakdown instead of opaque JSON string; no dependency on internal JsonType in public API - HGraph implementation now uses GetMemoryUsage() per component instead of CalcSerializeSize(), ensuring consistency with GetMemoryUsage() total - SINDI now returns empty map instead of empty string - GetEstimateBuildMemory -> EstimateBuildMemory: int64_t -> uint64_t, dropped redundant Get prefix for consistency with EstimateMemory - current_memory_usage_ atomic type: int64_t -> uint64_t - Updated all index implementations, datacell layer, tests, and eval tools Signed-off-by: LHT129 Co-authored-by: opencode --- docs/docs/en/src/advanced/memory.md | 37 ++++++++++++++++++- docs/docs/zh/src/advanced/memory.md | 31 +++++++++++++++- include/vsag/index.h | 16 ++++---- include/vsag/vsag_ext.h | 2 +- mockimpl/vsag/simpleflat.h | 2 +- src/algorithm/bruteforce/bruteforce.cpp | 4 +- src/algorithm/bruteforce/bruteforce.h | 2 +- src/algorithm/hgraph/hgraph.cpp | 2 +- src/algorithm/hgraph/hgraph.h | 2 +- src/algorithm/hgraph/hgraph_serialize.cpp | 33 +++++++++-------- src/algorithm/inner_index_interface.h | 14 +++---- src/algorithm/inner_index_interface_test.cpp | 2 +- src/algorithm/ivf/ivf.cpp | 6 +-- src/algorithm/ivf/ivf.h | 2 +- src/algorithm/ivf/ivf_nearest_partition.cpp | 6 +-- src/algorithm/ivf/ivf_nearest_partition.h | 2 +- src/algorithm/ivf/ivf_partition_strategy.h | 2 +- src/algorithm/lazy_hgraph/lazy_hgraph.cpp | 2 +- src/algorithm/lazy_hgraph/lazy_hgraph.h | 2 +- src/algorithm/pyramid/pyramid.cpp | 4 +- src/algorithm/sindi/sindi.cpp | 2 +- src/algorithm/sindi/sindi.h | 4 +- src/attr/attr_value_map.cpp | 8 ++-- src/attr/attr_value_map.h | 2 +- src/attr/multi_bitset_manager.cpp | 4 +- src/attr/multi_bitset_manager.h | 2 +- .../attribute_bucket_inverted_datacell.cpp | 4 +- .../attribute_bucket_inverted_datacell.h | 2 +- src/datacell/attribute_inverted_interface.h | 2 +- src/datacell/bucket_datacell.h | 4 +- src/datacell/bucket_interface.h | 2 +- src/datacell/compressed_graph_datacell.cpp | 4 +- src/datacell/compressed_graph_datacell.h | 2 +- src/datacell/extra_info_datacell.h | 6 +-- src/datacell/extra_info_interface.h | 2 +- src/datacell/flatten_datacell.h | 6 +-- src/datacell/flatten_interface.h | 2 +- src/datacell/graph_datacell.h | 4 +- src/datacell/graph_interface.h | 2 +- src/datacell/multi_vector_datacell.h | 2 +- src/datacell/multi_vector_datacell.inl | 4 +- src/datacell/rabitq_split_datacell.h | 6 +-- src/datacell/sparse_graph_datacell.cpp | 4 +- src/datacell/sparse_graph_datacell.h | 2 +- src/datacell/sparse_term_datacell.cpp | 4 +- src/datacell/sparse_term_datacell.h | 2 +- src/datacell/sparse_vector_datacell.h | 2 +- src/datacell/sparse_vector_datacell.inl | 4 +- src/impl/bitset/computable_bitset.h | 2 +- src/impl/bitset/fast_bitset.cpp | 4 +- src/impl/bitset/fast_bitset.h | 2 +- src/impl/bitset/sparse_bitset.cpp | 4 +- src/impl/bitset/sparse_bitset.h | 2 +- src/impl/label_table/label_remap.h | 2 +- src/impl/label_table/label_table.h | 2 +- src/impl/reverse_edge.cpp | 6 +-- src/impl/reverse_edge.h | 2 +- src/impl/reverse_edge_test.cpp | 2 +- src/index/diskann.cpp | 6 +-- src/index/diskann.h | 6 +-- src/index/hnsw.h | 2 +- src/index/index_impl.h | 10 ++--- src/io/async_io/aio_context.h | 2 +- src/utils/lock_strategy.cpp | 4 +- src/utils/lock_strategy.h | 6 +-- src/utils/resource_object.h | 4 +- src/utils/visited_list.h | 2 +- src/vsag_ext.cpp | 2 +- tests/test_hgraph.cpp | 8 ++-- tests/test_random_index.cpp | 2 +- tests/test_simple_index.cpp | 4 +- tools/eval/case/build_eval_case.cpp | 11 +++--- tools/eval/case/search_eval_case.cpp | 11 +++--- 73 files changed, 218 insertions(+), 157 deletions(-) diff --git a/docs/docs/en/src/advanced/memory.md b/docs/docs/en/src/advanced/memory.md index 0229a9995d..7e377a775a 100644 --- a/docs/docs/en/src/advanced/memory.md +++ b/docs/docs/en/src/advanced/memory.md @@ -58,12 +58,27 @@ if (index->CheckFeature(vsag::SUPPORT_ESTIMATE_MEMORY)) { See `examples/cpp/308_feature_estimate_memory.cpp` for a full run. +### `EstimateBuildMemory(num_elements)` + +`Index::EstimateBuildMemory(num_elements)` returns the estimated memory (in bytes) required +**during the build process** for `num_elements` vectors. Unlike `EstimateMemory`, which +estimates the steady-state size of the final index, this accounts for temporary buffers and +intermediate data structures that exist only while `Build` is running. The peak memory during +build is typically higher than the post-build footprint: + +```cpp +uint64_t peak = index->EstimateBuildMemory(1000000); // bytes +``` + +Currently only DiskANN provides a non-trivial implementation; other index types throw +an exception by default. + ### `GetMemoryUsage()` `Index::GetMemoryUsage()` returns the **current** memory footprint of an index in bytes: ```cpp -int64_t bytes = index->GetMemoryUsage(); +uint64_t bytes = index->GetMemoryUsage(); ``` Properties: @@ -86,6 +101,26 @@ Properties: See `examples/cpp/319_feature_get_memory_usage.cpp` for a runnable example, including a helper that compares the interface value with the process resident size. +### `GetMemoryUsageDetail()` + +`Index::GetMemoryUsageDetail()` returns a breakdown of the **current** memory usage by +component: + +```cpp +std::unordered_map detail = index->GetMemoryUsageDetail(); +for (const auto& [component, bytes] : detail) { + std::cout << component << ": " << bytes << " bytes\n"; +} +``` + +The returned map keys are component names and values are memory in bytes. This is useful for +understanding *where* the memory is going inside an index. + +Currently only HGraph provides a meaningful implementation, returning components such as +`basic_flatten_codes`, `bottom_graph`, `route_graph`, `neighbors_mutex`, `pool`, +`label_table`, `high_precise_codes`, `extra_infos`, and `raw_vector`. SINDI returns an empty +map. Other index types throw an exception by default. + ### Capability Flags | Flag | Meaning | diff --git a/docs/docs/zh/src/advanced/memory.md b/docs/docs/zh/src/advanced/memory.md index 8472aeb8a8..57b1453c13 100644 --- a/docs/docs/zh/src/advanced/memory.md +++ b/docs/docs/zh/src/advanced/memory.md @@ -56,12 +56,24 @@ if (index->CheckFeature(vsag::SUPPORT_ESTIMATE_MEMORY)) { 完整示例：`examples/cpp/308_feature_estimate_memory.cpp`。 +### `EstimateBuildMemory(num_elements)` + +`Index::EstimateBuildMemory(num_elements)` 返回构建 `num_elements` 条向量的索引时**构建过程中** +所需的预估内存（字节数）。与 `EstimateMemory`（估算最终索引的稳态大小）不同，该接口考虑了构建 +过程中仅临时存在的缓冲区与中间数据结构。构建期间的峰值内存通常高于构建完成后的内存占用： + +```cpp +uint64_t peak = index->EstimateBuildMemory(1000000); // 字节 +``` + +目前仅 DiskANN 提供了有效实现，其他索引类型默认抛出异常。 + ### `GetMemoryUsage()` `Index::GetMemoryUsage()` 返回索引**当前**占用的字节数： ```cpp -int64_t bytes = index->GetMemoryUsage(); +uint64_t bytes = index->GetMemoryUsage(); ``` 特性： @@ -80,6 +92,23 @@ int64_t bytes = index->GetMemoryUsage(); 可运行示例：`examples/cpp/319_feature_get_memory_usage.cpp`，其中包含一个辅助函数将接口值与进程 驻留内存进行对照。 +### `GetMemoryUsageDetail()` + +`Index::GetMemoryUsageDetail()` 返回索引**当前**内存占用按组件的细分： + +```cpp +std::unordered_map detail = index->GetMemoryUsageDetail(); +for (const auto& [component, bytes] : detail) { + std::cout << component << ": " << bytes << " bytes\n"; +} +``` + +返回的 map 的 key 为组件名，value 为对应内存字节数。该接口有助于了解索引内部的内存分布。 + +目前仅 HGraph 提供了有效实现，返回的组件包括 `basic_flatten_codes`、`bottom_graph`、 +`route_graph`、`neighbors_mutex`、`pool`、`label_table`、`high_precise_codes`、 +`extra_infos` 和 `raw_vector`。SINDI 返回空 map，其他索引类型默认抛出异常。 + ### 能力标志 | 标志 | 含义 | diff --git a/include/vsag/index.h b/include/vsag/index.h index 3c364efec8..8e8c638c49 100644 --- a/include/vsag/index.h +++ b/include/vsag/index.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -900,18 +901,15 @@ class Index { * * @return number of bytes occupied by the index. */ - [[nodiscard]] virtual int64_t + [[nodiscard]] virtual uint64_t GetMemoryUsage() const = 0; /** * @brief Return the memory usage of every component in the index * - * @return a json object that contains the memory usage of every component in the index + * @return a map from component name to memory usage in bytes */ - // TODO(deming): implement func for every types of index - // [[nodiscard]] virtual JsonType - // GetMemoryUsageDetail() const = 0; - [[nodiscard]] virtual std::string + [[nodiscard]] virtual std::unordered_map GetMemoryUsageDetail() const { throw std::runtime_error("Index does not support GetMemoryUsageDetail"); } @@ -933,9 +931,9 @@ class Index { * @param num_elements denotes the amount of data used to build the index. * @return estimated memory required during building. */ - [[nodiscard]] virtual int64_t - GetEstimateBuildMemory(const int64_t num_elements) const { - throw std::runtime_error("Index does not support estimate the memory while building"); + [[nodiscard]] virtual uint64_t + EstimateBuildMemory(uint64_t num_elements) const { + throw std::runtime_error("Index does not support EstimateBuildMemory"); } /** diff --git a/include/vsag/vsag_ext.h b/include/vsag/vsag_ext.h index e70cdecde3..510ff5c467 100644 --- a/include/vsag/vsag_ext.h +++ b/include/vsag/vsag_ext.h @@ -167,7 +167,7 @@ class IndexHandler { int64_t GetNumElements() const; - int64_t + uint64_t GetMemoryUsage() const; private: diff --git a/mockimpl/vsag/simpleflat.h b/mockimpl/vsag/simpleflat.h index 2be6c4f5ab..e12f639b93 100644 --- a/mockimpl/vsag/simpleflat.h +++ b/mockimpl/vsag/simpleflat.h @@ -76,7 +76,7 @@ class SimpleFlat : public Index { tl::expected Deserialize(const ReaderSet& reader_set) override; - int64_t + uint64_t GetMemoryUsage() const override { size_t ids_size = num_elements_ * sizeof(int64_t); size_t vector_size = num_elements_ * dim_ * sizeof(float); diff --git a/src/algorithm/bruteforce/bruteforce.cpp b/src/algorithm/bruteforce/bruteforce.cpp index d81a3b49ed..e0443d88e8 100644 --- a/src/algorithm/bruteforce/bruteforce.cpp +++ b/src/algorithm/bruteforce/bruteforce.cpp @@ -1044,9 +1044,9 @@ BruteForce::cal_memory_usage() { this->current_memory_usage_.store(memory_usage); } -int64_t +uint64_t BruteForce::GetMemoryUsage() const { - int64_t memory = 0; + uint64_t memory = 0; { std::shared_lock lock(this->memory_usage_mutex_); memory = this->current_memory_usage_.load(); diff --git a/src/algorithm/bruteforce/bruteforce.h b/src/algorithm/bruteforce/bruteforce.h index bf902fe43b..31800d881b 100644 --- a/src/algorithm/bruteforce/bruteforce.h +++ b/src/algorithm/bruteforce/bruteforce.h @@ -135,7 +135,7 @@ class BruteForce : public InnerIndexInterface { const AttributeSet& new_attrs, const AttributeSet& origin_attrs) override; - int64_t + uint64_t GetMemoryUsage() const override; private: diff --git a/src/algorithm/hgraph/hgraph.cpp b/src/algorithm/hgraph/hgraph.cpp index b3972483e5..c9513c6d59 100644 --- a/src/algorithm/hgraph/hgraph.cpp +++ b/src/algorithm/hgraph/hgraph.cpp @@ -670,7 +670,7 @@ HGraph::cal_memory_usage() { } std::unique_lock lock(this->memory_usage_mutex_); - this->current_memory_usage_.store(static_cast(memory)); + this->current_memory_usage_.store(memory); } } // namespace vsag diff --git a/src/algorithm/hgraph/hgraph.h b/src/algorithm/hgraph/hgraph.h index 1c59c3f200..e24a91e719 100644 --- a/src/algorithm/hgraph/hgraph.h +++ b/src/algorithm/hgraph/hgraph.h @@ -108,7 +108,7 @@ class HGraph : public InnerIndexInterface { void GetCodeByInnerId(InnerIdType inner_id, uint8_t* data) const override; - std::string + std::unordered_map GetMemoryUsageDetail() const override; std::pair diff --git a/src/algorithm/hgraph/hgraph_serialize.cpp b/src/algorithm/hgraph/hgraph_serialize.cpp index b4a49a2be1..aa552e6acd 100644 --- a/src/algorithm/hgraph/hgraph_serialize.cpp +++ b/src/algorithm/hgraph/hgraph_serialize.cpp @@ -396,28 +396,29 @@ HGraph::Deserialize(StreamReader& reader) { } } -std::string +std::unordered_map HGraph::GetMemoryUsageDetail() const { - JsonType memory_usage; - if (this->ignore_reorder_) { - this->use_reorder_ = false; + std::unordered_map memory_usage; + memory_usage["neighbors_mutex"] = this->neighbors_mutex_->GetMemoryUsage(); + memory_usage["pool"] = this->pool_->GetMemoryUsage(); + memory_usage["label_table"] = this->label_table_->GetMemoryUsage(); + memory_usage["basic_flatten_codes"] = this->basic_flatten_codes_->GetMemoryUsage(); + memory_usage["bottom_graph"] = this->bottom_graph_->GetMemoryUsage(); + uint64_t route_graph_memory = 0; + for (const auto& route_graph : this->route_graphs_) { + route_graph_memory += route_graph->GetMemoryUsage(); } - memory_usage["basic_flatten_codes"].SetUint64(this->basic_flatten_codes_->CalcSerializeSize()); - memory_usage["bottom_graph"].SetUint64(this->bottom_graph_->CalcSerializeSize()); + memory_usage["route_graph"] = route_graph_memory; if (this->has_precise_reorder()) { - memory_usage["high_precise_codes"].SetUint64( - this->high_precise_codes_->CalcSerializeSize()); + memory_usage["high_precise_codes"] = this->high_precise_codes_->GetMemoryUsage(); } - uint64_t route_graph_size = 0; - for (const auto& route_graph : this->route_graphs_) { - route_graph_size += route_graph->CalcSerializeSize(); - } - memory_usage["route_graph"].SetUint64(route_graph_size); if (this->extra_info_size_ > 0 && this->extra_infos_ != nullptr) { - memory_usage["extra_infos"].SetUint64(this->extra_infos_->CalcSerializeSize()); + memory_usage["extra_infos"] = this->extra_infos_->GetMemoryUsage(); + } + if (this->create_new_raw_vector_ && this->raw_vector_ != nullptr) { + memory_usage["raw_vector"] = this->raw_vector_->GetMemoryUsage(); } - memory_usage["__total_size__"].SetUint64(this->CalSerializeSize()); - return memory_usage.Dump(); + return memory_usage; } } // namespace vsag diff --git a/src/algorithm/inner_index_interface.h b/src/algorithm/inner_index_interface.h index a6f78f77dd..fc0fb5097d 100644 --- a/src/algorithm/inner_index_interface.h +++ b/src/algorithm/inner_index_interface.h @@ -17,6 +17,7 @@ #include #include +#include #include #include "container_types.h" @@ -253,10 +254,10 @@ class InnerIndexInterface { virtual DetailDataPtr GetDetailDataByName(const std::string& name, IndexDetailInfo& info) const; - [[nodiscard]] virtual int64_t - GetEstimateBuildMemory(const int64_t num_elements) const { + [[nodiscard]] virtual uint64_t + EstimateBuildMemory(uint64_t num_elements) const { throw VsagException(ErrorType::UNSUPPORTED_INDEX_OPERATION, - "Index doesn't support GetEstimateBuildMemory"); + "Index doesn't support EstimateBuildMemory"); } virtual void @@ -265,15 +266,14 @@ class InnerIndexInterface { [[nodiscard]] virtual IndexType GetIndexType() const = 0; - [[nodiscard]] virtual int64_t + [[nodiscard]] virtual uint64_t GetMemoryUsage() const { std::shared_lock lock(this->memory_usage_mutex_); return this->current_memory_usage_.load(); } - [[nodiscard]] virtual std::string + [[nodiscard]] virtual std::unordered_map GetMemoryUsageDetail() const { - // TODO(deming): implement func for every types of inner index throw VsagException(ErrorType::UNSUPPORTED_INDEX_OPERATION, "Index doesn't support GetMemoryUsageDetail"); } @@ -572,7 +572,7 @@ class InnerIndexInterface { protected: std::atomic total_count_{0}; - std::atomic current_memory_usage_{0}; + std::atomic current_memory_usage_{0}; mutable std::shared_mutex memory_usage_mutex_{}; bool has_raw_vector_{false}; diff --git a/src/algorithm/inner_index_interface_test.cpp b/src/algorithm/inner_index_interface_test.cpp index 6ce93c06c0..11e8f2c0b8 100644 --- a/src/algorithm/inner_index_interface_test.cpp +++ b/src/algorithm/inner_index_interface_test.cpp @@ -208,7 +208,7 @@ TEST_CASE("InnerIndexInterface NOT Implemented", "[ut][InnerIndexInterface]") { REQUIRE_THROWS(empty_index->Remove(0)); REQUIRE_THROWS(empty_index->GetNumberRemoved()); REQUIRE_THROWS(empty_index->EstimateMemory(1000)); - REQUIRE_THROWS(empty_index->GetEstimateBuildMemory(1000)); + REQUIRE_THROWS(empty_index->EstimateBuildMemory(1000)); REQUIRE_THROWS(empty_index->Feedback(nullptr, 10, "")); REQUIRE_THROWS(empty_index->GetStats()); REQUIRE_THROWS(empty_index->UpdateId(0, 1)); diff --git a/src/algorithm/ivf/ivf.cpp b/src/algorithm/ivf/ivf.cpp index 6d49a56f21..e9a31e3dcc 100644 --- a/src/algorithm/ivf/ivf.cpp +++ b/src/algorithm/ivf/ivf.cpp @@ -1307,12 +1307,12 @@ IVF::cal_memory_usage() { memory += location_map_.size() * sizeof(uint64_t); memory += partition_strategy_->GetMemoryUsage(); std::unique_lock lock(this->memory_usage_mutex_); - this->current_memory_usage_.store(static_cast(memory)); + this->current_memory_usage_.store(memory); } -int64_t +uint64_t IVF::GetMemoryUsage() const { - int64_t memory = 0; + uint64_t memory = 0; { std::shared_lock lock(this->memory_usage_mutex_); memory = this->current_memory_usage_.load(); diff --git a/src/algorithm/ivf/ivf.h b/src/algorithm/ivf/ivf.h index 31f15a1390..dfd3c1f3f9 100644 --- a/src/algorithm/ivf/ivf.h +++ b/src/algorithm/ivf/ivf.h @@ -169,7 +169,7 @@ class IVF : public InnerIndexInterface { const AttributeSet& new_attrs, const AttributeSet& origin_attrs) override; - [[nodiscard]] int64_t + [[nodiscard]] uint64_t GetMemoryUsage() const override; private: diff --git a/src/algorithm/ivf/ivf_nearest_partition.cpp b/src/algorithm/ivf/ivf_nearest_partition.cpp index a1d6f70a37..46985dd584 100644 --- a/src/algorithm/ivf/ivf_nearest_partition.cpp +++ b/src/algorithm/ivf/ivf_nearest_partition.cpp @@ -172,9 +172,9 @@ IVFNearestPartition::GetCentroid(BucketIdType bucket_id, Vector& centroid this->route_index_ptr_->GetCodeByInnerId(bucket_id, (uint8_t*)centroid.data()); } -[[nodiscard]] int64_t +[[nodiscard]] uint64_t IVFNearestPartition::GetMemoryUsage() const { - return static_cast(sizeof(IVFNearestPartition) + - this->route_index_ptr_->GetMemoryUsage()); + return static_cast(sizeof(IVFNearestPartition) + + this->route_index_ptr_->GetMemoryUsage()); } } // namespace vsag diff --git a/src/algorithm/ivf/ivf_nearest_partition.h b/src/algorithm/ivf/ivf_nearest_partition.h index 0830276ec0..d5d4b45db0 100644 --- a/src/algorithm/ivf/ivf_nearest_partition.h +++ b/src/algorithm/ivf/ivf_nearest_partition.h @@ -47,7 +47,7 @@ class IVFNearestPartition : public IVFPartitionStrategy { void Deserialize(lvalue_or_rvalue reader) override; - [[nodiscard]] int64_t + [[nodiscard]] uint64_t GetMemoryUsage() const override; public: diff --git a/src/algorithm/ivf/ivf_partition_strategy.h b/src/algorithm/ivf/ivf_partition_strategy.h index a5b1cb41d9..d3bda2aa66 100644 --- a/src/algorithm/ivf/ivf_partition_strategy.h +++ b/src/algorithm/ivf/ivf_partition_strategy.h @@ -90,7 +90,7 @@ class IVFPartitionStrategy { GetResidual( uint64_t n, const float* x, float* residuals, float* centroids, BucketIdType* assign); - virtual int64_t + virtual uint64_t GetMemoryUsage() const { return 0; } diff --git a/src/algorithm/lazy_hgraph/lazy_hgraph.cpp b/src/algorithm/lazy_hgraph/lazy_hgraph.cpp index e6272e6705..8f077999ea 100644 --- a/src/algorithm/lazy_hgraph/lazy_hgraph.cpp +++ b/src/algorithm/lazy_hgraph/lazy_hgraph.cpp @@ -325,7 +325,7 @@ LazyHGraph::GetExtraInfoByIds(const int64_t* ids, int64_t count, char* extra_inf ActiveIndex()->GetExtraInfoByIds(ids, count, extra_infos); } -int64_t +uint64_t LazyHGraph::GetMemoryUsage() const { std::shared_lock lock(this->phase_mutex_); return ActiveIndex()->GetMemoryUsage(); diff --git a/src/algorithm/lazy_hgraph/lazy_hgraph.h b/src/algorithm/lazy_hgraph/lazy_hgraph.h index fb88ced6df..b3f21654f2 100644 --- a/src/algorithm/lazy_hgraph/lazy_hgraph.h +++ b/src/algorithm/lazy_hgraph/lazy_hgraph.h @@ -112,7 +112,7 @@ class LazyHGraph : public InnerIndexInterface { void GetExtraInfoByIds(const int64_t* ids, int64_t count, char* extra_infos) const override; - [[nodiscard]] int64_t + [[nodiscard]] uint64_t GetMemoryUsage() const override; void diff --git a/src/algorithm/pyramid/pyramid.cpp b/src/algorithm/pyramid/pyramid.cpp index 106244be02..e784d09d2a 100644 --- a/src/algorithm/pyramid/pyramid.cpp +++ b/src/algorithm/pyramid/pyramid.cpp @@ -498,7 +498,7 @@ Pyramid::Deserialize(StreamReader& reader) { } resize(max_capacity); - this->current_memory_usage_ = static_cast(this->CalSerializeSize()); + this->current_memory_usage_ = this->CalSerializeSize(); } InnerIndexPtr @@ -516,7 +516,7 @@ Pyramid::ExportModel(const IndexCommonParam& param) const { } this->precise_codes_->ExportModel(index->precise_codes_); } - index->current_memory_usage_ = static_cast(index->CalSerializeSize()); + index->current_memory_usage_ = index->CalSerializeSize(); return index; } diff --git a/src/algorithm/sindi/sindi.cpp b/src/algorithm/sindi/sindi.cpp index 7a91cf4831..2119c60df9 100644 --- a/src/algorithm/sindi/sindi.cpp +++ b/src/algorithm/sindi/sindi.cpp @@ -1016,7 +1016,7 @@ SINDI::cal_memory_usage() { } std::unique_lock lock(this->memory_usage_mutex_); - this->current_memory_usage_.store(static_cast(memory)); + this->current_memory_usage_.store(memory); } void diff --git a/src/algorithm/sindi/sindi.h b/src/algorithm/sindi/sindi.h index 74f3a591e3..a686f89ebf 100644 --- a/src/algorithm/sindi/sindi.h +++ b/src/algorithm/sindi/sindi.h @@ -91,9 +91,9 @@ class SINDI : public InnerIndexInterface { void InitFeatures() override; - std::string + std::unordered_map GetMemoryUsageDetail() const override { - return ""; + return {}; } std::string diff --git a/src/attr/attr_value_map.cpp b/src/attr/attr_value_map.cpp index 4ebb01fc89..50dbfd404e 100644 --- a/src/attr/attr_value_map.cpp +++ b/src/attr/attr_value_map.cpp @@ -117,18 +117,18 @@ AttrValueMap::Deserialize(StreamReader& reader) { } template -int64_t +uint64_t get_memory_usage(const UnorderedMap& map) { - int64_t memory_usage = 0; + uint64_t memory_usage = 0; for (const auto& [key, value] : map) { memory_usage += sizeof(T) + value->GetMemoryUsage(); } return memory_usage; } -int64_t +uint64_t AttrValueMap::GetMemoryUsage() const { - int64_t memory_usage = sizeof(AttrValueMap); + uint64_t memory_usage = sizeof(AttrValueMap); memory_usage += get_memory_usage(int64_to_bitset_); memory_usage += get_memory_usage(int32_to_bitset_); memory_usage += get_memory_usage(int16_to_bitset_); diff --git a/src/attr/attr_value_map.h b/src/attr/attr_value_map.h index 54a9584b1b..511b4cda72 100644 --- a/src/attr/attr_value_map.h +++ b/src/attr/attr_value_map.h @@ -118,7 +118,7 @@ class AttrValueMap { void Deserialize(StreamReader& reader); - int64_t + uint64_t GetMemoryUsage() const; private: diff --git a/src/attr/multi_bitset_manager.cpp b/src/attr/multi_bitset_manager.cpp index b11e605c6e..bb21c3a326 100644 --- a/src/attr/multi_bitset_manager.cpp +++ b/src/attr/multi_bitset_manager.cpp @@ -105,7 +105,7 @@ MultiBitsetManager::Deserialize(lvalue_or_rvalue reader) { } } -int64_t +uint64_t MultiBitsetManager::GetMemoryUsage() const { auto memory_usage = sizeof(MultiBitsetManager); memory_usage += bitsets_.size() * sizeof(std::nullptr_t); @@ -113,7 +113,7 @@ MultiBitsetManager::GetMemoryUsage() const { for (auto* bitset : bitsets_) { memory_usage += bitset->GetMemoryUsage(); } - return static_cast(memory_usage); + return static_cast(memory_usage); } } // namespace vsag diff --git a/src/attr/multi_bitset_manager.h b/src/attr/multi_bitset_manager.h index b341b635d8..a6cf1b2891 100644 --- a/src/attr/multi_bitset_manager.h +++ b/src/attr/multi_bitset_manager.h @@ -122,7 +122,7 @@ class MultiBitsetManager { * * @return The current memory usage in bytes. */ - int64_t + uint64_t GetMemoryUsage() const; private: diff --git a/src/datacell/attribute_bucket_inverted_datacell.cpp b/src/datacell/attribute_bucket_inverted_datacell.cpp index ad8492069f..a1b1c8fa20 100644 --- a/src/datacell/attribute_bucket_inverted_datacell.cpp +++ b/src/datacell/attribute_bucket_inverted_datacell.cpp @@ -308,7 +308,7 @@ AttributeBucketInvertedDataCell::GetAttribute(BucketIdType bucket_id, } } -int64_t +uint64_t AttributeBucketInvertedDataCell::GetMemoryUsage() const { auto memory_usage = sizeof(AttributeBucketInvertedDataCell); @@ -316,7 +316,7 @@ AttributeBucketInvertedDataCell::GetMemoryUsage() const { memory_usage += value_map->GetMemoryUsage(); memory_usage += name.size() + sizeof(ValueMapPtr); } - return static_cast(memory_usage); + return static_cast(memory_usage); } } // namespace vsag diff --git a/src/datacell/attribute_bucket_inverted_datacell.h b/src/datacell/attribute_bucket_inverted_datacell.h index 5e4bc36603..971dc733d2 100644 --- a/src/datacell/attribute_bucket_inverted_datacell.h +++ b/src/datacell/attribute_bucket_inverted_datacell.h @@ -58,7 +58,7 @@ class AttributeBucketInvertedDataCell : public AttributeInvertedInterface { void GetAttribute(BucketIdType bucket_id, InnerIdType inner_id, AttributeSet* attr) override; - int64_t + uint64_t GetMemoryUsage() const override; private: diff --git a/src/datacell/attribute_inverted_interface.h b/src/datacell/attribute_inverted_interface.h index 74488775c2..63d93544f8 100644 --- a/src/datacell/attribute_inverted_interface.h +++ b/src/datacell/attribute_inverted_interface.h @@ -88,7 +88,7 @@ class AttributeInvertedInterface { return this->bitset_type_; } - virtual int64_t + virtual uint64_t GetMemoryUsage() const = 0; public: diff --git a/src/datacell/bucket_datacell.h b/src/datacell/bucket_datacell.h index 4e671f1557..b393d47998 100644 --- a/src/datacell/bucket_datacell.h +++ b/src/datacell/bucket_datacell.h @@ -117,9 +117,9 @@ class BucketDataCell : public BucketInterface { void GetCodesById(BucketIdType bucket_id, InnerIdType offset_id, uint8_t* data) const override; - [[nodiscard]] int64_t + [[nodiscard]] uint64_t GetMemoryUsage() const override { - int64_t memory = sizeof(BucketDataCell); + uint64_t memory = sizeof(BucketDataCell); for (BucketIdType bucket_id = 0; bucket_id < this->bucket_count_; bucket_id++) { memory += this->datas_[bucket_id].GetMemoryUsage(); memory += this->inner_ids_[bucket_id].size() * sizeof(InnerIdType); diff --git a/src/datacell/bucket_interface.h b/src/datacell/bucket_interface.h index 6b54f934b6..c313a8e39f 100644 --- a/src/datacell/bucket_interface.h +++ b/src/datacell/bucket_interface.h @@ -119,7 +119,7 @@ class BucketInterface { virtual void Unpack(){}; - [[nodiscard]] virtual int64_t + [[nodiscard]] virtual uint64_t GetMemoryUsage() const = 0; public: diff --git a/src/datacell/compressed_graph_datacell.cpp b/src/datacell/compressed_graph_datacell.cpp index 0f1793fdf5..fc10b64fc6 100644 --- a/src/datacell/compressed_graph_datacell.cpp +++ b/src/datacell/compressed_graph_datacell.cpp @@ -158,7 +158,7 @@ CompressedGraphDataCell::CheckIdExists(InnerIdType id) const { return id < neighbor_sets_.size() && neighbor_sets_[id] != nullptr; } -int64_t +uint64_t CompressedGraphDataCell::GetMemoryUsage() const { auto memory = sizeof(CompressedGraphDataCell); memory += neighbor_sets_.size() * sizeof(std::nullptr_t); @@ -167,7 +167,7 @@ CompressedGraphDataCell::GetMemoryUsage() const { memory += encoder->SizeInBytes(); } } - return static_cast(memory); + return static_cast(memory); } Vector diff --git a/src/datacell/compressed_graph_datacell.h b/src/datacell/compressed_graph_datacell.h index 7e96708187..51c40dcc0f 100644 --- a/src/datacell/compressed_graph_datacell.h +++ b/src/datacell/compressed_graph_datacell.h @@ -60,7 +60,7 @@ class CompressedGraphDataCell : public GraphInterface { void Deserialize(StreamReader& reader) override; - int64_t + uint64_t GetMemoryUsage() const override; Vector diff --git a/src/datacell/extra_info_datacell.h b/src/datacell/extra_info_datacell.h index afafdff083..e387468468 100644 --- a/src/datacell/extra_info_datacell.h +++ b/src/datacell/extra_info_datacell.h @@ -80,7 +80,7 @@ class ExtraInfoDataCell : public ExtraInfoInterface { void Deserialize(StreamReader& reader) override; - int64_t + uint64_t GetMemoryUsage() const override; void @@ -185,9 +185,9 @@ ExtraInfoDataCell::Deserialize(StreamReader& reader) { } template -int64_t +uint64_t ExtraInfoDataCell::GetMemoryUsage() const { - int64_t memory = sizeof(ExtraInfoDataCell); + uint64_t memory = sizeof(ExtraInfoDataCell); if (IOTmpl::InMemory) { memory += this->io_->GetMemoryUsage(); } diff --git a/src/datacell/extra_info_interface.h b/src/datacell/extra_info_interface.h index 8fd8a16e87..9441d56f3e 100644 --- a/src/datacell/extra_info_interface.h +++ b/src/datacell/extra_info_interface.h @@ -68,7 +68,7 @@ class ExtraInfoInterface { virtual bool GetExtraInfoById(InnerIdType id, char* extra_info) const = 0; - virtual int64_t + virtual uint64_t GetMemoryUsage() const { return 0; } diff --git a/src/datacell/flatten_datacell.h b/src/datacell/flatten_datacell.h index 3a7ac1750c..b86434eb56 100644 --- a/src/datacell/flatten_datacell.h +++ b/src/datacell/flatten_datacell.h @@ -176,7 +176,7 @@ class FlattenDataCell : public FlattenInterface { return common_param_; } - int64_t + uint64_t GetMemoryUsage() const override; public: @@ -502,9 +502,9 @@ FlattenDataCell::MergeOther(const FlattenInterfacePtr& other, } template -int64_t +uint64_t FlattenDataCell::GetMemoryUsage() const { - int64_t memory = sizeof(FlattenDataCell); + uint64_t memory = sizeof(FlattenDataCell); if (IOTmpl::InMemory) { memory += this->io_->GetMemoryUsage(); } diff --git a/src/datacell/flatten_interface.h b/src/datacell/flatten_interface.h index af77dceb83..6cf2a169ba 100644 --- a/src/datacell/flatten_interface.h +++ b/src/datacell/flatten_interface.h @@ -145,7 +145,7 @@ class FlattenInterface { throw VsagException(ErrorType::INTERNAL_ERROR, "InitIO not implemented in FlattenInterface"); } - virtual int64_t + virtual uint64_t GetMemoryUsage() const { return 0; } diff --git a/src/datacell/graph_datacell.h b/src/datacell/graph_datacell.h index 9d7c3584bf..5591a51f3b 100644 --- a/src/datacell/graph_datacell.h +++ b/src/datacell/graph_datacell.h @@ -107,9 +107,9 @@ class GraphDataCell : public GraphInterface { void MergeOther(GraphInterfacePtr other, uint64_t bias) override; - int64_t + uint64_t GetMemoryUsage() const override { - int64_t memory = sizeof(GraphDataCell) + node_versions_.size() * sizeof(uint8_t); + uint64_t memory = sizeof(GraphDataCell) + node_versions_.size() * sizeof(uint8_t); if (IOTmpl::InMemory) { memory += io_->GetMemoryUsage(); } diff --git a/src/datacell/graph_interface.h b/src/datacell/graph_interface.h index 0c5aae2e28..a5ee628b41 100644 --- a/src/datacell/graph_interface.h +++ b/src/datacell/graph_interface.h @@ -88,7 +88,7 @@ class GraphInterface { "GetIds in GraphInterface is not implemented"); } - virtual int64_t + virtual uint64_t GetMemoryUsage() const { return 0; } diff --git a/src/datacell/multi_vector_datacell.h b/src/datacell/multi_vector_datacell.h index b491cb0c50..189b386886 100644 --- a/src/datacell/multi_vector_datacell.h +++ b/src/datacell/multi_vector_datacell.h @@ -109,7 +109,7 @@ class MultiVectorDataCell : public FlattenInterface { void Deserialize(lvalue_or_rvalue reader) override; - int64_t + uint64_t GetMemoryUsage() const override; private: diff --git a/src/datacell/multi_vector_datacell.inl b/src/datacell/multi_vector_datacell.inl index 1ab1b749ec..2ea98cbe06 100644 --- a/src/datacell/multi_vector_datacell.inl +++ b/src/datacell/multi_vector_datacell.inl @@ -218,9 +218,9 @@ MultiVectorDataCell::Query(float* result_dists, } template -int64_t +uint64_t MultiVectorDataCell::GetMemoryUsage() const { - int64_t memory = sizeof(MultiVectorDataCell); + uint64_t memory = sizeof(MultiVectorDataCell); memory += this->offset_io_->size_; if (IOTmpl::InMemory) { memory += this->io_->GetMemoryUsage(); diff --git a/src/datacell/rabitq_split_datacell.h b/src/datacell/rabitq_split_datacell.h index 380001de43..8c83187bcb 100644 --- a/src/datacell/rabitq_split_datacell.h +++ b/src/datacell/rabitq_split_datacell.h @@ -117,7 +117,7 @@ class RaBitQSplitCodeStorage { io_->Deserialize(reader); } - [[nodiscard]] int64_t + [[nodiscard]] uint64_t GetMemoryUsage() const { if constexpr (IOTmpl::InMemory) { return io_->GetMemoryUsage(); @@ -655,9 +655,9 @@ class RaBitQSplitDataCell : public FlattenInterface { this->max_capacity_ = capacity; } - int64_t + uint64_t GetMemoryUsage() const override { - int64_t memory = sizeof(RaBitQSplitDataCell); + uint64_t memory = sizeof(RaBitQSplitDataCell); memory += this->x_bit_cell_->GetMemoryUsage(); memory += this->supplement_cell_->GetMemoryUsage(); memory += sizeof(RaBitQuantizer); diff --git a/src/datacell/sparse_graph_datacell.cpp b/src/datacell/sparse_graph_datacell.cpp index 569f5f89f0..61c7103248 100644 --- a/src/datacell/sparse_graph_datacell.cpp +++ b/src/datacell/sparse_graph_datacell.cpp @@ -267,7 +267,7 @@ SparseGraphDataCell::GetIds() const { return ids; } -int64_t +uint64_t SparseGraphDataCell::GetMemoryUsage() const { auto memory = sizeof(SparseGraphDataCell); memory += neighbors_.size() * (sizeof(InnerIdType) + maximum_degree_ * sizeof(InnerIdType) + @@ -276,7 +276,7 @@ SparseGraphDataCell::GetMemoryUsage() const { if (reverse_edges_) { memory += reverse_edges_->GetMemoryUsage(); } - return static_cast(memory); + return static_cast(memory); } void diff --git a/src/datacell/sparse_graph_datacell.h b/src/datacell/sparse_graph_datacell.h index 10875a5c3b..1bfdf36c7b 100644 --- a/src/datacell/sparse_graph_datacell.h +++ b/src/datacell/sparse_graph_datacell.h @@ -77,7 +77,7 @@ class SparseGraphDataCell : public GraphInterface { Vector GetIds() const override; - int64_t + uint64_t GetMemoryUsage() const override; DuplicateTrackerPtr diff --git a/src/datacell/sparse_term_datacell.cpp b/src/datacell/sparse_term_datacell.cpp index 2671427961..bbe36c50a5 100644 --- a/src/datacell/sparse_term_datacell.cpp +++ b/src/datacell/sparse_term_datacell.cpp @@ -394,7 +394,7 @@ SparseTermDataCell::CalcDistanceByInnerId(const SparseTermComputerPtr& computer, return 1 + ip; } -int64_t +uint64_t SparseTermDataCell::GetMemoryUsage() const { auto memory = sizeof(SparseTermDataCell); memory += term_ids_.capacity() * sizeof(std::unique_ptr>); @@ -413,7 +413,7 @@ SparseTermDataCell::GetMemoryUsage() const { } memory += sizeof(QuantizationParams); memory += term_sizes_.capacity() * sizeof(uint32_t); - return static_cast(memory); + return static_cast(memory); } void diff --git a/src/datacell/sparse_term_datacell.h b/src/datacell/sparse_term_datacell.h index b5a0ebaeda..bc98922f28 100644 --- a/src/datacell/sparse_term_datacell.h +++ b/src/datacell/sparse_term_datacell.h @@ -115,7 +115,7 @@ class SparseTermDataCell { void GetSparseVector(uint32_t base_id, SparseVector* data, Allocator* specified_allocator); - [[nodiscard]] int64_t + [[nodiscard]] uint64_t GetMemoryUsage() const; private: diff --git a/src/datacell/sparse_vector_datacell.h b/src/datacell/sparse_vector_datacell.h index 5fe4c49bbd..19ccce0d2c 100644 --- a/src/datacell/sparse_vector_datacell.h +++ b/src/datacell/sparse_vector_datacell.h @@ -144,7 +144,7 @@ class SparseVectorDataCell : public FlattenInterface { this->io_ = io; } - int64_t + uint64_t GetMemoryUsage() const override; private: diff --git a/src/datacell/sparse_vector_datacell.inl b/src/datacell/sparse_vector_datacell.inl index 0a95f3687e..0e04d94f12 100644 --- a/src/datacell/sparse_vector_datacell.inl +++ b/src/datacell/sparse_vector_datacell.inl @@ -289,9 +289,9 @@ SparseVectorDataCell::SparseVectorDataCell( } template -int64_t +uint64_t SparseVectorDataCell::GetMemoryUsage() const { - int64_t memory = sizeof(SparseVectorDataCell); + uint64_t memory = sizeof(SparseVectorDataCell); memory += this->offset_io_->size_; if (IOTmpl::InMemory) { memory += this->io_->GetMemoryUsage(); diff --git a/src/impl/bitset/computable_bitset.h b/src/impl/bitset/computable_bitset.h index 4afacb992e..ffbdb618d5 100644 --- a/src/impl/bitset/computable_bitset.h +++ b/src/impl/bitset/computable_bitset.h @@ -139,7 +139,7 @@ class ComputableBitset : public Bitset { * * @return The current memory usage in bytes. */ - virtual int64_t + virtual uint64_t GetMemoryUsage() const = 0; }; diff --git a/src/impl/bitset/fast_bitset.cpp b/src/impl/bitset/fast_bitset.cpp index af5641c510..67090e5761 100644 --- a/src/impl/bitset/fast_bitset.cpp +++ b/src/impl/bitset/fast_bitset.cpp @@ -272,9 +272,9 @@ FastBitset::resize(uint32_t new_size, uint64_t fill) { size_ = new_size; } -int64_t +uint64_t FastBitset::GetMemoryUsage() const { - return static_cast(sizeof(FastBitset) + this->size_ * sizeof(uint64_t)); + return static_cast(sizeof(FastBitset) + this->size_ * sizeof(uint64_t)); } } // namespace vsag diff --git a/src/impl/bitset/fast_bitset.h b/src/impl/bitset/fast_bitset.h index e9d4814210..ec789e6c0a 100644 --- a/src/impl/bitset/fast_bitset.h +++ b/src/impl/bitset/fast_bitset.h @@ -77,7 +77,7 @@ class FastBitset : public ComputableBitset { std::string Dump() override; - int64_t + uint64_t GetMemoryUsage() const override; private: diff --git a/src/impl/bitset/sparse_bitset.cpp b/src/impl/bitset/sparse_bitset.cpp index 3104c0f4db..acccbc9c5e 100644 --- a/src/impl/bitset/sparse_bitset.cpp +++ b/src/impl/bitset/sparse_bitset.cpp @@ -115,9 +115,9 @@ SparseBitset::Clear() { this->r_ = std::move(roaring::Roaring()); } -int64_t +uint64_t SparseBitset::GetMemoryUsage() const { - return static_cast(sizeof(SparseBitset) + r_.getSizeInBytes()); + return static_cast(sizeof(SparseBitset) + r_.getSizeInBytes()); } } // namespace vsag diff --git a/src/impl/bitset/sparse_bitset.h b/src/impl/bitset/sparse_bitset.h index a5ec180972..544e09b68f 100644 --- a/src/impl/bitset/sparse_bitset.h +++ b/src/impl/bitset/sparse_bitset.h @@ -76,7 +76,7 @@ class SparseBitset : public ComputableBitset { void Clear() override; - int64_t + uint64_t GetMemoryUsage() const override; private: diff --git a/src/impl/label_table/label_remap.h b/src/impl/label_table/label_remap.h index 48978c038c..10f59d90f5 100644 --- a/src/impl/label_table/label_remap.h +++ b/src/impl/label_table/label_remap.h @@ -126,7 +126,7 @@ class LabelRemap { * @note This includes hash table overhead (buckets, pointers, load factor) * The estimate uses a multiplier to account for container overhead */ - int64_t + uint64_t GetMemoryUsage() const { return Size() * (sizeof(LabelType) + sizeof(InnerIdType)) * 2; } diff --git a/src/impl/label_table/label_table.h b/src/impl/label_table/label_table.h index 9712550a13..819fb97e7b 100644 --- a/src/impl/label_table/label_table.h +++ b/src/impl/label_table/label_table.h @@ -224,7 +224,7 @@ class LabelTable { * Get memory usage of the label table. * @return The memory usage in bytes. */ - int64_t + uint64_t GetMemoryUsage() { return sizeof(LabelTable) + label_table_.size() * sizeof(LabelType) + label_remap_.GetMemoryUsage() + deleted_ids_.size() * sizeof(InnerIdType) * 2; diff --git a/src/impl/reverse_edge.cpp b/src/impl/reverse_edge.cpp index 94fce3b71d..b9a91671f3 100644 --- a/src/impl/reverse_edge.cpp +++ b/src/impl/reverse_edge.cpp @@ -72,15 +72,15 @@ ReverseEdge::Clear() { reverse_edges_.clear(); } -int64_t +uint64_t ReverseEdge::GetMemoryUsage() const { std::shared_lock rlock(mutex_); - int64_t usage = sizeof(ReverseEdge); + uint64_t usage = sizeof(ReverseEdge); for (const auto& [id, neighbors] : reverse_edges_) { usage += sizeof(id); usage += sizeof(std::unique_ptr>); if (neighbors) { - usage += static_cast(neighbors->capacity() * sizeof(InnerIdType)); + usage += neighbors->capacity() * sizeof(InnerIdType); } } return usage; diff --git a/src/impl/reverse_edge.h b/src/impl/reverse_edge.h index 4426a6afb1..11d9bbfd5a 100644 --- a/src/impl/reverse_edge.h +++ b/src/impl/reverse_edge.h @@ -45,7 +45,7 @@ class ReverseEdge { void Resize(InnerIdType new_size); - int64_t + uint64_t GetMemoryUsage() const; private: diff --git a/src/impl/reverse_edge_test.cpp b/src/impl/reverse_edge_test.cpp index 049fcc6c11..5882bb7a61 100644 --- a/src/impl/reverse_edge_test.cpp +++ b/src/impl/reverse_edge_test.cpp @@ -139,7 +139,7 @@ TEST_CASE("ReverseEdge Basic Operations", "[ut][ReverseEdge]") { reverse_edge.AddReverseEdge(1, 2); reverse_edge.AddReverseEdge(3, 2); - int64_t usage = reverse_edge.GetMemoryUsage(); + uint64_t usage = reverse_edge.GetMemoryUsage(); REQUIRE(usage > 0); REQUIRE(usage > sizeof(ReverseEdge)); } diff --git a/src/index/diskann.cpp b/src/index/diskann.cpp index 82f0905ef7..50120d2dda 100644 --- a/src/index/diskann.cpp +++ b/src/index/diskann.cpp @@ -1100,9 +1100,9 @@ DiskANN::GetStats() const { return j.Dump(4); } -int64_t -DiskANN::GetEstimateBuildMemory(const int64_t num_elements) const { - int64_t estimate_memory_usage = 0; +uint64_t +DiskANN::EstimateBuildMemory(uint64_t num_elements) const { + uint64_t estimate_memory_usage = 0; // Memory usage of graph (1.365 is the relaxation factor used by DiskANN during graph construction.) estimate_memory_usage += (num_elements * R_ * sizeof(uint32_t) // NOLINT + num_elements * (R_ + 1) * sizeof(uint32_t)) * diff --git a/src/index/diskann.h b/src/index/diskann.h index 51dcde92d1..bc94346fc8 100644 --- a/src/index/diskann.h +++ b/src/index/diskann.h @@ -174,7 +174,7 @@ class DiskANN : public Index { return index_->get_data_num(); } - int64_t + uint64_t GetMemoryUsage() const override { if (status_ == MEMORY) { return index_->get_memory_usage() + disk_pq_compressed_vectors_.str().size() + @@ -186,8 +186,8 @@ class DiskANN : public Index { return 0; } - int64_t - GetEstimateBuildMemory(const int64_t num_elements) const override; + uint64_t + EstimateBuildMemory(uint64_t num_elements) const override; std::string GetStats() const override; diff --git a/src/index/hnsw.h b/src/index/hnsw.h index 2a318b9210..2ed7cd4d5c 100644 --- a/src/index/hnsw.h +++ b/src/index/hnsw.h @@ -308,7 +308,7 @@ class HNSW : public Index { return this->get_num_elements(); } - int64_t + uint64_t GetMemoryUsage() const override { return this->get_memory_usage(); } diff --git a/src/index/index_impl.h b/src/index/index_impl.h index c3b4c7ec9d..b546ef95b9 100644 --- a/src/index/index_impl.h +++ b/src/index/index_impl.h @@ -232,9 +232,9 @@ class IndexImpl : public Index { SAFE_CALL(return this->inner_index_->GetDetailDataByName(name, info)); } - [[nodiscard]] int64_t - GetEstimateBuildMemory(const int64_t num_elements) const override { - return this->inner_index_->GetEstimateBuildMemory(num_elements); + [[nodiscard]] uint64_t + EstimateBuildMemory(uint64_t num_elements) const override { + return this->inner_index_->EstimateBuildMemory(num_elements); } virtual tl::expected @@ -247,12 +247,12 @@ class IndexImpl : public Index { return this->inner_index_->GetIndexType(); } - [[nodiscard]] int64_t + [[nodiscard]] uint64_t GetMemoryUsage() const override { return this->inner_index_->GetMemoryUsage(); } - [[nodiscard]] std::string + [[nodiscard]] std::unordered_map GetMemoryUsageDetail() const override { return this->inner_index_->GetMemoryUsageDetail(); } diff --git a/src/io/async_io/aio_context.h b/src/io/async_io/aio_context.h index a78c9fe67e..129224238b 100644 --- a/src/io/async_io/aio_context.h +++ b/src/io/async_io/aio_context.h @@ -68,7 +68,7 @@ class IOContext : public ResourceObject { * * @return Memory usage in bytes (sizeof(IOContext) plus heap-allocated iocb pointers). */ - int64_t + uint64_t GetMemoryUsage() const override { return sizeof(IOContext) + DEFAULT_REQUEST_COUNT * sizeof(struct iocb); } diff --git a/src/utils/lock_strategy.cpp b/src/utils/lock_strategy.cpp index 61687f5a3d..81fa691b61 100644 --- a/src/utils/lock_strategy.cpp +++ b/src/utils/lock_strategy.cpp @@ -57,9 +57,9 @@ PointsMutex::Resize(uint32_t new_element_num) { element_num_ = new_element_num; } -int64_t +uint64_t PointsMutex::GetMemoryUsage() { - return static_cast( + return static_cast( neighbors_mutex_.size() * (sizeof(std::shared_ptr) + sizeof(std::shared_mutex))); } diff --git a/src/utils/lock_strategy.h b/src/utils/lock_strategy.h index 5a33b2df0b..66f415306c 100644 --- a/src/utils/lock_strategy.h +++ b/src/utils/lock_strategy.h @@ -42,7 +42,7 @@ class MutexArray { virtual void Resize(uint32_t new_element_num) = 0; - virtual int64_t + virtual uint64_t GetMemoryUsage() = 0; }; @@ -65,7 +65,7 @@ class PointsMutex : public MutexArray { void Resize(uint32_t new_element_num) override; - int64_t + uint64_t GetMemoryUsage() override; private: @@ -96,7 +96,7 @@ class EmptyMutex : public MutexArray { Resize(uint32_t new_element_num) override { } - int64_t + uint64_t GetMemoryUsage() override { return 0; } diff --git a/src/utils/resource_object.h b/src/utils/resource_object.h index d9f5892dc0..10704d1227 100644 --- a/src/utils/resource_object.h +++ b/src/utils/resource_object.h @@ -53,9 +53,9 @@ class ResourceObject { * The memory usage should include all dynamically allocated memory, whether * directly or indirectly, used by the resource. * - * @return int64_t The memory usage of the resource object in bytes. + * @return uint64_t The memory usage of the resource object in bytes. */ - virtual int64_t + virtual uint64_t GetMemoryUsage() const = 0; }; diff --git a/src/utils/visited_list.h b/src/utils/visited_list.h index 40c0313ea7..84d787d5c2 100644 --- a/src/utils/visited_list.h +++ b/src/utils/visited_list.h @@ -51,7 +51,7 @@ class VisitedList : public ResourceObject { void Reset() override; - int64_t + uint64_t GetMemoryUsage() const override { return sizeof(VisitedList) + sizeof(VisitedListType) * this->max_size_; } diff --git a/src/vsag_ext.cpp b/src/vsag_ext.cpp index c68d639d22..dc815b87a2 100644 --- a/src/vsag_ext.cpp +++ b/src/vsag_ext.cpp @@ -184,7 +184,7 @@ IndexHandler::GetNumElements() const { return index_->GetNumElements(); } -int64_t +uint64_t IndexHandler::GetMemoryUsage() const { return index_->GetMemoryUsage(); } diff --git a/tests/test_hgraph.cpp b/tests/test_hgraph.cpp index df2490631d..ae4ebb7f2a 100644 --- a/tests/test_hgraph.cpp +++ b/tests/test_hgraph.cpp @@ -325,10 +325,10 @@ HGraphTestIndex::TestGeneral(const TestIndex::IndexPtr& index, void HGraphTestIndex::TestMemoryUsageDetail(const IndexPtr& index) { - auto memory_detail = vsag::JsonType::Parse(index->GetMemoryUsageDetail()); - REQUIRE(memory_detail.Contains("basic_flatten_codes")); - REQUIRE(memory_detail.Contains("bottom_graph")); - REQUIRE(memory_detail.Contains("route_graph")); + auto memory_detail = index->GetMemoryUsageDetail(); + REQUIRE(memory_detail.count("basic_flatten_codes") > 0); + REQUIRE(memory_detail.count("bottom_graph") > 0); + REQUIRE(memory_detail.count("route_graph") > 0); } } // namespace fixtures diff --git a/tests/test_random_index.cpp b/tests/test_random_index.cpp index d8d6a11ac0..6314b9aa76 100644 --- a/tests/test_random_index.cpp +++ b/tests/test_random_index.cpp @@ -136,5 +136,5 @@ TEST_CASE("Test Random Index", "[ft][random]") { REQUIRE(range_result.has_value()); } - REQUIRE(diskann->GetMemoryUsage() < diskann->GetEstimateBuildMemory(max_elements) * 1.2); + REQUIRE(diskann->GetMemoryUsage() < diskann->EstimateBuildMemory(max_elements) * 6 / 5); } diff --git a/tests/test_simple_index.cpp b/tests/test_simple_index.cpp index 54aaa2bbfb..2be9b3df4d 100644 --- a/tests/test_simple_index.cpp +++ b/tests/test_simple_index.cpp @@ -90,7 +90,7 @@ class SimpleIndex : public Index { return 0; } - int64_t + uint64_t GetMemoryUsage() const override { return 0; } @@ -114,7 +114,7 @@ TEST_CASE("Test Simple Index", "[ft][simple_index]") { REQUIRE_FALSE(index->Remove(0).has_value()); REQUIRE_FALSE(index->CheckFeature(IndexFeature::SUPPORT_ESTIMATE_MEMORY)); REQUIRE_THROWS(index->EstimateMemory(1000)); - REQUIRE_THROWS(index->GetEstimateBuildMemory(1000)); + REQUIRE_THROWS(index->EstimateBuildMemory(1000)); REQUIRE_FALSE(index->Feedback(dataset->query_, 10, "").has_value()); REQUIRE_THROWS_MATCHES(index->GetStats(), std::runtime_error, diff --git a/tools/eval/case/build_eval_case.cpp b/tools/eval/case/build_eval_case.cpp index 7d842f810e..3eb243ec46 100644 --- a/tools/eval/case/build_eval_case.cpp +++ b/tools/eval/case/build_eval_case.cpp @@ -105,13 +105,12 @@ BuildEvalCase::process_result() { result["index_info"] = JsonType::parse(config_.build_param); result["action"] = "build"; result["index"] = config_.index_name; - // TODO(deming): remove try-catch after implement GetMemoryUsageDetail try { - result["memory_detail(B)"] = this->index_->GetMemoryUsageDetail(); - } catch (std::runtime_error& e) { - // if GetMemoryUsageDetail not implemented - logger_->Error(e.what()); - } catch (vsag::VsagException& e) { + auto detail = this->index_->GetMemoryUsageDetail(); + for (const auto& [name, size] : detail) { + result["memory_detail(B)"][name] = size; + } + } catch (const std::exception& e) { logger_->Error(e.what()); } return result; diff --git a/tools/eval/case/search_eval_case.cpp b/tools/eval/case/search_eval_case.cpp index 5bf111bba3..3a12c02708 100644 --- a/tools/eval/case/search_eval_case.cpp +++ b/tools/eval/case/search_eval_case.cpp @@ -262,13 +262,12 @@ SearchEvalCase::process_result() { result["index_info"] = JsonType::parse(config_.build_param); result["search_param"] = config_.search_param; result["index"] = config_.index_name; - // TODO(deming): remove try-catch after implement GetMemoryUsageDetail try { - result["memory_detail(B)"] = this->index_->GetMemoryUsageDetail(); - } catch (std::runtime_error& e) { - // if GetMemoryUsageDetail not implemented - logger_->Error(e.what()); - } catch (vsag::VsagException& e) { + auto detail = this->index_->GetMemoryUsageDetail(); + for (const auto& [name, size] : detail) { + result["memory_detail(B)"][name] = size; + } + } catch (const std::exception& e) { logger_->Error(e.what()); } EvalCase::MergeJsonType(this->basic_info_, result);