Skip to content

fix(hgraph): expose skip_ratio and skip_strategy search parameters (v0.18 backport) #2369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LHT129 wants to merge 1 commit into
base: 0.18
Choose a base branch
from
Open

fix(hgraph): expose skip_ratio and skip_strategy search parameters (v0.18 backport) #2369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/algorithm/hgraph.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -943,6 +943,8 @@ HGraph::KnnSearch(const DatasetPtr& query,
search_param.topk = static_cast<int64_t>(search_param.ef);
search_param.consider_duplicate = this->label_table_->CompressDuplicateData();
search_param.parallel_search_thread_count = params.parallel_search_thread_count;
search_param.skip_ratio = params.skip_ratio;
search_param.skip_strategy_type = params.skip_strategy_type;
Comment thread
LHT129 marked this conversation as resolved.

search_result = this->search_one_graph(query_data,
this->bottom_graph_,
Expand Down Expand Up @@ -1153,6 +1155,8 @@ HGraph::RangeSearch(const DatasetPtr& query,
search_param.consider_duplicate = true;
search_param.range_search_limit_size = static_cast<int>(limited_size);
search_param.parallel_search_thread_count = params.parallel_search_thread_count;
search_param.skip_ratio = params.skip_ratio;
search_param.skip_strategy_type = params.skip_strategy_type;

auto search_result = this->search_one_graph(raw_query,
this->bottom_graph_,
Expand Down Expand Up @@ -2197,6 +2201,8 @@ HGraph::SearchWithRequest(const SearchRequest& request) const {
stats.is_timeout.store(false, std::memory_order_relaxed);
}
search_param.parallel_search_thread_count = params.parallel_search_thread_count;
search_param.skip_ratio = params.skip_ratio;
search_param.skip_strategy_type = params.skip_strategy_type;

// hops_limit only takes effect when it's greater than ef_search
if (params.hops_limit <= static_cast<uint32_t>(params.ef_search)) {
Expand Down
12 changes: 12 additions & 0 deletions src/algorithm/hgraph_parameter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,18 @@ HGraphSearchParameters::FromJson(const std::string& json_string) {
params[INDEX_TYPE_HGRAPH][HGRAPH_USE_EXTRA_INFO_FILTER].GetBool();
}

if (params[INDEX_TYPE_HGRAPH].Contains(HNSW_PARAMETER_SKIP_RATIO)) {
obj.skip_ratio = params[INDEX_TYPE_HGRAPH][HNSW_PARAMETER_SKIP_RATIO].GetFloat();
CHECK_ARGUMENT((0.0F <= obj.skip_ratio) and (obj.skip_ratio <= 1.0F), // NOLINT
fmt::format("skip_ratio({}) must be in range [0.0, 1.0]", obj.skip_ratio));
}
Comment thread
LHT129 marked this conversation as resolved.
if (params[INDEX_TYPE_HGRAPH].Contains(HNSW_PARAMETER_SKIP_STRATEGY)) {
CHECK_ARGUMENT(
params[INDEX_TYPE_HGRAPH][HNSW_PARAMETER_SKIP_STRATEGY].IsString(),
fmt::format("parameters[{}] must be string type", HNSW_PARAMETER_SKIP_STRATEGY));
obj.skip_strategy_type = parse_filter_search_skip_strategy_type(
params[INDEX_TYPE_HGRAPH][HNSW_PARAMETER_SKIP_STRATEGY].GetString());
}
return obj;
}
} // namespace vsag
4 changes: 4 additions & 0 deletions src/algorithm/hgraph_parameter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "data_type.h"
#include "index_search_parameter.h"
#include "inner_index_parameter.h"
#include "utils/filter_search_skip_strategy.h"
#include "utils/pointer_define.h"
#include "vsag/constants.h"

Expand Down Expand Up @@ -81,6 +82,9 @@ class HGraphSearchParameters : public IndexSearchParameter {
uint32_t hops_limit{std::numeric_limits<uint32_t>::max()};
bool use_reorder{false};
bool use_extra_info_filter{false};
float skip_ratio{0.2F};
FilterSearchSkipStrategyType skip_strategy_type{
FilterSearchSkipStrategyType::DETERMINISTIC_ACCUMULATIVE};

private:
HGraphSearchParameters() = default;
Expand Down
29 changes: 29 additions & 0 deletions src/algorithm/hgraph_parameter_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,3 +228,32 @@ TEST_CASE("HGraph maps label_remap_type to inner index parameter", "[ut][HGraphP
REQUIRE(typed_param != nullptr);
REQUIRE(typed_param->label_remap_type == vsag::LabelRemapType::ROBIN);
}

TEST_CASE("HGraphSearchParameters parses skip_ratio and skip_strategy",
"[ut][HGraphSearchParameters][skip_ratio]") {
SECTION("default values") {
auto params = vsag::HGraphSearchParameters::FromJson(R"({"hgraph": {"ef_search": 32}})");
REQUIRE(params.skip_ratio == 0.2F);
REQUIRE(params.skip_strategy_type ==
vsag::FilterSearchSkipStrategyType::DETERMINISTIC_ACCUMULATIVE);
}

SECTION("custom skip_ratio") {
auto params = vsag::HGraphSearchParameters::FromJson(
R"({"hgraph": {"ef_search": 32, "skip_ratio": 0.5}})");
REQUIRE(params.skip_ratio == 0.5F);
}

SECTION("custom skip_strategy") {
auto params = vsag::HGraphSearchParameters::FromJson(
R"({"hgraph": {"ef_search": 32, "skip_strategy": "random"}})");
REQUIRE(params.skip_strategy_type == vsag::FilterSearchSkipStrategyType::RANDOM);
}
Comment thread
LHT129 marked this conversation as resolved.

SECTION("rejects out-of-range skip_ratio") {
REQUIRE_THROWS(vsag::HGraphSearchParameters::FromJson(
R"({"hgraph": {"ef_search": 32, "skip_ratio": 1.5}})"));
REQUIRE_THROWS(vsag::HGraphSearchParameters::FromJson(
R"({"hgraph": {"ef_search": 32, "skip_ratio": -0.1}})"));
}
}
2 changes: 1 addition & 1 deletion src/impl/inner_search_param.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ class InnerSearchParam {
uint64_t ef{10};
uint32_t hops_limit{std::numeric_limits<uint32_t>::max()};
FilterPtr is_inner_id_allowed{nullptr};
float skip_ratio{0.8F};
float skip_ratio{0.2F};
FilterSearchSkipStrategyType skip_strategy_type{
FilterSearchSkipStrategyType::DETERMINISTIC_ACCUMULATIVE};
InnerSearchMode search_mode{KNN_SEARCH};
Expand Down
2 changes: 1 addition & 1 deletion src/index/hnsw_zparameters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ struct HnswSearchParameters {
public:
// required vars
int64_t ef_search;
float skip_ratio{0.9};
float skip_ratio{0.1F};
FilterSearchSkipStrategyType skip_strategy_type{
FilterSearchSkipStrategyType::DETERMINISTIC_ACCUMULATIVE};
bool use_conjugate_graph_search;
Expand Down
7 changes: 2 additions & 5 deletions src/utils/filter_search_skip_strategy.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,7 @@ constexpr const char* DETERMINISTIC_ACCUMULATIVE_SKIP_STRATEGY = "deterministic_

double
get_retain_ratio(float valid_ratio, float skip_ratio) {
if (valid_ratio == 1.0F) {
return 1.0;
}
return static_cast<double>(1.0F - valid_ratio) * static_cast<double>(skip_ratio);
return static_cast<double>(1.0F - valid_ratio) * static_cast<double>(1.0F - skip_ratio);
}

class RandomFilterSearchSkipStrategy : public FilterSearchSkipStrategy {
Expand All @@ -39,7 +36,7 @@ class RandomFilterSearchSkipStrategy : public FilterSearchSkipStrategy {

bool
ShouldVisit() override {
return generator_.NextFloat() > visit_threshold_;
return generator_.NextFloat() >= visit_threshold_;
}

private:
Expand Down
16 changes: 8 additions & 8 deletions src/utils/filter_search_skip_strategy_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ TEST_CASE("Filter search skip strategy parse", "[ut][filter_search_skip_strategy
FilterSearchSkipStrategyType::RANDOM);
REQUIRE(parse_filter_search_skip_strategy_type("deterministic_accumulative") ==
FilterSearchSkipStrategyType::DETERMINISTIC_ACCUMULATIVE);
REQUIRE(std::string(filter_search_skip_strategy_type_to_string(
FilterSearchSkipStrategyType::RANDOM)) == "random");
REQUIRE(std::string(filter_search_skip_strategy_type_to_string(
FilterSearchSkipStrategyType::DETERMINISTIC_ACCUMULATIVE)) ==
"deterministic_accumulative");
Expand All @@ -32,7 +34,7 @@ TEST_CASE("Filter search skip strategy parse", "[ut][filter_search_skip_strategy

TEST_CASE("Accumulative ShouldVisit is deterministic", "[ut][filter_search_skip_strategy]") {
constexpr float valid_ratio = 0.5F;
constexpr float skip_ratio = 0.8F;
constexpr float skip_ratio = 0.2F;
std::vector<bool> first_sequence;
std::vector<bool> second_sequence;

Expand All @@ -47,7 +49,7 @@ TEST_CASE("Accumulative ShouldVisit is deterministic", "[ut][filter_search_skip_
}

REQUIRE(first_sequence == second_sequence);
// visit_ratio = 0.5 + 0.5*0.8 = 0.9
// visit_ratio = 0.5 + (1-0.5)*(1-0.2) = 0.5 + 0.4 = 0.9
// Accumulative pattern: F,T,T,T,T,T,T,T,T,T repeated = 18/20 true
std::vector<bool> expected = {false, true, true, true, true, true, true, true, true, true,
false, true, true, true, true, true, true, true, true, true};
Expand All @@ -63,18 +65,16 @@ TEST_CASE("ShouldVisit edge cases", "[ut][filter_search_skip_strategy]") {
}
}

SECTION("skip ratio zero with low valid ratio visits less") {
SECTION("skip ratio zero means no skipping") {
auto strategy = create_filter_search_skip_strategy(
FilterSearchSkipStrategyType::DETERMINISTIC_ACCUMULATIVE, 0.5F, 0.0F);
// visit_ratio = 0.5 + 0.5*0 = 0.5
// Accumulative pattern: F,T,F,T,... alternating = exactly 50/100 true
// visit_ratio = 0.5 + (1-0.5)*(1-0.0) = 0.5 + 0.5 = 1.0
// Accumulative pattern: all true since visit_ratio = 1.0
std::vector<bool> sequence;
for (uint64_t i = 0; i < 20; ++i) {
sequence.emplace_back(strategy->ShouldVisit());
}
std::vector<bool> expected = {false, true, false, true, false, true, false,
true, false, true, false, true, false, true,
false, true, false, true, false, true};
std::vector<bool> expected(20, true); // All true since we visit everything
REQUIRE(sequence == expected);
}
}
Loading