From a269f8c530931562166092f7b4e5fedc73f8c897 Mon Sep 17 00:00:00 2001 From: rajeshsub <4209324+rajeshsub@users.noreply.github.com> Date: Fri, 5 Jun 2026 22:44:20 +1000 Subject: [PATCH] Add token set word matching, fix re2 capture group bug and settings round-trip bug, update readme --- .pre-commit-config.yaml | 3 +- CMakeLists.txt | 2 + README.md | 8 +- src/core/CMakeLists.txt | 1 + src/core/search/SearchEngine.cpp | 52 +++--- src/core/search/SearchEngine.h | 18 +- src/core/search/TokenMatcher.cpp | 41 +++++ src/core/search/TokenMatcher.h | 29 +++ src/core/settings/Settings.cpp | 18 +- tests/mocks/MockFileSystemScanner.h | 7 +- tests/mocks/MockIndexStore.h | 31 ++-- tests/mocks/MockUsnJournalMonitor.h | 10 +- tests/test_DriveEnumerator.cpp | 1 + tests/test_IndexSerializer.cpp | 37 ++-- tests/test_Indexer.cpp | 57 +++--- tests/test_SearchEngine.cpp | 268 +++++++++++++++++++++++----- tests/test_Settings.cpp | 11 +- 17 files changed, 418 insertions(+), 176 deletions(-) create mode 100644 src/core/search/TokenMatcher.cpp create mode 100644 src/core/search/TokenMatcher.h diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e54b89..a9fbe59 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: exclude: '^.*\.bat$' - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v18.1.8 + rev: v22.1.5 hooks: - id: clang-format types_or: [c++, c] @@ -26,6 +26,7 @@ repos: hooks: - id: cppcheck args: + - '--language=c++' - '--std=c++20' - '--enable=warning,style,performance' - '--suppress=missingIncludeSystem' diff --git a/CMakeLists.txt b/CMakeLists.txt index 74454e9..b3d2416 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,8 @@ if(ENABLE_ASAN) message(FATAL_ERROR "ENABLE_ASAN is only supported with MSVC.") endif() add_compile_options(/fsanitize=address) + # /INCREMENTAL is incompatible with ASAN metadata; suppress LNK4300 by disabling it. + add_link_options(/INCREMENTAL:NO) # Abseil/RE2/GTest are built without ASAN so their STL annotation values are 0. # Disable all MSVC STL annotations in our code to match; otherwise LNK2038 at link. add_compile_definitions( diff --git a/README.md b/README.md index a853561..f4305f0 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Blazing fast file search for Windows. winindex builds a full index of your local - **Fallback scanner** — on FAT32, or without elevation on NTFS, a fast `FindFirstFile` BFS scanner is used automatically - **Regex support** — powered by [RE2](https://github.com/google/re2); toggle with Alt+1 - **SIMD-accelerated substring search** — dispatch to AVX2 or SSE4.2 when compiled with `/arch:AVX2`, scalar fallback otherwise +- **Word-level matching** — queries with spaces, underscores, or hyphens match filenames by token set, so `just rosy guitar` finds `LedZep_Just-Rosy_June-Bug_guitar.flac` even though the words are non-adjacent and separated by different delimiters - **Search modes** — case-sensitive, whole-word, match full path, ignore diacritics; all togglable from the Search menu - **Change detection** — USN journal replay and `ReadDirectoryChangesW` watcher are wired in and ready; live background monitoring is in active development - **Portable mode** — place a `winindex.ini` next to the executable and all data stays in that directory @@ -60,6 +61,7 @@ Blazing fast file search for Windows. winindex builds a full index of your local Each keystroke (after a 150 ms debounce) spawns a background `std::thread`: - **Substring mode** — the needle and each filename are lowercased once; `SimdFindSubstring` dispatches to the fastest available SIMD path at runtime. +- **Token-set mode** — when the query contains a separator character (space, `_`, `-`, `.`), both the query and each filename are split into tokens and the file matches if every query token appears somewhere in the filename token set. This lets `just rosy guitar` match `LedZep_Just-Rosy_June-Bug_guitar.flac` without regex. Single-word queries skip this path entirely, keeping the SIMD fast path. - **Regex mode** — filenames (or full paths in match-path mode) are converted to UTF-8 and matched with a compiled `RE2` pattern. - Results are capped at 10 000 and rendered in a virtual `LVS_OWNERDATA` ListView for zero-copy display. @@ -75,7 +77,7 @@ Each keystroke (after a 150 ms debounce) spawns a background `std::thread`: | Tool | Minimum version | |------|----------------| -| Windows | 10 or 11 | +| Windows | 10 or above | | Visual Studio Build Tools | 2026 (local builds); CI auto-detects 2022+ | | CMake | 3.28 | | Git | any recent | @@ -138,7 +140,7 @@ Settings are stored in `%APPDATA%\winindex\winindex.ini` (or next to the `.exe` - `%APPDATA%` (Roaming) - `%LOCALAPPDATA%` -These are merged into the saved exclusion list on every startup, so new defaults take effect on existing installs automatically. +These are applied as the initial default on a new install. Once a user has saved their own exclusion list, that list is used as-is and the defaults are not re-injected. --- @@ -166,7 +168,7 @@ winindex/ src/ core/ indexer/ MftScanner, FindFileScanner, Indexer, ChangeWatcher, USN journal - search/ SearchEngine, SIMD search, RE2 integration + search/ SearchEngine, SIMD search, RE2 integration, TokenMatcher (word-level matching) settings/ Settings (INI), PathUtils storage/ IndexStore, IndexSerializer (binary format + CRC-32) ui/ diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 3391a2c..f52c6ed 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -7,6 +7,7 @@ add_library(winindex_core STATIC indexer/Indexer.cpp search/SearchEngine.cpp search/SimdSearch.cpp + search/TokenMatcher.cpp storage/IndexStore.cpp storage/IndexSerializer.cpp settings/Settings.cpp diff --git a/src/core/search/SearchEngine.cpp b/src/core/search/SearchEngine.cpp index 2e78963..ccbc4fb 100644 --- a/src/core/search/SearchEngine.cpp +++ b/src/core/search/SearchEngine.cpp @@ -1,6 +1,7 @@ #include "SearchEngine.h" #include "SimdSearch.h" +#include "TokenMatcher.h" #define WIN32_LEAN_AND_MEAN #include @@ -23,20 +24,6 @@ std::string SearchEngine::WideToUtf8(const std::wstring& s) { return r; } -// Very basic diacritic normalization via NFC -> ASCII fold using WinAPI -std::wstring SearchEngine::NormalizeDiacritics(const std::wstring& s) { - // FoldString with MAP_PRECOMPOSED + custom: use LCMapString for normalization - int needed = - LCMapStringEx(LOCALE_NAME_INVARIANT, LCMAP_LINGUISTIC_CASING | LCMAP_LOWERCASE, s.c_str(), - static_cast(s.size()), nullptr, 0, nullptr, nullptr, 0); - if (needed <= 0) - return s; - std::wstring result(needed, L'\0'); - LCMapStringEx(LOCALE_NAME_INVARIANT, LCMAP_LINGUISTIC_CASING | LCMAP_LOWERCASE, s.c_str(), - static_cast(s.size()), result.data(), needed, nullptr, nullptr, 0); - return result; -} - bool SearchEngine::MatchesWholeWord(const std::wstring& text, size_t pos, size_t len) { auto isWordChar = [](wchar_t c) { return iswalnum(c) || c == L'_'; }; if (pos > 0 && isWordChar(text[pos - 1])) @@ -120,6 +107,19 @@ std::vector SearchEngine::SearchSubstring( return q; }(); + // Token-set matching: pre-compute once, shared read-only across threads. + // Only activated when the query contains separator chars (space/_/-/.) + // so single-word queries take the unmodified SIMD-only path. + const bool doTokenMatch = !options.caseSensitive && TokenMatcher::QueryHasSeparators(query); + // lowercaseQuery owns the storage that sortedQueryTokens views reference. + std::wstring lowercaseQuery; + std::vector sortedQueryTokens; + if (doTokenMatch) { + lowercaseQuery = needle; // needle is already lowercased at this point + sortedQueryTokens = TokenMatcher::TokenizeView(lowercaseQuery); + std::sort(sortedQueryTokens.begin(), sortedQueryTokens.end()); + } + unsigned int numThreads = std::max(1u, std::thread::hardware_concurrency()); uint64_t chunkSize = (entryCount + numThreads - 1) / numThreads; @@ -166,11 +166,23 @@ std::vector SearchEngine::SearchSubstring( size_t pos = SimdFindSubstring(haystackData, haystackLen, needle.c_str(), needle.size()); - if (pos == std::wstring::npos) - continue; - if (options.wholeWord) { - // Reconstruct wstring view for word-boundary check + bool tokenMatch = false; + if (pos == std::wstring::npos) { + // Token-set fallback: fires only when SIMD missed and the query + // had separators (e.g. "just rosy", "rosy guitar flac"). + if (!doTokenMatch) + continue; + std::wstring haystackStr(haystackData, haystackLen); + auto fnTokens = TokenMatcher::TokenizeView(haystackStr); + std::sort(fnTokens.begin(), fnTokens.end()); + if (!TokenMatcher::AllQueryTokensPresent(sortedQueryTokens, fnTokens)) + continue; + tokenMatch = true; + pos = 0; + } + + if (options.wholeWord && !tokenMatch) { std::wstring_view hayView(haystackData, haystackLen); if (!MatchesWholeWord(std::wstring(hayView), pos, needle.size())) continue; @@ -179,7 +191,7 @@ std::vector SearchEngine::SearchSubstring( SearchResult sr; sr.entry = &e; sr.matchStart = static_cast(pos); - sr.matchLen = static_cast(needle.size()); + sr.matchLen = tokenMatch ? 0u : static_cast(needle.size()); local.push_back(sr); } return local; @@ -191,7 +203,7 @@ std::vector SearchEngine::SearchSubstring( for (auto& f : futures) { auto chunk = f.get(); - for (auto& sr : chunk) { + for (const auto& sr : chunk) { if (results.size() >= maxResults) goto done; results.push_back(sr); diff --git a/src/core/search/SearchEngine.h b/src/core/search/SearchEngine.h index 87fb1f2..22f58d3 100644 --- a/src/core/search/SearchEngine.h +++ b/src/core/search/SearchEngine.h @@ -15,19 +15,19 @@ class SearchEngine : public ISearchEngine { private: // Regex search path - std::vector SearchRegex(const std::wstring& query, const FileEntry* entries, - uint64_t entryCount, const SearchOptions& options, - uint32_t maxResults, - const std::atomic& cancelToken); + static std::vector SearchRegex(const std::wstring& query, + const FileEntry* entries, uint64_t entryCount, + const SearchOptions& options, uint32_t maxResults, + const std::atomic& cancelToken); // SIMD substring search path (parallelized) - std::vector SearchSubstring(const std::wstring& query, const FileEntry* entries, - uint64_t entryCount, const SearchOptions& options, - uint32_t maxResults, - const std::atomic& cancelToken); + static std::vector SearchSubstring(const std::wstring& query, + const FileEntry* entries, uint64_t entryCount, + const SearchOptions& options, + uint32_t maxResults, + const std::atomic& cancelToken); static bool MatchesWholeWord(const std::wstring& text, size_t matchPos, size_t matchLen); - static std::wstring NormalizeDiacritics(const std::wstring& s); static std::string WideToUtf8(const std::wstring& s); }; diff --git a/src/core/search/TokenMatcher.cpp b/src/core/search/TokenMatcher.cpp new file mode 100644 index 0000000..b2c4162 --- /dev/null +++ b/src/core/search/TokenMatcher.cpp @@ -0,0 +1,41 @@ +#include "TokenMatcher.h" + +#include + +namespace winindex { +namespace TokenMatcher { + +bool QueryHasSeparators(const std::wstring& query) { + for (wchar_t c : query) + if (IsTokenSep(c)) + return true; + return false; +} + +std::vector TokenizeView(const std::wstring& s) { + std::vector tokens; + const std::wstring_view sv(s); + size_t start = 0; + for (size_t i = 0; i <= sv.size(); ++i) { + if (i == sv.size() || IsTokenSep(sv[i])) { + if (i > start) + tokens.emplace_back(sv.substr(start, i - start)); + start = i + 1; + } + } + return tokens; +} + +bool AllQueryTokensPresent(const std::vector& sortedQueryTokens, + const std::vector& sortedFilenameTokens) { + if (sortedQueryTokens.empty()) + return false; + for (const auto& qt : sortedQueryTokens) { + if (!std::binary_search(sortedFilenameTokens.begin(), sortedFilenameTokens.end(), qt)) + return false; + } + return true; +} + +} // namespace TokenMatcher +} // namespace winindex diff --git a/src/core/search/TokenMatcher.h b/src/core/search/TokenMatcher.h new file mode 100644 index 0000000..403c80b --- /dev/null +++ b/src/core/search/TokenMatcher.h @@ -0,0 +1,29 @@ +#pragma once +#include +#include +#include +#include + +namespace winindex { +namespace TokenMatcher { + +inline bool IsTokenSep(wchar_t c) { + return c == L' ' || c == L'_' || c == L'-' || c == L'.'; +} + +// Returns true if query contains at least one separator character. +// Used to gate token-set matching — single-word queries skip this path. +bool QueryHasSeparators(const std::wstring& query); + +// Split a (pre-lowercased) wstring on separator chars into wstring_view slices. +// Consecutive separators produce no empty tokens. +// The returned views reference `s`, which must outlive them. +std::vector TokenizeView(const std::wstring& s); + +// Returns true if every token in sortedQueryTokens is present in +// sortedFilenameTokens (exact equality). Both must be pre-sorted. +bool AllQueryTokensPresent(const std::vector& sortedQueryTokens, + const std::vector& sortedFilenameTokens); + +} // namespace TokenMatcher +} // namespace winindex diff --git a/src/core/settings/Settings.cpp b/src/core/settings/Settings.cpp index b42d205..6f0b014 100644 --- a/src/core/settings/Settings.cpp +++ b/src/core/settings/Settings.cpp @@ -41,9 +41,7 @@ void Settings::Load() { m_selectedDrives.push_back(token); } - // Excluded paths: load user-configured list, then merge in system defaults. - // System defaults (AppData, Windows, etc.) are always enforced so that - // adding new defaults in a code update takes effect on existing installs. + // Excluded paths: use saved list if present, otherwise seed with defaults. std::wstring exclStr = ReadString(L"Indexing", L"ExcludedPaths"); if (exclStr.empty()) { m_excludedPaths = DefaultExcludedPaths(); @@ -54,20 +52,6 @@ void Settings::Load() { while (std::getline(ss, token, L'|')) if (!token.empty()) m_excludedPaths.push_back(token); - - // Merge in any defaults not already present so new defaults take effect - // on existing installs without requiring a manual settings reset. - for (const auto& def : DefaultExcludedPaths()) { - bool found = false; - for (const auto& e : m_excludedPaths) { - if (_wcsicmp(e.c_str(), def.c_str()) == 0) { - found = true; - break; - } - } - if (!found) - m_excludedPaths.push_back(def); - } } m_reindexIntervalHours = static_cast( diff --git a/tests/mocks/MockFileSystemScanner.h b/tests/mocks/MockFileSystemScanner.h index 06a6dfd..3ec49ff 100644 --- a/tests/mocks/MockFileSystemScanner.h +++ b/tests/mocks/MockFileSystemScanner.h @@ -1,5 +1,6 @@ #pragma once #include + #include "indexer/IFileSystemScanner.h" namespace winindex { @@ -8,11 +9,9 @@ class MockFileSystemScanner : public IFileSystemScanner { public: MOCK_METHOD(bool, IsMftAvailable, (const std::wstring& root), (const, override)); MOCK_METHOD(void, Scan, - (const ScanOptions& options, - ScanCallback onFile, - ProgressCallback onProgress, + (const ScanOptions& options, ScanCallback onFile, ProgressCallback onProgress, const std::atomic& cancelToken), (override)); }; -} // namespace winindex +} // namespace winindex diff --git a/tests/mocks/MockIndexStore.h b/tests/mocks/MockIndexStore.h index e1cfdd3..e8cd55a 100644 --- a/tests/mocks/MockIndexStore.h +++ b/tests/mocks/MockIndexStore.h @@ -1,26 +1,25 @@ #pragma once #include + #include "storage/IIndexStore.h" namespace winindex { class MockIndexStore : public IIndexStore { public: - MOCK_METHOD(bool, IsIndexValid, (), (const, override)); - MOCK_METHOD(void, Load, (), (override)); - MOCK_METHOD(void, Save, (), (override)); - MOCK_METHOD(void, BeginWrite, (), (override)); - MOCK_METHOD(void, AddEntry, (const FileEntry& e), (override)); - MOCK_METHOD(void, EndWrite, (), (override)); - MOCK_METHOD(void, ApplyAdd, (const FileEntry& e), (override)); - MOCK_METHOD(void, ApplyRemove, (const std::wstring& path), (override)); - MOCK_METHOD(void, ApplyRename, (const std::wstring& o, - const std::wstring& n), (override)); - MOCK_METHOD(uint64_t, GetEntryCount, (), (const, override)); - MOCK_METHOD(const FileEntry*, GetEntries, (), (const, override)); - MOCK_METHOD(uint64_t, GetSavedUsn, (const std::wstring& root), (const, override)); - MOCK_METHOD(void, SetSavedUsn, (const std::wstring& root, - uint64_t usn), (override)); + MOCK_METHOD(bool, IsIndexValid, (), (const, override)); + MOCK_METHOD(void, Load, (), (override)); + MOCK_METHOD(void, Save, (), (override)); + MOCK_METHOD(void, BeginWrite, (), (override)); + MOCK_METHOD(void, AddEntry, (const FileEntry& e), (override)); + MOCK_METHOD(void, EndWrite, (), (override)); + MOCK_METHOD(void, ApplyAdd, (const FileEntry& e), (override)); + MOCK_METHOD(void, ApplyRemove, (const std::wstring& path), (override)); + MOCK_METHOD(void, ApplyRename, (const std::wstring& o, const std::wstring& n), (override)); + MOCK_METHOD(uint64_t, GetEntryCount, (), (const, override)); + MOCK_METHOD(const FileEntry*, GetEntries, (), (const, override)); + MOCK_METHOD(uint64_t, GetSavedUsn, (const std::wstring& root), (const, override)); + MOCK_METHOD(void, SetSavedUsn, (const std::wstring& root, uint64_t usn), (override)); }; -} // namespace winindex +} // namespace winindex diff --git a/tests/mocks/MockUsnJournalMonitor.h b/tests/mocks/MockUsnJournalMonitor.h index e2a3621..c1b4a29 100644 --- a/tests/mocks/MockUsnJournalMonitor.h +++ b/tests/mocks/MockUsnJournalMonitor.h @@ -1,5 +1,6 @@ #pragma once #include + #include "indexer/IUsnJournalMonitor.h" namespace winindex { @@ -8,12 +9,11 @@ class MockUsnJournalMonitor : public IUsnJournalMonitor { public: MOCK_METHOD(bool, IsAvailable, (const std::wstring& root), (const, override)); MOCK_METHOD(uint64_t, ReplaySince, - (const std::wstring& root, uint64_t savedUsn, ChangeCallback onChange), - (override)); + (const std::wstring& root, uint64_t savedUsn, ChangeCallback onChange), (override)); MOCK_METHOD(void, StartMonitoring, - (const std::wstring& root, uint64_t startUsn, - ChangeCallback onChange, const std::atomic& stopToken), + (const std::wstring& root, uint64_t startUsn, ChangeCallback onChange, + const std::atomic& stopToken), (override)); }; -} // namespace winindex +} // namespace winindex diff --git a/tests/test_DriveEnumerator.cpp b/tests/test_DriveEnumerator.cpp index c7a78e1..29f8d1f 100644 --- a/tests/test_DriveEnumerator.cpp +++ b/tests/test_DriveEnumerator.cpp @@ -1,4 +1,5 @@ #include + #include "indexer/DriveEnumerator.h" using namespace winindex; diff --git a/tests/test_IndexSerializer.cpp b/tests/test_IndexSerializer.cpp index 9c85e73..6abf0b9 100644 --- a/tests/test_IndexSerializer.cpp +++ b/tests/test_IndexSerializer.cpp @@ -1,6 +1,7 @@ #include #define WIN32_LEAN_AND_MEAN #include + #include "storage/IndexSerializer.h" #include #include @@ -18,9 +19,7 @@ class IndexSerializerTest : public ::testing::Test { tmpPath = std::wstring(tmp) + L"winindex_test.idx"; } - void TearDown() override { - _wremove(tmpPath.c_str()); - } + void TearDown() override { _wremove(tmpPath.c_str()); } }; TEST_F(IndexSerializerTest, RoundtripEmptyIndex) { @@ -39,13 +38,13 @@ TEST_F(IndexSerializerTest, RoundtripEmptyIndex) { TEST_F(IndexSerializerTest, RoundtripSingleEntry) { FileEntry e; - e.name = L"hello.txt"; - e.path = L"C:\\Users\\test\\hello.txt"; - e.size = 12345; + e.name = L"hello.txt"; + e.path = L"C:\\Users\\test\\hello.txt"; + e.size = 12345; e.lastModified = 999888777; - e.attributes = FILE_ATTRIBUTE_NORMAL; + e.attributes = FILE_ATTRIBUTE_NORMAL; - std::vector entries = { e }; + std::vector entries = {e}; std::unordered_map usnMap; usnMap[L"C:\\"] = 42; @@ -57,23 +56,23 @@ TEST_F(IndexSerializerTest, RoundtripSingleEntry) { ASSERT_TRUE(IndexSerializer::Deserialize(tmpPath, loaded, loadedUsn, ts)); ASSERT_EQ(loaded.size(), 1u); - EXPECT_EQ(loaded[0].name, e.name); - EXPECT_EQ(loaded[0].path, e.path); - EXPECT_EQ(loaded[0].size, e.size); + EXPECT_EQ(loaded[0].name, e.name); + EXPECT_EQ(loaded[0].path, e.path); + EXPECT_EQ(loaded[0].size, e.size); EXPECT_EQ(loaded[0].lastModified, e.lastModified); - EXPECT_EQ(loaded[0].attributes, e.attributes); - EXPECT_EQ(loadedUsn[L"C:\\"], 42u); + EXPECT_EQ(loaded[0].attributes, e.attributes); + EXPECT_EQ(loadedUsn[L"C:\\"], 42u); } TEST_F(IndexSerializerTest, RoundtripManyEntries) { std::vector entries; for (int i = 0; i < 10000; ++i) { FileEntry e; - e.name = L"file_" + std::to_wstring(i) + L".dat"; - e.path = L"C:\\data\\file_" + std::to_wstring(i) + L".dat"; - e.size = static_cast(i) * 1024; + e.name = L"file_" + std::to_wstring(i) + L".dat"; + e.path = L"C:\\data\\file_" + std::to_wstring(i) + L".dat"; + e.size = static_cast(i) * 1024; e.lastModified = static_cast(i); - e.attributes = FILE_ATTRIBUTE_NORMAL; + e.attributes = FILE_ATTRIBUTE_NORMAL; entries.push_back(e); } @@ -105,6 +104,6 @@ TEST_F(IndexSerializerTest, MissingFileFails) { std::vector loaded; std::unordered_map usnMap; uint64_t ts = 0; - EXPECT_FALSE(IndexSerializer::Deserialize(L"C:\\does_not_exist_winindex.idx", - loaded, usnMap, ts)); + EXPECT_FALSE( + IndexSerializer::Deserialize(L"C:\\does_not_exist_winindex.idx", loaded, usnMap, ts)); } diff --git a/tests/test_Indexer.cpp b/tests/test_Indexer.cpp index b1dc5bd..26aaad2 100644 --- a/tests/test_Indexer.cpp +++ b/tests/test_Indexer.cpp @@ -1,27 +1,28 @@ -#include #include +#include + #include "indexer/Indexer.h" #include "mocks/MockFileSystemScanner.h" -#include "mocks/MockUsnJournalMonitor.h" #include "mocks/MockIndexStore.h" +#include "mocks/MockUsnJournalMonitor.h" #include "settings/Settings.h" #define WIN32_LEAN_AND_MEAN #include using namespace winindex; using ::testing::_; -using ::testing::Return; -using ::testing::Invoke; using ::testing::AtLeast; +using ::testing::Invoke; +using ::testing::Return; class IndexerTest : public ::testing::Test { protected: std::shared_ptr mftScanner; std::shared_ptr findScanner; std::shared_ptr usnMonitor; - std::shared_ptr indexStore; - std::shared_ptr settings; - std::unique_ptr indexer; + std::shared_ptr indexStore; + std::shared_ptr settings; + std::unique_ptr indexer; std::wstring tmpDir; @@ -31,16 +32,16 @@ class IndexerTest : public ::testing::Test { tmpDir = std::wstring(tmp) + L"winindex_indexer_test"; CreateDirectoryW(tmpDir.c_str(), nullptr); - mftScanner = std::make_shared(); + mftScanner = std::make_shared(); findScanner = std::make_shared(); - usnMonitor = std::make_shared(); - indexStore = std::make_shared(); - settings = std::make_shared(true, tmpDir); + usnMonitor = std::make_shared(); + indexStore = std::make_shared(); + settings = std::make_shared(true, tmpDir); settings->Load(); - settings->SetSelectedDrives({ L"C:\\" }); + settings->SetSelectedDrives({L"C:\\"}); - indexer = std::make_unique( - mftScanner, findScanner, usnMonitor, indexStore, settings); + indexer = + std::make_unique(mftScanner, findScanner, usnMonitor, indexStore, settings); } void TearDown() override { @@ -54,7 +55,7 @@ TEST_F(IndexerTest, LoadsExistingValidIndex) { EXPECT_CALL(*indexStore, Load()).Times(1); EXPECT_CALL(*indexStore, GetEntryCount()).WillRepeatedly(Return(42)); // Should NOT call scanner when index is valid - EXPECT_CALL(*mftScanner, Scan(_, _, _, _)).Times(0); + EXPECT_CALL(*mftScanner, Scan(_, _, _, _)).Times(0); EXPECT_CALL(*findScanner, Scan(_, _, _, _)).Times(0); indexer->StartIndexing(false); @@ -72,14 +73,14 @@ TEST_F(IndexerTest, FullScanWhenIndexInvalid) { EXPECT_CALL(*mftScanner, IsMftAvailable(std::wstring(L"C:\\"))).WillOnce(Return(false)); EXPECT_CALL(*findScanner, IsMftAvailable(_)).WillRepeatedly(Return(false)); EXPECT_CALL(*findScanner, Scan(_, _, _, _)) - .WillOnce(Invoke([](const ScanOptions&, ScanCallback cb, - ProgressCallback, const std::atomic&) { - FileEntry e; - e.name = L"test.txt"; - e.path = L"C:\\test.txt"; - e.size = 100; - cb(e); - })); + .WillOnce(Invoke( + [](const ScanOptions&, ScanCallback cb, ProgressCallback, const std::atomic&) { + FileEntry e; + e.name = L"test.txt"; + e.path = L"C:\\test.txt"; + e.size = 100; + cb(e); + })); EXPECT_CALL(*indexStore, AddEntry(_)).Times(1); @@ -88,14 +89,14 @@ TEST_F(IndexerTest, FullScanWhenIndexInvalid) { } TEST_F(IndexerTest, ForceRebuildIgnoresValidIndex) { - EXPECT_CALL(*indexStore, IsIndexValid()).Times(0); // force=true skips check + EXPECT_CALL(*indexStore, IsIndexValid()).Times(0); // force=true skips check EXPECT_CALL(*indexStore, BeginWrite()).Times(1); EXPECT_CALL(*indexStore, EndWrite()).Times(1); EXPECT_CALL(*indexStore, Save()).Times(1); - EXPECT_CALL(*mftScanner, IsMftAvailable(std::wstring(L"C:\\"))).WillOnce(Return(false)); + EXPECT_CALL(*mftScanner, IsMftAvailable(std::wstring(L"C:\\"))).WillOnce(Return(false)); EXPECT_CALL(*findScanner, Scan(_, _, _, _)).Times(1); - EXPECT_CALL(*indexStore, AddEntry(_)).Times(0); + EXPECT_CALL(*indexStore, AddEntry(_)).Times(0); indexer->StartIndexing(true /*force*/); indexer->WaitForCompletion(); @@ -110,9 +111,7 @@ TEST_F(IndexerTest, StatusCallbackFired) { EXPECT_CALL(*findScanner, Scan(_, _, _, _)).Times(1); std::vector states; - indexer->SetStatusCallback([&](const IndexerStatus& s) { - states.push_back(s.state); - }); + indexer->SetStatusCallback([&](const IndexerStatus& s) { states.push_back(s.state); }); indexer->StartIndexing(false); indexer->WaitForCompletion(); diff --git a/tests/test_SearchEngine.cpp b/tests/test_SearchEngine.cpp index df04fa1..b853936 100644 --- a/tests/test_SearchEngine.cpp +++ b/tests/test_SearchEngine.cpp @@ -1,6 +1,8 @@ #include + #include "search/SearchEngine.h" #include "search/SimdSearch.h" +#include "search/TokenMatcher.h" #include #include @@ -12,6 +14,8 @@ static std::vector MakeEntries( for (auto& [name, path] : items) { FileEntry e; e.name = name; + e.nameLower = name; + std::transform(e.nameLower.begin(), e.nameLower.end(), e.nameLower.begin(), ::towlower); e.path = path; e.size = 0; e.lastModified = 0; @@ -29,96 +33,91 @@ class SearchEngineTest : public ::testing::Test { TEST_F(SearchEngineTest, BasicSubstringMatch) { auto entries = MakeEntries({ - { L"report_2024.xlsx", L"C:\\docs\\report_2024.xlsx" }, - { L"summary.pdf", L"C:\\docs\\summary.pdf" }, - { L"report_q1.docx", L"C:\\docs\\report_q1.docx" }, + {L"report_2024.xlsx", L"C:\\docs\\report_2024.xlsx"}, + {L"summary.pdf", L"C:\\docs\\summary.pdf"}, + {L"report_q1.docx", L"C:\\docs\\report_q1.docx"}, }); SearchOptions opts{}; - auto results = engine.Search(L"report", entries.data(), entries.size(), - opts, 100, cancel); + auto results = engine.Search(L"report", entries.data(), entries.size(), opts, 100, cancel); EXPECT_EQ(results.size(), 2u); } TEST_F(SearchEngineTest, CaseSensitiveMatch) { auto entries = MakeEntries({ - { L"Report.txt", L"C:\\Report.txt" }, - { L"report.txt", L"C:\\report.txt" }, + {L"Report.txt", L"C:\\Report.txt"}, + {L"report.txt", L"C:\\report.txt"}, }); SearchOptions opts{}; opts.caseSensitive = true; - auto results = engine.Search(L"Report", entries.data(), entries.size(), - opts, 100, cancel); + auto results = engine.Search(L"Report", entries.data(), entries.size(), opts, 100, cancel); ASSERT_EQ(results.size(), 1u); EXPECT_EQ(results[0].entry->name, L"Report.txt"); } TEST_F(SearchEngineTest, CaseInsensitiveMatch) { auto entries = MakeEntries({ - { L"REPORT.txt", L"C:\\REPORT.txt" }, - { L"report.txt", L"C:\\report.txt" }, + {L"REPORT.txt", L"C:\\REPORT.txt"}, + {L"report.txt", L"C:\\report.txt"}, }); SearchOptions opts{}; opts.caseSensitive = false; - auto results = engine.Search(L"report", entries.data(), entries.size(), - opts, 100, cancel); + auto results = engine.Search(L"report", entries.data(), entries.size(), opts, 100, cancel); EXPECT_EQ(results.size(), 2u); } TEST_F(SearchEngineTest, WholeWordMatch) { auto entries = MakeEntries({ - { L"report.txt", L"C:\\report.txt" }, - { L"reports_final.txt", L"C:\\reports_final.txt" }, + {L"report.txt", L"C:\\report.txt"}, + {L"reports_final.txt", L"C:\\reports_final.txt"}, }); SearchOptions opts{}; opts.wholeWord = true; - auto results = engine.Search(L"report", entries.data(), entries.size(), - opts, 100, cancel); + auto results = engine.Search(L"report", entries.data(), entries.size(), opts, 100, cancel); ASSERT_EQ(results.size(), 1u); EXPECT_EQ(results[0].entry->name, L"report.txt"); } TEST_F(SearchEngineTest, MatchPathOption) { auto entries = MakeEntries({ - { L"file.txt", L"C:\\Projects\\report\\file.txt" }, - { L"other.txt", L"C:\\Documents\\other.txt" }, + {L"file.txt", L"C:\\Projects\\report\\file.txt"}, + {L"other.txt", L"C:\\Documents\\other.txt"}, }); SearchOptions opts{}; opts.matchPath = true; - auto results = engine.Search(L"report", entries.data(), entries.size(), - opts, 100, cancel); + auto results = engine.Search(L"report", entries.data(), entries.size(), opts, 100, cancel); ASSERT_EQ(results.size(), 1u); EXPECT_EQ(results[0].entry->name, L"file.txt"); } TEST_F(SearchEngineTest, RegexMatch) { auto entries = MakeEntries({ - { L"invoice_001.pdf", L"C:\\invoice_001.pdf" }, - { L"invoice_abc.pdf", L"C:\\invoice_abc.pdf" }, - { L"summary.pdf", L"C:\\summary.pdf" }, + {L"invoice_001.pdf", L"C:\\invoice_001.pdf"}, + {L"invoice_abc.pdf", L"C:\\invoice_abc.pdf"}, + {L"summary.pdf", L"C:\\summary.pdf"}, }); SearchOptions opts{}; opts.useRegex = true; - auto results = engine.Search(L"invoice_\\d+", entries.data(), entries.size(), - opts, 100, cancel); + auto results = + engine.Search(L"invoice_\\d+", entries.data(), entries.size(), opts, 100, cancel); ASSERT_EQ(results.size(), 1u); EXPECT_EQ(results[0].entry->name, L"invoice_001.pdf"); } TEST_F(SearchEngineTest, InvalidRegexReturnsEmpty) { auto entries = MakeEntries({ - { L"file.txt", L"C:\\file.txt" }, + {L"file.txt", L"C:\\file.txt"}, }); SearchOptions opts{}; opts.useRegex = true; - auto results = engine.Search(L"[invalid(regex", entries.data(), entries.size(), - opts, 100, cancel); + auto results = + engine.Search(L"[invalid(regex", entries.data(), entries.size(), opts, 100, cancel); EXPECT_TRUE(results.empty()); } @@ -132,16 +131,14 @@ TEST_F(SearchEngineTest, MaxResultsCap) { } SearchOptions opts{}; - auto results = engine.Search(L"fi", entries.data(), entries.size(), - opts, 10, cancel); + auto results = engine.Search(L"fi", entries.data(), entries.size(), opts, 10, cancel); EXPECT_LE(results.size(), 10u); } TEST_F(SearchEngineTest, QueryTooShortReturnsEmpty) { - auto entries = MakeEntries({ { L"file.txt", L"C:\\file.txt" } }); + auto entries = MakeEntries({{L"file.txt", L"C:\\file.txt"}}); SearchOptions opts{}; - auto results = engine.Search(L"f", entries.data(), entries.size(), - opts, 100, cancel); + auto results = engine.Search(L"f", entries.data(), entries.size(), opts, 100, cancel); EXPECT_TRUE(results.empty()); } @@ -154,14 +151,189 @@ TEST_F(SearchEngineTest, CancelTokenAbortsSearch) { entries.push_back(e); } - std::atomic cancelNow{true}; // already cancelled + std::atomic cancelNow{true}; // already cancelled SearchOptions opts{}; - auto results = engine.Search(L"somefile", entries.data(), entries.size(), - opts, 10000, cancelNow); + auto results = + engine.Search(L"somefile", entries.data(), entries.size(), opts, 10000, cancelNow); // With immediate cancel the result set should be small or empty EXPECT_LT(results.size(), 10000u); } +// --------------------------------------------------------------------------- +// Token-set matching tests +// --------------------------------------------------------------------------- + +static const wchar_t* kLedZepName = L"LedZep_Just-Rosy_June-Bug_guitar_4082.flac"; +static const wchar_t* kLedZepPath = L"C:\\music\\LedZep_Just-Rosy_June-Bug_guitar_4082.flac"; + +class TokenSetMatchTest : public ::testing::Test { +protected: + SearchEngine engine; + std::atomic cancel{false}; + std::vector entries = MakeEntries({ + {kLedZepName, kLedZepPath}, + {L"unrelated_song.mp3", L"C:\\music\\unrelated_song.mp3"}, + }); + SearchOptions opts{}; // caseSensitive=false, useRegex=false +}; + +TEST_F(TokenSetMatchTest, SpaceSeparatorMatchesHyphen) { + auto r = engine.Search(L"just rosy", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, UpperCaseQuerySpaceSep) { + auto r = engine.Search(L"Just rosy", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, UnderscoreSeparatorMatchesHyphen) { + auto r = engine.Search(L"just_rosy", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, MixedSeparatorsInQuery) { + auto r = engine.Search(L"just rosy june", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, NonAdjacentTokens) { + auto r = engine.Search(L"just rosy guitar", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, TokensFromDifferentParts) { + auto r = engine.Search(L"rosy guitar flac", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, LedZepPlusRosy) { + auto r = engine.Search(L"LedZep rosy", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, LedZepPlusFlac) { + auto r = engine.Search(L"ledzep flac", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, AllTokensMustMatch_NegativeCase) { + // "piano" is not in the filename — should not match + auto r = engine.Search(L"just rosy piano", entries.data(), entries.size(), opts, 100, cancel); + EXPECT_EQ(r.size(), 0u); +} + +TEST_F(TokenSetMatchTest, SingleWordQueryUsesSimdPath) { + // "ledzep" is a direct substring — still found via SIMD path + auto r = engine.Search(L"ledzep", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, ExactHyphenSubstringStillWorks) { + // "just-rosy" is a literal substring — SIMD finds it without token path + auto r = engine.Search(L"just-rosy", entries.data(), entries.size(), opts, 100, cancel); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(r[0].entry->name, kLedZepName); +} + +TEST_F(TokenSetMatchTest, CaseSensitiveModeSkipsTokenPath) { + SearchOptions caseSens{}; + caseSens.caseSensitive = true; + // lowercase "just rosy" won't match mixed-case filename in case-sensitive mode + auto r = engine.Search(L"just rosy", entries.data(), entries.size(), caseSens, 100, cancel); + EXPECT_EQ(r.size(), 0u); +} + +TEST_F(TokenSetMatchTest, PartialTokenDoesNotMatch) { + // "led" and "zep" are not independent tokens in the filename ("ledzep" is one token) + auto r = engine.Search(L"led zep", entries.data(), entries.size(), opts, 100, cancel); + EXPECT_EQ(r.size(), 0u); +} + +// --------------------------------------------------------------------------- +// TokenMatcher unit tests +// --------------------------------------------------------------------------- + +TEST(TokenMatcherTest, TokenizeBasicSplit) { + std::wstring s = L"just-rosy_june"; + auto tokens = TokenMatcher::TokenizeView(s); + ASSERT_EQ(tokens.size(), 3u); + EXPECT_EQ(tokens[0], L"just"); + EXPECT_EQ(tokens[1], L"rosy"); + EXPECT_EQ(tokens[2], L"june"); +} + +TEST(TokenMatcherTest, TokenizeConsecutiveSepsSkipped) { + std::wstring s = L"a--b"; + auto tokens = TokenMatcher::TokenizeView(s); + ASSERT_EQ(tokens.size(), 2u); + EXPECT_EQ(tokens[0], L"a"); + EXPECT_EQ(tokens[1], L"b"); +} + +TEST(TokenMatcherTest, TokenizeNoSepReturnsSingle) { + std::wstring s = L"ledzep"; + auto tokens = TokenMatcher::TokenizeView(s); + ASSERT_EQ(tokens.size(), 1u); + EXPECT_EQ(tokens[0], L"ledzep"); +} + +TEST(TokenMatcherTest, TokenizeDotHandled) { + std::wstring s = L"song.flac"; + auto tokens = TokenMatcher::TokenizeView(s); + ASSERT_EQ(tokens.size(), 2u); + EXPECT_EQ(tokens[0], L"song"); + EXPECT_EQ(tokens[1], L"flac"); +} + +TEST(TokenMatcherTest, QueryHasSeparators_True) { + EXPECT_TRUE(TokenMatcher::QueryHasSeparators(L"just rosy")); + EXPECT_TRUE(TokenMatcher::QueryHasSeparators(L"just_rosy")); + EXPECT_TRUE(TokenMatcher::QueryHasSeparators(L"just-rosy")); +} + +TEST(TokenMatcherTest, QueryHasSeparators_False) { + EXPECT_FALSE(TokenMatcher::QueryHasSeparators(L"ledzep")); + EXPECT_FALSE(TokenMatcher::QueryHasSeparators(L"guitar")); +} + +TEST(TokenMatcherTest, AllQueryTokensPresent_AllMatch) { + std::wstring q = L"just rosy"; + auto qt = TokenMatcher::TokenizeView(q); + std::sort(qt.begin(), qt.end()); + std::wstring fn = L"ledzep just rosy june bug guitar 4082 flac"; + auto ft = TokenMatcher::TokenizeView(fn); + std::sort(ft.begin(), ft.end()); + EXPECT_TRUE(TokenMatcher::AllQueryTokensPresent(qt, ft)); +} + +TEST(TokenMatcherTest, AllQueryTokensPresent_OneMissing) { + std::wstring q = L"just piano"; + auto qt = TokenMatcher::TokenizeView(q); + std::sort(qt.begin(), qt.end()); + std::wstring fn = L"ledzep just rosy june"; + auto ft = TokenMatcher::TokenizeView(fn); + std::sort(ft.begin(), ft.end()); + EXPECT_FALSE(TokenMatcher::AllQueryTokensPresent(qt, ft)); +} + +TEST(TokenMatcherTest, AllQueryTokensPresent_EmptyQueryReturnsFalse) { + std::vector emptyQ; + std::wstring fn = L"ledzep just rosy"; + auto ft = TokenMatcher::TokenizeView(fn); + std::sort(ft.begin(), ft.end()); + EXPECT_FALSE(TokenMatcher::AllQueryTokensPresent(emptyQ, ft)); +} + // SIMD detection test TEST(SimdTest, DetectCaps) { auto caps = winindex::DetectSimdCaps(); @@ -173,29 +345,29 @@ TEST(SimdTest, DetectCaps) { TEST(SimdTest, FindSubstringBasic) { std::wstring hay = L"hello world foo"; std::wstring needle = L"world"; - size_t pos = winindex::SimdFindSubstring(hay.c_str(), hay.size(), - needle.c_str(), needle.size()); + size_t pos = + winindex::SimdFindSubstring(hay.c_str(), hay.size(), needle.c_str(), needle.size()); EXPECT_EQ(pos, 6u); } TEST(SimdTest, FindSubstringNotFound) { - std::wstring hay = L"hello world"; + std::wstring hay = L"hello world"; std::wstring needle = L"xyz"; - size_t pos = winindex::SimdFindSubstring(hay.c_str(), hay.size(), - needle.c_str(), needle.size()); + size_t pos = + winindex::SimdFindSubstring(hay.c_str(), hay.size(), needle.c_str(), needle.size()); EXPECT_EQ(pos, std::wstring::npos); } TEST(SimdTest, FindSubstringAtStart) { - std::wstring hay = L"startmatch"; + std::wstring hay = L"startmatch"; std::wstring needle = L"start"; - EXPECT_EQ(winindex::SimdFindSubstring( - hay.c_str(), hay.size(), needle.c_str(), needle.size()), 0u); + EXPECT_EQ(winindex::SimdFindSubstring(hay.c_str(), hay.size(), needle.c_str(), needle.size()), + 0u); } TEST(SimdTest, FindSubstringAtEnd) { - std::wstring hay = L"helloend"; + std::wstring hay = L"helloend"; std::wstring needle = L"end"; - EXPECT_EQ(winindex::SimdFindSubstring( - hay.c_str(), hay.size(), needle.c_str(), needle.size()), 5u); + EXPECT_EQ(winindex::SimdFindSubstring(hay.c_str(), hay.size(), needle.c_str(), needle.size()), + 5u); } diff --git a/tests/test_Settings.cpp b/tests/test_Settings.cpp index 87e0ab0..5543b5c 100644 --- a/tests/test_Settings.cpp +++ b/tests/test_Settings.cpp @@ -1,4 +1,5 @@ #include + #include "settings/Settings.h" #define WIN32_LEAN_AND_MEAN #include @@ -53,10 +54,10 @@ TEST_F(SettingsTest, ManualOnlyPersists) { TEST_F(SettingsTest, SearchOptionsPersist) { settings->Load(); SearchOptions opts{}; - opts.useRegex = true; + opts.useRegex = true; opts.caseSensitive = true; - opts.wholeWord = false; - opts.matchPath = true; + opts.wholeWord = false; + opts.matchPath = true; opts.ignoreDiacritics = false; settings->SetSearchOptions(opts); settings->Save(); @@ -73,7 +74,7 @@ TEST_F(SettingsTest, SearchOptionsPersist) { TEST_F(SettingsTest, SelectedDrivesPersist) { settings->Load(); - settings->SetSelectedDrives({ L"C:\\", L"D:\\" }); + settings->SetSelectedDrives({L"C:\\", L"D:\\"}); settings->Save(); Settings s2(true, tmpDir); @@ -86,7 +87,7 @@ TEST_F(SettingsTest, SelectedDrivesPersist) { TEST_F(SettingsTest, ExcludedPathsPersist) { settings->Load(); - settings->SetExcludedPaths({ L"C:\\Windows", L"C:\\Program Files" }); + settings->SetExcludedPaths({L"C:\\Windows", L"C:\\Program Files"}); settings->Save(); Settings s2(true, tmpDir);