Bug fixes and improvements

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         exclude: '^.*\.bat$'
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v18.1.8
+    rev: v22.1.5
     hooks:
       - id: clang-format
         types_or: [c++, c]
@@ -26,6 +26,7 @@ repos:
     hooks:
       - id: cppcheck
         args:
+          - '--language=c++'
           - '--std=c++20'
           - '--enable=warning,style,performance'
           - '--suppress=missingIncludeSystem'

CMakeLists.txt

@@ -58,6 +58,8 @@ if(ENABLE_ASAN)
         message(FATAL_ERROR "ENABLE_ASAN is only supported with MSVC.")
     endif()
     add_compile_options(/fsanitize=address)
+    # /INCREMENTAL is incompatible with ASAN metadata; suppress LNK4300 by disabling it.
+    add_link_options(/INCREMENTAL:NO)
     # Abseil/RE2/GTest are built without ASAN so their STL annotation values are 0.
     # Disable all MSVC STL annotations in our code to match; otherwise LNK2038 at link.
     add_compile_definitions(

README.md

@@ -18,6 +18,7 @@ Blazing fast file search for Windows. winindex builds a full index of your local
 - **Fallback scanner** — on FAT32, or without elevation on NTFS, a fast `FindFirstFile` BFS scanner is used automatically
 - **Regex support** — powered by [RE2](https://github.com/google/re2); toggle with Alt+1
 - **SIMD-accelerated substring search** — dispatch to AVX2 or SSE4.2 when compiled with `/arch:AVX2`, scalar fallback otherwise
+- **Word-level matching** — queries with spaces, underscores, or hyphens match filenames by token set, so `just rosy guitar` finds `LedZep_Just-Rosy_June-Bug_guitar.flac` even though the words are non-adjacent and separated by different delimiters
 - **Search modes** — case-sensitive, whole-word, match full path, ignore diacritics; all togglable from the Search menu
 - **Change detection** — USN journal replay and `ReadDirectoryChangesW` watcher are wired in and ready; live background monitoring is in active development
 - **Portable mode** — place a `winindex.ini` next to the executable and all data stays in that directory
@@ -60,6 +61,7 @@ Blazing fast file search for Windows. winindex builds a full index of your local
 Each keystroke (after a 150 ms debounce) spawns a background `std::thread`:
 
 - **Substring mode** — the needle and each filename are lowercased once; `SimdFindSubstring` dispatches to the fastest available SIMD path at runtime.
+- **Token-set mode** — when the query contains a separator character (space, `_`, `-`, `.`), both the query and each filename are split into tokens and the file matches if every query token appears somewhere in the filename token set. This lets `just rosy guitar` match `LedZep_Just-Rosy_June-Bug_guitar.flac` without regex. Single-word queries skip this path entirely, keeping the SIMD fast path.
 - **Regex mode** — filenames (or full paths in match-path mode) are converted to UTF-8 and matched with a compiled `RE2` pattern.
 - Results are capped at 10 000 and rendered in a virtual `LVS_OWNERDATA` ListView for zero-copy display.
 
@@ -75,7 +77,7 @@ Each keystroke (after a 150 ms debounce) spawns a background `std::thread`:
 
 | Tool | Minimum version |
 |------|----------------|
-| Windows | 10 or 11 |
+| Windows | 10 or above |
 | Visual Studio Build Tools | 2026 (local builds); CI auto-detects 2022+ |
 | CMake | 3.28 |
 | Git | any recent |
@@ -138,7 +140,7 @@ Settings are stored in `%APPDATA%\winindex\winindex.ini` (or next to the `.exe`
 - `%APPDATA%` (Roaming)
 - `%LOCALAPPDATA%`
 
-These are merged into the saved exclusion list on every startup, so new defaults take effect on existing installs automatically.
+These are applied as the initial default on a new install. Once a user has saved their own exclusion list, that list is used as-is and the defaults are not re-injected.
 
 ---
 
@@ -166,7 +168,7 @@ winindex/
   src/
     core/
       indexer/      MftScanner, FindFileScanner, Indexer, ChangeWatcher, USN journal
-      search/       SearchEngine, SIMD search, RE2 integration
+      search/       SearchEngine, SIMD search, RE2 integration, TokenMatcher (word-level matching)
       settings/     Settings (INI), PathUtils
       storage/      IndexStore, IndexSerializer (binary format + CRC-32)
     ui/

src/core/CMakeLists.txt

@@ -7,6 +7,7 @@ add_library(winindex_core STATIC
     indexer/Indexer.cpp
     search/SearchEngine.cpp
     search/SimdSearch.cpp
+    search/TokenMatcher.cpp
     storage/IndexStore.cpp
     storage/IndexSerializer.cpp
     settings/Settings.cpp

src/core/search/SearchEngine.cpp

@@ -1,6 +1,7 @@
 #include "SearchEngine.h"
 
 #include "SimdSearch.h"
+#include "TokenMatcher.h"
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 
@@ -23,20 +24,6 @@ std::string SearchEngine::WideToUtf8(const std::wstring& s) {
     return r;
 }
 
-// Very basic diacritic normalization via NFC -> ASCII fold using WinAPI
-std::wstring SearchEngine::NormalizeDiacritics(const std::wstring& s) {
-    // FoldString with MAP_PRECOMPOSED + custom: use LCMapString for normalization
-    int needed =
-        LCMapStringEx(LOCALE_NAME_INVARIANT, LCMAP_LINGUISTIC_CASING | LCMAP_LOWERCASE, s.c_str(),
-                      static_cast<int>(s.size()), nullptr, 0, nullptr, nullptr, 0);
-    if (needed <= 0)
-        return s;
-    std::wstring result(needed, L'\0');
-    LCMapStringEx(LOCALE_NAME_INVARIANT, LCMAP_LINGUISTIC_CASING | LCMAP_LOWERCASE, s.c_str(),
-                  static_cast<int>(s.size()), result.data(), needed, nullptr, nullptr, 0);
-    return result;
-}
-
 bool SearchEngine::MatchesWholeWord(const std::wstring& text, size_t pos, size_t len) {
     auto isWordChar = [](wchar_t c) { return iswalnum(c) || c == L'_'; };
     if (pos > 0 && isWordChar(text[pos - 1]))
@@ -120,6 +107,19 @@ std::vector<SearchResult> SearchEngine::SearchSubstring(
         return q;
     }();
 
+    // Token-set matching: pre-compute once, shared read-only across threads.
+    // Only activated when the query contains separator chars (space/_/-/.)
+    // so single-word queries take the unmodified SIMD-only path.
+    const bool doTokenMatch = !options.caseSensitive && TokenMatcher::QueryHasSeparators(query);
+    // lowercaseQuery owns the storage that sortedQueryTokens views reference.
+    std::wstring lowercaseQuery;
+    std::vector<std::wstring_view> sortedQueryTokens;
+    if (doTokenMatch) {
+        lowercaseQuery = needle;  // needle is already lowercased at this point
+        sortedQueryTokens = TokenMatcher::TokenizeView(lowercaseQuery);
+        std::sort(sortedQueryTokens.begin(), sortedQueryTokens.end());
+    }
+
     unsigned int numThreads = std::max(1u, std::thread::hardware_concurrency());
     uint64_t chunkSize = (entryCount + numThreads - 1) / numThreads;
 
@@ -166,11 +166,23 @@ std::vector<SearchResult> SearchEngine::SearchSubstring(
 
                     size_t pos =
                         SimdFindSubstring(haystackData, haystackLen, needle.c_str(), needle.size());
-                    if (pos == std::wstring::npos)
-                        continue;
 
-                    if (options.wholeWord) {
-                        // Reconstruct wstring view for word-boundary check
+                    bool tokenMatch = false;
+                    if (pos == std::wstring::npos) {
+                        // Token-set fallback: fires only when SIMD missed and the query
+                        // had separators (e.g. "just rosy", "rosy guitar flac").
+                        if (!doTokenMatch)
+                            continue;
+                        std::wstring haystackStr(haystackData, haystackLen);
+                        auto fnTokens = TokenMatcher::TokenizeView(haystackStr);
+                        std::sort(fnTokens.begin(), fnTokens.end());
+                        if (!TokenMatcher::AllQueryTokensPresent(sortedQueryTokens, fnTokens))
+                            continue;
+                        tokenMatch = true;
+                        pos = 0;
+                    }
+
+                    if (options.wholeWord && !tokenMatch) {
                         std::wstring_view hayView(haystackData, haystackLen);
                         if (!MatchesWholeWord(std::wstring(hayView), pos, needle.size()))
                             continue;
@@ -179,7 +191,7 @@ std::vector<SearchResult> SearchEngine::SearchSubstring(
                     SearchResult sr;
                     sr.entry = &e;
                     sr.matchStart = static_cast<uint32_t>(pos);
-                    sr.matchLen = static_cast<uint32_t>(needle.size());
+                    sr.matchLen = tokenMatch ? 0u : static_cast<uint32_t>(needle.size());
                     local.push_back(sr);
                 }
                 return local;
@@ -191,7 +203,7 @@ std::vector<SearchResult> SearchEngine::SearchSubstring(
 
     for (auto& f : futures) {
         auto chunk = f.get();
-        for (auto& sr : chunk) {
+        for (const auto& sr : chunk) {
             if (results.size() >= maxResults)
                 goto done;
             results.push_back(sr);

src/core/search/SearchEngine.h

@@ -15,19 +15,19 @@ class SearchEngine : public ISearchEngine {
 
 private:
     // Regex search path
-    std::vector<SearchResult> SearchRegex(const std::wstring& query, const FileEntry* entries,
-                                          uint64_t entryCount, const SearchOptions& options,
-                                          uint32_t maxResults,
-                                          const std::atomic<bool>& cancelToken);
+    static std::vector<SearchResult> SearchRegex(const std::wstring& query,
+                                                 const FileEntry* entries, uint64_t entryCount,
+                                                 const SearchOptions& options, uint32_t maxResults,
+                                                 const std::atomic<bool>& cancelToken);
 
     // SIMD substring search path (parallelized)
-    std::vector<SearchResult> SearchSubstring(const std::wstring& query, const FileEntry* entries,
-                                              uint64_t entryCount, const SearchOptions& options,
-                                              uint32_t maxResults,
-                                              const std::atomic<bool>& cancelToken);
+    static std::vector<SearchResult> SearchSubstring(const std::wstring& query,
+                                                     const FileEntry* entries, uint64_t entryCount,
+                                                     const SearchOptions& options,
+                                                     uint32_t maxResults,
+                                                     const std::atomic<bool>& cancelToken);
 
     static bool MatchesWholeWord(const std::wstring& text, size_t matchPos, size_t matchLen);
-    static std::wstring NormalizeDiacritics(const std::wstring& s);
     static std::string WideToUtf8(const std::wstring& s);
 };
 

src/core/search/TokenMatcher.cpp

@@ -0,0 +1,41 @@
+#include "TokenMatcher.h"
+
+#include <algorithm>
+
+namespace winindex {
+namespace TokenMatcher {
+
+bool QueryHasSeparators(const std::wstring& query) {
+    for (wchar_t c : query)
+        if (IsTokenSep(c))
+            return true;
+    return false;
+}
+
+std::vector<std::wstring_view> TokenizeView(const std::wstring& s) {
+    std::vector<std::wstring_view> tokens;
+    const std::wstring_view sv(s);
+    size_t start = 0;
+    for (size_t i = 0; i <= sv.size(); ++i) {
+        if (i == sv.size() || IsTokenSep(sv[i])) {
+            if (i > start)
+                tokens.emplace_back(sv.substr(start, i - start));
+            start = i + 1;
+        }
+    }
+    return tokens;
+}
+
+bool AllQueryTokensPresent(const std::vector<std::wstring_view>& sortedQueryTokens,
+                           const std::vector<std::wstring_view>& sortedFilenameTokens) {
+    if (sortedQueryTokens.empty())
+        return false;
+    for (const auto& qt : sortedQueryTokens) {
+        if (!std::binary_search(sortedFilenameTokens.begin(), sortedFilenameTokens.end(), qt))
+            return false;
+    }
+    return true;
+}
+
+}  // namespace TokenMatcher
+}  // namespace winindex

src/core/search/TokenMatcher.h

@@ -0,0 +1,29 @@
+#pragma once
+#include <cwctype>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace winindex {
+namespace TokenMatcher {
+
+inline bool IsTokenSep(wchar_t c) {
+    return c == L' ' || c == L'_' || c == L'-' || c == L'.';
+}
+
+// Returns true if query contains at least one separator character.
+// Used to gate token-set matching — single-word queries skip this path.
+bool QueryHasSeparators(const std::wstring& query);
+
+// Split a (pre-lowercased) wstring on separator chars into wstring_view slices.
+// Consecutive separators produce no empty tokens.
+// The returned views reference `s`, which must outlive them.
+std::vector<std::wstring_view> TokenizeView(const std::wstring& s);
+
+// Returns true if every token in sortedQueryTokens is present in
+// sortedFilenameTokens (exact equality). Both must be pre-sorted.
+bool AllQueryTokensPresent(const std::vector<std::wstring_view>& sortedQueryTokens,
+                           const std::vector<std::wstring_view>& sortedFilenameTokens);
+
+}  // namespace TokenMatcher
+}  // namespace winindex

src/core/settings/Settings.cpp

@@ -41,9 +41,7 @@ void Settings::Load() {
                 m_selectedDrives.push_back(token);
     }
 
-    // Excluded paths: load user-configured list, then merge in system defaults.
-    // System defaults (AppData, Windows, etc.) are always enforced so that
-    // adding new defaults in a code update takes effect on existing installs.
+    // Excluded paths: use saved list if present, otherwise seed with defaults.
     std::wstring exclStr = ReadString(L"Indexing", L"ExcludedPaths");
     if (exclStr.empty()) {
         m_excludedPaths = DefaultExcludedPaths();
@@ -54,20 +52,6 @@ void Settings::Load() {
         while (std::getline(ss, token, L'|'))
             if (!token.empty())
                 m_excludedPaths.push_back(token);
-
-        // Merge in any defaults not already present so new defaults take effect
-        // on existing installs without requiring a manual settings reset.
-        for (const auto& def : DefaultExcludedPaths()) {
-            bool found = false;
-            for (const auto& e : m_excludedPaths) {
-                if (_wcsicmp(e.c_str(), def.c_str()) == 0) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                m_excludedPaths.push_back(def);
-        }
     }
 
     m_reindexIntervalHours = static_cast<uint64_t>(

tests/mocks/MockFileSystemScanner.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <gmock/gmock.h>
+
 #include "indexer/IFileSystemScanner.h"
 
 namespace winindex {
@@ -8,11 +9,9 @@ class MockFileSystemScanner : public IFileSystemScanner {
 public:
     MOCK_METHOD(bool, IsMftAvailable, (const std::wstring& root), (const, override));
     MOCK_METHOD(void, Scan,
-                (const ScanOptions& options,
-                 ScanCallback onFile,
-                 ProgressCallback onProgress,
+                (const ScanOptions& options, ScanCallback onFile, ProgressCallback onProgress,
                  const std::atomic<bool>& cancelToken),
                 (override));
 };
 
-} // namespace winindex
+}  // namespace winindex

tests/mocks/MockIndexStore.h

@@ -1,26 +1,25 @@
 #pragma once
 #include <gmock/gmock.h>
+
 #include "storage/IIndexStore.h"
 
 namespace winindex {
 
 class MockIndexStore : public IIndexStore {
 public:
-    MOCK_METHOD(bool,            IsIndexValid,  (),                                (const, override));
-    MOCK_METHOD(void,            Load,          (),                                (override));
-    MOCK_METHOD(void,            Save,          (),                                (override));
-    MOCK_METHOD(void,            BeginWrite,    (),                                (override));
-    MOCK_METHOD(void,            AddEntry,      (const FileEntry& e),              (override));
-    MOCK_METHOD(void,            EndWrite,      (),                                (override));
-    MOCK_METHOD(void,            ApplyAdd,      (const FileEntry& e),              (override));
-    MOCK_METHOD(void,            ApplyRemove,   (const std::wstring& path),        (override));
-    MOCK_METHOD(void,            ApplyRename,   (const std::wstring& o,
-                                                 const std::wstring& n),           (override));
-    MOCK_METHOD(uint64_t,        GetEntryCount, (),                                (const, override));
-    MOCK_METHOD(const FileEntry*, GetEntries,   (),                                (const, override));
-    MOCK_METHOD(uint64_t,        GetSavedUsn,   (const std::wstring& root),        (const, override));
-    MOCK_METHOD(void,            SetSavedUsn,   (const std::wstring& root,
-                                                 uint64_t usn),                    (override));
+    MOCK_METHOD(bool, IsIndexValid, (), (const, override));
+    MOCK_METHOD(void, Load, (), (override));
+    MOCK_METHOD(void, Save, (), (override));
+    MOCK_METHOD(void, BeginWrite, (), (override));
+    MOCK_METHOD(void, AddEntry, (const FileEntry& e), (override));
+    MOCK_METHOD(void, EndWrite, (), (override));
+    MOCK_METHOD(void, ApplyAdd, (const FileEntry& e), (override));
+    MOCK_METHOD(void, ApplyRemove, (const std::wstring& path), (override));
+    MOCK_METHOD(void, ApplyRename, (const std::wstring& o, const std::wstring& n), (override));
+    MOCK_METHOD(uint64_t, GetEntryCount, (), (const, override));
+    MOCK_METHOD(const FileEntry*, GetEntries, (), (const, override));
+    MOCK_METHOD(uint64_t, GetSavedUsn, (const std::wstring& root), (const, override));
+    MOCK_METHOD(void, SetSavedUsn, (const std::wstring& root, uint64_t usn), (override));
 };
 
-} // namespace winindex
+}  // namespace winindex

tests/mocks/MockUsnJournalMonitor.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <gmock/gmock.h>
+
 #include "indexer/IUsnJournalMonitor.h"
 
 namespace winindex {
@@ -8,12 +9,11 @@ class MockUsnJournalMonitor : public IUsnJournalMonitor {
 public:
     MOCK_METHOD(bool, IsAvailable, (const std::wstring& root), (const, override));
     MOCK_METHOD(uint64_t, ReplaySince,
-                (const std::wstring& root, uint64_t savedUsn, ChangeCallback onChange),
-                (override));
+                (const std::wstring& root, uint64_t savedUsn, ChangeCallback onChange), (override));
     MOCK_METHOD(void, StartMonitoring,
-                (const std::wstring& root, uint64_t startUsn,
-                 ChangeCallback onChange, const std::atomic<bool>& stopToken),
+                (const std::wstring& root, uint64_t startUsn, ChangeCallback onChange,
+                 const std::atomic<bool>& stopToken),
                 (override));
 };
 
-} // namespace winindex
+}  // namespace winindex

tests/test_DriveEnumerator.cpp

@@ -1,4 +1,5 @@
 #include <gtest/gtest.h>
+
 #include "indexer/DriveEnumerator.h"
 
 using namespace winindex;