fix(index): make a crashed worker's log survive and name the run

src/foundation/log.c

@@ -32,6 +32,25 @@ static _Atomic CBMLogFormat g_log_format = CBM_LOG_FORMAT_TEXT;
 static _Atomic cbm_log_sink_fn g_log_sink = (cbm_log_sink_fn)NULL;
 static _Atomic CBMLogSinkMode g_log_sink_mode = CBM_LOG_SINK_REPLACE;
 
+/* See cbm_log_set_crash_durable in log.h. Read on every emitted line, so it
+ * follows the same relaxed-atomic discipline as the four above. */
+static _Atomic bool g_log_crash_durable = false;
+
+void cbm_log_set_crash_durable(bool enabled) {
+    if (enabled) {
+        /* Best effort by contract: setvbuf is only guaranteed before a stream's
+         * first operation, so a process that has already written to stderr
+         * keeps its buffering. The per-line flush in emit_line is what makes
+         * the durability guarantee hold either way. */
+        (void)setvbuf(stderr, NULL, _IONBF, 0);
+    }
+    atomic_store_explicit(&g_log_crash_durable, enabled, memory_order_relaxed);
+}
+
+bool cbm_log_crash_durable(void) {
+    return atomic_load_explicit(&g_log_crash_durable, memory_order_relaxed);
+}
+
 /* CBM_LOG_LEVEL support — distilled from #414 (closes #413, thanks @santanusinha). */
 void cbm_log_init_from_env(void) {
     /* getenv() is safe here: this runs at startup before any thread is created,
@@ -225,6 +244,13 @@ static void emit_line(const char *line) {
         }
     }
     (void)fprintf(stderr, "%s\n", line);
+    if (atomic_load_explicit(&g_log_crash_durable, memory_order_relaxed)) {
+        /* The line is complete here and the stream lock is released, so a
+         * process that dies on the very next instruction still leaves this
+         * line on disk. Free when stderr is unbuffered; one write() per line
+         * when setvbuf was refused. */
+        (void)fflush(stderr);
+    }
 }
 
 void cbm_log(CBMLogLevel level, const char *msg, ...) {

src/foundation/log.h

@@ -92,6 +92,29 @@ void cbm_log_mcp_request(const char *method, const char *tool_name, bool is_erro
 void cbm_log_http_request(const char *component, const char *method, const char *path, int status,
                           int64_t duration_ms, size_t request_bytes, size_t response_bytes);
 
+/* Crash-durable log stream.
+ *
+ * Enable in a process whose stderr is redirected to a FILE that has to survive
+ * the process dying abnormally — today that is the supervised index worker,
+ * whose `.worker-*.log` is the only post-mortem evidence a contained crash,
+ * SIGKILL or hang leaves behind. Default stdio buffering loses exactly that
+ * evidence: the C standard only promises stderr is "not fully buffered", and
+ * the Windows CRT gives a redirected stderr FULL buffering, so a worker that
+ * aborts or is killed takes its whole diagnostic with it and the user is left
+ * holding a 0-byte log (#1070, #1130, #1132, #1133, #1145, #1450).
+ *
+ * Two mechanisms, deliberately both: setvbuf(_IONBF) covers EVERY writer to
+ * the stream (including the plain fprintf(stderr, …) startup errors that
+ * explain a worker which never got as far as logging), and a per-line flush
+ * covers the case where setvbuf is refused because the stream was already
+ * written to — it is only guaranteed before a stream's first operation.
+ *
+ * Also the process-wide answer to "is this log post-mortem evidence?", which
+ * is what makes the per-file breadcrumb worth its volume in a worker and not
+ * anywhere else. Cost is ~0: flushing an unbuffered stream writes nothing. */
+void cbm_log_set_crash_durable(bool enabled);
+bool cbm_log_crash_durable(void);
+
 /* Optional log sink callback — called with the formatted log line. */
 typedef void (*cbm_log_sink_fn)(const char *line);
 void cbm_log_set_sink(cbm_log_sink_fn fn);

src/main.c

@@ -2398,6 +2398,14 @@ int main(int argc, char **argv) {
     }
 #endif
     cbm_daemon_process_role_t role = cbm_daemon_process_role(argc, argv);
+    if (role == CBM_DAEMON_PROCESS_WORKER) {
+        /* Before this process writes ANYTHING. A worker's stderr is a file the
+         * supervisor keeps for post-mortem, and setvbuf only binds before a
+         * stream's first operation — claim it here so even the "could not
+         * start" messages below reach disk. The header follows once the argv
+         * grammar has been validated. */
+        cbm_log_set_crash_durable(true);
+    }
     if (role == CBM_DAEMON_PROCESS_INVALID) {
         (void)fprintf(stderr, "codebase-memory-mcp: invalid internal process arguments\n");
         return EXIT_FAILURE;
@@ -2610,6 +2618,13 @@ int main(int argc, char **argv) {
                           cbm_index_worker_argv_status_message(worker_status));
             return EXIT_FAILURE;
         }
+        /* First thing a worker records, and the only thing six 0-byte-log
+         * reports were missing: who I am, what I was asked to index, with what
+         * arguments. Everything below here can crash and the log still names
+         * the run. */
+        char *worker_repo_path = cbm_mcp_get_string_arg(invocation.args_json, "repo_path");
+        cbm_index_worker_log_begin(invocation.args_json, worker_repo_path);
+        free(worker_repo_path);
         cbm_daemon_ipc_endpoint_t *worker_endpoint = cbm_daemon_bootstrap_endpoint_new(NULL);
         cbm_project_lock_manager_t *worker_project_locks =
             worker_endpoint ? cbm_project_lock_manager_new(worker_endpoint) : NULL;

src/mcp/index_supervisor.c

@@ -31,6 +31,12 @@
 #define worker_getpid getpid
 #endif
 
+/* Same release-injected macro (and same fallback) main.c and cli.c use; the
+ * worker log's header must name the build the user actually ran. */
+#ifndef CBM_VERSION
+#define CBM_VERSION "dev"
+#endif
+
 _Static_assert(CBM_INDEX_WORKER_BUILD_FINGERPRINT_SIZE == CBM_DAEMON_BUILD_FINGERPRINT_SIZE,
                "worker and daemon build fingerprint sizes must match");
 
@@ -84,6 +90,28 @@ size_t cbm_index_worker_memory_budget_bytes(void) {
     return g_worker_memory_budget_bytes;
 }
 
+/* Worker log startup header — see index_supervisor.h. */
+static atomic_flag g_worker_log_begun = ATOMIC_FLAG_INIT;
+
+void cbm_index_worker_log_begin(const char *args_json, const char *repo_path) {
+    /* Durability first, header second: if the header itself is the last thing
+     * this process ever manages to write, it must already be on disk. */
+    cbm_log_set_crash_durable(true);
+    if (atomic_flag_test_and_set_explicit(&g_worker_log_begun, memory_order_relaxed)) {
+        return; /* the CLI arg parser re-installs the worker role; header once */
+    }
+    char pid_text[CBM_SZ_32];
+    (void)snprintf(pid_text, sizeof(pid_text), "%ld", (long)worker_getpid());
+    const char *build = cbm_index_supervisor_build_fingerprint();
+    /* A control record, not an info line: a user who set CBM_LOG_LEVEL to warn
+     * or error would otherwise still hand us a 0-byte log, which is the whole
+     * defect. JSON-encoded, so a repo path with spaces or a quote survives. */
+    cbm_log_control(CBM_INDEX_WORKER_LOG_START_EVENT, "version", CBM_VERSION, "build",
+                    build ? build : "", "pid", pid_text, "repo_path", repo_path ? repo_path : "",
+                    "args", args_json ? args_json : "");
+    (void)fflush(stderr);
+}
+
 static bool worker_fingerprint_valid(const char *fingerprint);
 
 bool cbm_index_supervisor_capture_build_fingerprint(void) {

src/mcp/index_supervisor.h

@@ -49,6 +49,27 @@ bool cbm_index_worker_active(void);
 const char *cbm_index_worker_response_out(void);
 size_t cbm_index_worker_memory_budget_bytes(void);
 
+/* Event name of the worker log's startup header. Shared with the tests so the
+ * contract has exactly one spelling. */
+#define CBM_INDEX_WORKER_LOG_START_EVENT "index.worker.start"
+
+/* Worker-side: open the worker log for post-mortem use. Call this FIRST in a
+ * process admitted by the worker argv grammar, before any indexing work.
+ *
+ * Six reports (#1070, #1130, #1132, #1133, #1145, #1450) describe a worker that
+ * died and left a log of 0 bytes: nothing was ever flushed, so every one of them
+ * is unreproducible and unattributable. This makes the log always say something:
+ *   1. the stream becomes crash-durable (see cbm_log_set_crash_durable), so a
+ *      crash, a SIGKILL or a hang can no longer swallow what was written;
+ *   2. a startup header — version, build fingerprint, pid, repo path and the
+ *      worker's own arguments — is written and flushed synchronously, so even a
+ *      worker that dies before its first unit of work is identifiable.
+ *
+ * It fixes no crash. It converts an empty file into a report we can act on.
+ * Idempotent: the worker role is installed twice (process entry, then the CLI
+ * arg parser), and the header is written once. */
+void cbm_index_worker_log_begin(const char *args_json, const char *repo_path);
+
 /* Capture the exact executable-image fingerprint once, during process startup
  * before any worker can be launched. Repeated calls return the original capture
  * and never re-hash a pathname that an installer may since have replaced. */

src/pipeline/pass_parallel.c

@@ -827,8 +827,16 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
         }
 
         /* Per-file start log: shows which file each worker is processing.
-         * Critical for diagnosing stuck workers on large vendored files. */
-        if (sort_pos < PP_LOG_THRESH) { /* first 2 rounds of workers = most interesting */
+         * Critical for diagnosing stuck workers on large vendored files.
+         *
+         * Under a crash-durable log (i.e. a supervised worker) EVERY file gets
+         * its line, not just the first rounds. That log is the only evidence a
+         * contained crash or a kill leaves behind, and #1145/#1130 are
+         * unattributable precisely because it never named the file that was in
+         * flight — the run ends with the culprit still on the last lines. One
+         * line per file, never per node. */
+        if (sort_pos < PP_LOG_THRESH || /* first 2 rounds of workers = most interesting */
+            cbm_log_crash_durable()) {
             cbm_log_info("parallel.extract.file.start", "pos", itoa_log(sort_pos), "size_kb",
                          itoa_log(source_len / CBM_SZ_1K), "path", fi->rel_path);
         }

tests/test_index_supervisor.c

@@ -762,11 +762,99 @@ TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained) {
     PASS();
 }
 
+/* #1070, #1130, #1132, #1133, #1145, #1450: six reports of an indexing worker
+ * that died leaving "the worker log file is completely empty (0 KB)". Nothing
+ * was ever flushed, so not one of them is reproducible or attributable — the
+ * hint says "crashed on a file" and the file is never named.
+ *
+ * The repro: the worker starts with a FULLY BUFFERED stderr (what the Windows
+ * CRT hands a redirected stderr; tf_maybe_run_index_worker forces the same
+ * state on POSIX so this binds on all three legs), writes diagnostics, then is
+ * SIGKILLed — #1070's own `signal=9`, and the death that runs no cleanup and
+ * flushes nothing. The supervisor keeps the log of a failed worker, so the log
+ * on disk afterwards is exactly what a user would attach to an issue.
+ *
+ * Asserted: it is not empty, it carries the startup header with the version,
+ * pid, repo path and the worker's own arguments, and the line written after the
+ * header survived too. Reverting the fix leaves the file at 0 bytes. */
+TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run) {
+    char cache[INDEX_SUPERVISOR_TEST_PATH_CAP];
+    (void)snprintf(cache, sizeof(cache), "%s/cbm-index-logheader-XXXXXX", cbm_tmpdir());
+    ASSERT_NOT_NULL(cbm_mkdtemp(cache));
+    const char *old_cache = getenv("CBM_CACHE_DIR");
+    char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL;
+    (void)cbm_setenv("CBM_CACHE_DIR", cache, 1);
+
+    /* A repo path with a space: the header must survive JSON-escaping intact,
+     * because Windows reporters index paths like C:/Users/Some Name/repo. */
+    char repo_path[INDEX_SUPERVISOR_TEST_PATH_CAP];
+    (void)snprintf(repo_path, sizeof(repo_path), "%s/some repo", cache);
+    char args[INDEX_SUPERVISOR_TEST_PATH_CAP];
+    (void)snprintf(args, sizeof(args),
+                   "{\"__cbm_test_worker\":\"buffered-kill\",\"repo_path\":\"%s\"}", repo_path);
+
+    cbm_index_worker_handle_t *handle = NULL;
+    int start_rc = cbm_index_worker_start(args, 0, false, NULL, NULL, &handle);
+    char log_path[INDEX_SUPERVISOR_TEST_PATH_CAP] = {0};
+    if (handle) {
+        (void)snprintf(log_path, sizeof(log_path), "%s", cbm_index_worker_log_path(handle));
+    }
+    const cbm_index_worker_result_t *result = NULL;
+    bool terminal = handle && index_supervisor_test_poll_terminal(
+                                  handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result);
+    cbm_proc_outcome_t outcome = result ? result->outcome : CBM_PROC_SPAWN_FAILED;
+
+    /* Read the log the way a reporter would: after the worker is gone. */
+    long log_size = log_path[0] ? cbm_file_size(log_path) : -1;
+    char log_text[8192] = {0};
+    FILE *log = log_path[0] ? cbm_fopen(log_path, "rb") : NULL;
+    if (log) {
+        size_t used = fread(log_text, 1, sizeof(log_text) - 1, log);
+        log_text[used] = '\0';
+        (void)fclose(log);
+    }
+    bool has_header =
+        strstr(log_text, "\"event\":\"" CBM_INDEX_WORKER_LOG_START_EVENT "\"") != NULL;
+    bool names_repo =
+        strstr(log_text, "\"repo_path\":\"") != NULL && strstr(log_text, "some repo") != NULL;
+    bool names_args =
+        strstr(log_text, "\"args\":\"") != NULL && strstr(log_text, "buffered-kill") != NULL;
+    bool names_version = strstr(log_text, "\"version\":\"") != NULL;
+    bool names_pid = strstr(log_text, "\"pid\":\"") != NULL;
+    bool kept_post_header_line = strstr(log_text, "index.worker.buffered_kill_probe") != NULL;
+
+    if (terminal) {
+        cbm_index_worker_destroy(handle);
+    } else {
+        index_supervisor_test_dump("buffered-kill worker log", log_path);
+        index_supervisor_test_cleanup_handle(handle);
+    }
+    if (!has_header) {
+        index_supervisor_test_dump("buffered-kill worker log", log_path);
+    }
+    (void)cbm_unlink(log_path);
+    index_supervisor_test_restore_env("CBM_CACHE_DIR", saved_cache);
+    (void)th_rmtree(cache);
+
+    ASSERT_EQ(start_rc, 0);
+    ASSERT_TRUE(terminal);
+    ASSERT_TRUE(outcome != CBM_PROC_CLEAN); /* killed: the supervisor keeps the log */
+    ASSERT_TRUE(log_size > 0);              /* the 0-byte log of the six reports */
+    ASSERT_TRUE(has_header);
+    ASSERT_TRUE(names_version);
+    ASSERT_TRUE(names_pid);
+    ASSERT_TRUE(names_repo);
+    ASSERT_TRUE(names_args);
+    ASSERT_TRUE(kept_post_header_line);
+    PASS();
+}
+
 SUITE(index_supervisor) {
     RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar);
     RUN_TEST(index_supervisor_async_jobs_are_isolated_cancellable_and_terminal_cached);
     RUN_TEST(index_supervisor_sync_wrapper_forwards_cancel_and_drains_tree);
     RUN_TEST(index_supervisor_terminal_log_lifecycle_matches_outcome_and_profiling);
     RUN_TEST(index_supervisor_drains_terminal_backlog_into_request_progress_callback);
     RUN_TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained);
+    RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run);
 }

tests/test_main.c

@@ -13,6 +13,8 @@ int tf_skip_count = 0;
 #include "test_daemon_runtime_contract.h"
 #include "foundation/compat.h"     /* cbm_setenv — #845 supervisor kill switch */
 #include "foundation/compat_fs.h"  /* cbm_fopen — worker response file */
+#include "foundation/constants.h"  /* CBM_SZ_4K — forced stderr buffer */
+#include "foundation/log.h"        /* crash-durable worker log probe */
 #include "foundation/mem.h"        /* cbm_mem_init — worker budget */
 #include "foundation/platform.h"   /* cbm_file_exists — blocking-git marker */
 #include "daemon/runtime.h"        /* bounded worker response probe */
@@ -234,6 +236,27 @@ static void tf_index_worker_probe(const char *args_json, const char *response_ou
         fflush(NULL);
         abort();
     }
+    if (strstr(args_json, "\"buffered-kill\"")) {
+        /* The 0-byte-worker-log repro. tf_maybe_run_index_worker has already
+         * put stderr into the FULL buffering a redirected stderr gets from the
+         * Windows CRT (see there), so this line only reaches the log if the
+         * production worker-log entry made the stream crash-durable.
+         *
+         * Then die the way the reports die. NOT abort(): Darwin's abort() runs
+         * the stdio cleanup handler, so it flushes the very buffer this probe
+         * exists to strand — under abort the reverted build still produced a
+         * populated log and the repro was silently toothless. SIGKILL cannot be
+         * caught, blocked or handled, so no cleanup of any kind runs. It is
+         * also literally #1070's death (`signal=9`) and how #1130's hung worker
+         * is terminated. */
+        cbm_log_info("index.worker.buffered_kill_probe", "phase", "before_kill");
+#ifdef _WIN32
+        TerminateProcess(GetCurrentProcess(), 9);
+#else
+        (void)raise(SIGKILL);
+#endif
+        _Exit(2); /* unreachable: neither primitive returns */
+    }
     if (strstr(args_json, "\"oversize\"")) {
         FILE *response = response_out ? cbm_fopen(response_out, "wb") : NULL;
         bool written = false;
@@ -299,6 +322,23 @@ static int tf_maybe_run_index_worker(int argc, char **argv) {
         return 1;
     }
 
+    /* WHY force full buffering: on POSIX stderr is unbuffered by default, so the
+     * 0-byte worker log of #1070/#1130/#1132/#1133/#1145/#1450 is invisible on
+     * two thirds of the ladder — the Windows CRT is what gives a redirected
+     * stderr FULL buffering. Starting the probe from the Windows default makes
+     * the crash-durability contract testable identically on every OS we own,
+     * instead of a Windows-only claim nobody can run locally. Scoped to the one
+     * probe that asserts it, and set before the production entry below, which is
+     * the code under test. */
+    static char tf_worker_forced_buffer[CBM_SZ_4K];
+    if (invocation.args_json && strstr(invocation.args_json, "\"buffered-kill\"")) {
+        (void)setvbuf(stderr, tf_worker_forced_buffer, _IOFBF, sizeof(tf_worker_forced_buffer));
+    }
+    /* Mirror the production worker entry (run_cli's caller in main.c): the log
+     * header is the first thing a worker records. */
+    char *worker_repo_path = cbm_mcp_get_string_arg(invocation.args_json, "repo_path");
+    cbm_index_worker_log_begin(invocation.args_json, worker_repo_path);
+    free(worker_repo_path);
     cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread,
                                       invocation.marker_file, invocation.quarantine_file,
                                       invocation.memory_budget_bytes);