correctly summarize ws fixes

Author: lws-team

CMakeLists.txt

@@ -104,4 +104,14 @@ add_test(NAME fixdiff7
 		-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/runtest.cmake
 	 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/7)
 
+add_test(NAME fixdiff8
+	 COMMAND ${CMAKE_COMMAND}
+	 	-DCMD=$<TARGET_FILE:${PROJECT_NAME}>
+		-DSRC=deaddrop.js
+		-DPATCH=gemini.patch
+		-DEXPSHA=0549dec027cd6e589d998c4fc423beaab919a10a563e4f3e770c952dc3b0c55c
+		-DEXPSHA_WIN=2e6b9b12ae0128c9edfc109744b9c67848712b0521c322a45104895aa4cbc3b1
+		-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/runtest.cmake
+	 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/8)
+
 

README.md

@@ -33,17 +33,194 @@ headers with accurate line counts on stdout.
 It silently repairs:
 
  1. New empty + lines with only whitespace are rewritten to be empty blank lines
+
+Example:
+  
+```
+diff --git a/deaddrop.js b/deaddrop.js
+index 8f804f0..7913254 100644
+--- a/deaddrop.js
++++ b/deaddrop.js
+@@ -165,13 +165,21 @@
+                    ts = d.getFullYear() + '-' + pad(d.getMonth() + 1) + '-' +
+                         pad(d.getDate()) + '_' + pad(d.getHours()) + '-' +
+                         pad(d.getMinutes()) + '-' + pad(d.getSeconds()),
++<tab>
+                    formData = new FormData(), blob;
+...
+```
+
+The added line is rewritten to be an empty blank line.
+
  2. Diff stanzas that do not contain any +/- lines are removed
+ 
+Example:
+ 
+```
+--- a/deaddrop.js
++++ b/deaddrop.js
+@@ -3,6 +3,11 @@
+        var server_max_size = 0, username = "", ws;
+
+        function san(s)
+        {
+```
+
  3. Original lines in diff that differ from real line in file only by
     whitespace are rewritten to contain the correct whitespace
+
+Example: file contains `<tab><tab>abc`
+
+```
+--- a/deaddrop.js
++++ b/deaddrop.js
+@@ -3,6 +3,11 @@
+ <space><space>abc
+...
+```
+
+The output patch is rewritten to match what is already in the file at that
+line for whitespace, so the output patch contains `<tab><tab>abc`
+
  4. All stanza header line offsets and counts are recomputed from the actual
     match in the original source and counting before and after lines in the diff,
     the incoming @@ line is completely ignored and rewritten with actual info
+
+Example incoming patch stanza headers can be nonsense
+
+```
+--- a/deaddrop.js
++++ b/deaddrop.js
+@@ -123,16 +345,5 @@
+ <space><space>abc
+...
+```
+
+The correct headers will be rewritten in place of the wrong ones.
+
  5. Extra lead-in context lines to stanza by removing until only 3
+ 
+```
+--- a/deaddrop.js
++++ b/deaddrop.js
+@@ -3,6 +3,11 @@
+                                                "<tr><th>User</th><th>IP Address</th>" +
+                                                "<th>Platform</th><th>Client</th></tr>";
+
+                                        for (n = 0; n < j.connected_users.length; n++) {
+                                                var u = j.connected_users[n];
+                                                s_users += "<tr><td>" + san(u.user) +
+                                                        "</td><td>" + san(u.ip) +
+                                                        "</td><td>" + san(u.platform) +
+                                                        "</td><td>" + san(u.browser) +
+                                                        "</td></tr>";
+                                        }
+                                        s_users += "</table>";
+                                        t_users.innerHTML = s_users;
+                                }
++                       };
++
++                       ws.onclose = function() {
+...
+```
+
+This will be rewritten to reduce the lead-in to the normal 3
+
+```
+--- a/deaddrop.js
++++ b/deaddrop.js
+@@ -14,6 +14,11 @@
+                                        s_users += "</table>";
+                                        t_users.innerHTML = s_users;
+                                }
++                       };
++
++                       ws.onclose = function() {
+...
+```
+ 
  6. Excessive lead-out-context is removed, missing lead-out context is added.
     Diffs adding to EOF with missing or wrong context caused by
     LLM losing blank lines at the original EOF are rewritten by checking
     the original source file for extra lines and adding them as needed.
+
+Example 1: excessive led-out removed
+
+```
+...
+                                        s_users += "</table>";
+                                        t_users.innerHTML = s_users;
+                                }
++                       };
++
++                       ws.onclose = function() {
+                                var u = j.connected_users[n];
+                                s_users += "<tr><td>" + san(u.user) +
+                                        "</td><td>" + san(u.ip) +
+                                        "</td><td>" + san(u.platform) +
+                                        "</td><td>" + san(u.browser) +
+                                        "</td></tr>";
+```
+
+This will be trimmed to
+
+```
+...
+                                        s_users += "</table>";
+                                        t_users.innerHTML = s_users;
+                                }
++                       };
++
++                       ws.onclose = function() {
+                                var u = j.connected_users[n];
+                                s_users += "<tr><td>" + san(u.user) +
+                                        "</td><td>" + san(u.ip) +
+```
+
+Sometimes at EOT, the LLM does not know what is in the file properly, this leads
+to missing lead-out context.
+
+Example 2:  missing EOT context
+
+Actual file ending
+
+```
+...
+A
+B
+<cr>
+<cr>
+```
+
+patch:
+
+```
+--- a/deaddrop.js
++++ b/deaddrop.js
+@@ -14,6 +14,11 @@
+ A
+ B
++C
++D
+...
+```
+
+fixdiff will realize the situation and fix the stanza by fetching the extra 
+lines from the original file and adding them as context at the end.
+
+```
+--- a/deaddrop.js
++++ b/deaddrop.js
+@@ -14,6 +14,11 @@
+ A
+ B
++C
++D
+ <cr>
+ <cr>
+...
+```
+
  7. Unexpected blank lines in a stanza (without space, + or -) are either ignored
     if happening at the end of the stanza, or rewritten to be context by adding
     a space at the beginning, if the normal diff resumes.

fixdiff.c

@@ -116,6 +116,9 @@ typedef struct {
 
 	int		pending_empty_lines;
 
+	int		whitespace_corrected[64];
+	int		count_whitespace_corrected;
+
 	char		ongoing;
 	char		skip_this_one;
 	char		lead_in_active;
@@ -380,7 +383,7 @@ fixdiff_find_original(dp_t *pdp, int *line_start)
 			elog("Unable to skip temp lines\n");
 			return 1;
 		}
-		elog("Stanza %d: removing extra lead-in\n", pdp->stanzas);
+		elog("    stanza %d: removing extra lead-in\n", pdp->stanzas);
 		pdp->lead_in--;
 		pdp->lead_in_corrected++;
 		pdp->pre--;
@@ -399,13 +402,16 @@ fixdiff_find_original(dp_t *pdp, int *line_start)
 		return 1;
 	}
 
+	pdp->count_whitespace_corrected = 0;
+
 	/*
 	 * Outer loop walks through each line in source.
 	 * Inner loop tries to match starting from that line
 	 */
 
 	while (!hit) {
 		line_ending_t let, les;
+		int n;
 
 		init_lbuf(&lb, "src_comp");
 		lb.fd = open(pdp->pf, OFLAGS(O_RDONLY));
@@ -437,7 +443,8 @@ fixdiff_find_original(dp_t *pdp, int *line_start)
 				break;
 
 			if (!ls) {
-				elog("Failed to match, best chunk %d lines started at %s:%d (tabs shown below as >)\n",
+				elog("**** Failed to match, best chunk %d lines started at %s:%d "
+				     "(tabs shown below as >)\n",
 				     lmc, pdp->pf, lg_lis);
 				elog("last match: patch = '%s"
 				     "',         source = '%s'\n", b1, b2);
@@ -484,7 +491,12 @@ fixdiff_find_original(dp_t *pdp, int *line_start)
 				if ((p1 < p1_end) != (p2 < p2_end))
 					goto record_breakage;
 
-				elog("(fixable whitespace-only difference at stanza line %d)\n", lb_temp.li);
+				for (n = 0; n < pdp->count_whitespace_corrected; n++)
+					if (pdp->whitespace_corrected[n] == lb_temp.li)
+						break;
+				if (n == pdp->count_whitespace_corrected &&
+				    pdp->count_whitespace_corrected < sizeof(pdp->whitespace_corrected) / sizeof(int))
+					pdp->whitespace_corrected[pdp->count_whitespace_corrected++] = lb_temp.li;
 
 				/*
 				 * We have to take care about picking up windows _TEXT
@@ -592,8 +604,7 @@ fixdiff_find_original(dp_t *pdp, int *line_start)
 				}
 
 				if (lea != LE_ZERO)
-					if (write(lb_temp.fd, "\n", TO_POSLEN(1)) !=
-						  (ssize_t)1) {
+					if (write(lb_temp.fd, "\n", TO_POSLEN(1)) != (ssize_t)1) {
 						close(lb_ef.fd);
 						pdp->reason = "failed to write extra "
 								"stanza trailer to temp file";
@@ -608,12 +619,16 @@ fixdiff_find_original(dp_t *pdp, int *line_start)
 			}
 
 			if (a)
-				elog("Stanza %d: detected patch at EOF: "
+				elog("    stanza %d: detected patch at EOF: "
 						  "added %d context at end\n",
 					pdp->stanzas, a);
 
 			close(lb_ef.fd);
 		}
+
+		if (pdp->count_whitespace_corrected)
+			elog("    stanza %d: fixed %d lines with whitespace-only fuzz\n",
+			     pdp->stanzas, pdp->count_whitespace_corrected);
 	}
 
 out:
@@ -642,7 +657,7 @@ fixdiff_stanza_end(dp_t *pdp)
 	}
 
 	if (dp.pending_empty_lines)
-		elog("  Dropped %d unexpected empty lines\n", dp.pending_empty_lines);
+		elog("    stanza %d: Dropped %d unexpected empty lines\n", pdp->stanzas, dp.pending_empty_lines);
 
 	if (fixdiff_find_original(pdp, &orig)) {
 		elog("Unable to find original stanza in source\n");
@@ -885,7 +900,8 @@ main(int argc, char *argv[])
 			    (in[0] == ' ' || in[0] == '-' || in[0] == '+')) {
 				char ctx[3];
 
-				elog("  Treating %d unexpected newline(s) as context\n", dp.pending_empty_lines);
+				elog("    stanza %d: Treating %d unexpected newline(s) as context\n",
+					dp.stanzas, dp.pending_empty_lines);
 
 				ctx[0] = ' ';
 				ctx[1] = '\n';
@@ -959,15 +975,15 @@ main(int argc, char *argv[])
 							in[2] = in[l1 + 1];
 							in[3] = '\0';
 							l = 3;
-							elog("  Reducing %u char whitespace-only line to CRLF\n",
-							     (unsigned int)l1);
+							elog("    stanza %d: Reducing %u char whitespace-only "
+								"line to CRLF\n", dp.stanzas, (unsigned int)l1);
 						} else
 							if (l1 > 1 && in[l1] == 0x0a && (l - l1) == 1) {
 								in[1] = in[l1];
 								in[2] = '\0';
 								l = 2;
-								elog("  Reducing %d char whitespace-only line to LF\n",
-									(unsigned int)l1);
+								elog("    stanza %d: Reducing %d char whitespace-only"
+									" line to LF\n", dp.stanzas, (unsigned int)l1);
 							}
 
 						dp.post++;