fix(health): remove TCP half-close that breaks Docker health check (#2)

## Summary

- **Root cause**: The health check client called `writer.shutdown()`
after sending the HTTP request, which sends a TCP FIN. Hyper interprets
the FIN as the connection closing prematurely and aborts before sending
the response — producing `"connection closed before message completed"`
errors every 2s and marking the container unhealthy.
- **Fix**: Remove the stream split and half-close. The GET request is
self-framing and the `Connection: close` header tells hyper to close
after responding, giving `read_to_string` its EOF naturally.
- Downgrade connection-error log from `error!` to `warn!` since client
resets are normal.

Closes #1

## Test plan

- [x] `cargo build` / `cargo test` / `cargo clippy` pass
- [x] Run `docker compose up` with the image rebuilt from this branch
and verify container shows `(healthy)`
- [x] Confirm no `"connection closed before message completed"` errors
in logs

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: tyrchen

.github/workflows/s3-test.yml

@@ -31,7 +31,7 @@ jobs:
           SSM_SKIP_SIGNATURE_VALIDATION=true \
           GATEWAY_LISTEN=0.0.0.0:4566 \
           LOG_LEVEL=warn \
-          cargo run --release -p ruststack-server &
+          cargo run --release -p ruststack-server > /tmp/ruststack.log 2>&1 &
 
           for i in $(seq 1 30); do
             if curl -sf http://127.0.0.1:4566/_localstack/health > /dev/null 2>&1; then
@@ -52,6 +52,25 @@ jobs:
           echo "AWS_ACCESS_KEY_ID=test" >> "$GITHUB_ENV"
           echo "AWS_SECRET_ACCESS_KEY=test" >> "$GITHUB_ENV"
 
+      # ── Health Check Verification ────────────────────────────────────
+      - name: "Health Check: verify --health-check flag and no connection errors"
+        run: |
+          set -euo pipefail
+
+          # Run the built-in health check multiple times
+          for i in $(seq 1 5); do
+            ./target/release/ruststack-server --health-check
+          done
+
+          # Verify the server log has no "connection closed before message completed" errors
+          if grep -q "connection closed before message completed" /tmp/ruststack.log; then
+            echo "::error::Server logged 'connection closed before message completed' — health check is broken"
+            cat /tmp/ruststack.log
+            exit 1
+          fi
+
+          echo "Health check passed 5 times with no connection errors"
+
       # ── Bucket Operations ───────────────────────────────────────────
       - name: "Bucket Operations: create, head, list, locate, delete"
         run: |

apps/ruststack-server/src/main.rs

@@ -39,7 +39,7 @@ use anyhow::{Context, Result};
 use hyper_util::rt::{TokioExecutor, TokioIo};
 use hyper_util::server::conn::auto::Builder as HttpConnBuilder;
 use tokio::net::TcpListener;
-use tracing::{error, info, warn};
+use tracing::{info, warn};
 use tracing_subscriber::EnvFilter;
 
 #[cfg(feature = "dynamodb")]
@@ -252,7 +252,7 @@ async fn serve(listener: TcpListener, service: GatewayService) -> Result<()> {
 
                 tokio::spawn(async move {
                     if let Err(e) = conn.await {
-                        error!(peer_addr = %peer_addr, error = %e, "connection error");
+                        warn!(peer_addr = %peer_addr, error = %e, "connection error");
                     }
                 });
             }
@@ -333,19 +333,20 @@ async fn run_health_check(addr: &str) -> Result<()> {
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
     use tokio::net::TcpStream;
 
-    let stream = TcpStream::connect(addr)
+    let mut stream = TcpStream::connect(addr)
         .await
         .with_context(|| format!("cannot connect to {addr}"))?;
 
-    let (mut reader, mut writer) = stream.into_split();
-
     let request =
         format!("GET /_localstack/health HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n");
-    writer.write_all(request.as_bytes()).await?;
-    writer.shutdown().await?;
+    stream.write_all(request.as_bytes()).await?;
+    // Do not half-close the write side: the HTTP request is self-framing
+    // (GET with no body) and the Connection: close header tells hyper not
+    // to expect further requests.  Hyper will close after responding,
+    // which gives read_to_string its EOF.
 
     let mut response = String::new();
-    reader.read_to_string(&mut response).await?;
+    stream.read_to_string(&mut response).await?;
 
     // Accept any 200 response that reports at least one running service.
     if response.contains("200 OK") && response.contains("\"running\"") {

tests/integration/src/lib.rs

@@ -238,6 +238,7 @@ mod test_bucket;
 mod test_cors;
 mod test_dynamodb;
 mod test_error;
+mod test_health;
 mod test_lambda;
 mod test_list;
 mod test_multipart;

tests/integration/src/test_health.rs

@@ -0,0 +1,90 @@
+//! Health check integration tests.
+//!
+//! Verifies that the `/_localstack/health` endpoint works correctly and that
+//! raw TCP health checks (as used by Docker `HEALTHCHECK`) do not produce
+//! "connection closed before message completed" errors.
+
+#[cfg(test)]
+mod tests {
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpStream;
+
+    use crate::endpoint_url;
+
+    /// Extract host:port from the endpoint URL.
+    fn endpoint_addr() -> String {
+        let url = endpoint_url();
+        url.strip_prefix("http://").unwrap_or(&url).to_owned()
+    }
+
+    /// Simulate the exact same TCP-level health check that the Docker
+    /// HEALTHCHECK uses (`ruststack-server --health-check`).  The fix for
+    /// issue #1 removed the `writer.shutdown()` call; this test verifies
+    /// the server responds correctly without a TCP half-close.
+    #[tokio::test]
+    #[ignore = "requires running server"]
+    async fn test_should_respond_to_raw_tcp_health_check() {
+        let addr = endpoint_addr();
+
+        let mut stream = TcpStream::connect(&addr)
+            .await
+            .unwrap_or_else(|e| panic!("cannot connect to {addr}: {e}"));
+
+        let request = format!(
+            "GET /_localstack/health HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n",
+        );
+        stream
+            .write_all(request.as_bytes())
+            .await
+            .expect("failed to write request");
+        // Intentionally NOT calling stream.shutdown() — this is the fix.
+
+        let mut response = String::new();
+        stream
+            .read_to_string(&mut response)
+            .await
+            .expect("failed to read response");
+
+        assert!(
+            response.contains("200 OK"),
+            "expected 200 OK in response, got: {response}",
+        );
+        assert!(
+            response.contains("\"running\""),
+            "expected 'running' in response, got: {response}",
+        );
+    }
+
+    /// Run multiple sequential health checks to ensure the server does not
+    /// accumulate connection errors over time.
+    #[tokio::test]
+    #[ignore = "requires running server"]
+    async fn test_should_handle_repeated_health_checks() {
+        let addr = endpoint_addr();
+
+        for i in 0..10 {
+            let mut stream = TcpStream::connect(&addr)
+                .await
+                .unwrap_or_else(|e| panic!("connect #{i} to {addr} failed: {e}"));
+
+            let request = format!(
+                "GET /_localstack/health HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n",
+            );
+            stream
+                .write_all(request.as_bytes())
+                .await
+                .unwrap_or_else(|e| panic!("write #{i} failed: {e}"));
+
+            let mut response = String::new();
+            stream
+                .read_to_string(&mut response)
+                .await
+                .unwrap_or_else(|e| panic!("read #{i} failed: {e}"));
+
+            assert!(
+                response.contains("200 OK"),
+                "health check #{i} failed: {response}",
+            );
+        }
+    }
+}