From cc74dfe0dc02bd4273f2d935233e33e013b0fbfd Mon Sep 17 00:00:00 2001 From: Nikola Katsarov Date: Mon, 3 Aug 2026 15:42:37 +0300 Subject: [PATCH] fix(sentry): drop Redis connect-time failures from the container-restart race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sentry #957 accumulated 808 events since 2026-07-01 and buried the rest of the fleetq project. It is not a chronic outage: the 100 most recent events fall into three bursts (2026-07-21, -07-22, -08-03) with days of silence between, each lining up with a container recreate / redis config change. Culprit is /var/www/artisan — scheduler and queue workers reconnecting while redis is still coming up. Same shape as the existing SQLSTATE[08006] postgres filter, for the other datastore. Scoped to Predis StreamInitException, which is thrown only from StreamFactory during stream init, so a read/write error on an established connection still reports (regression-tested). Also drops the prometheus client's own wrapper of the same failure reaching the /metrics scrape (#1082), which the class check cannot catch. A sustained outage stays visible: HealthController pings Redis and flips /api/v1/health to 503 degraded. --- .../Sentry/BeforeSendFilter.php | 24 ++++++++++ .../Sentry/BeforeSendFilterWebhookTest.php | 46 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/app/Infrastructure/Sentry/BeforeSendFilter.php b/app/Infrastructure/Sentry/BeforeSendFilter.php index d7d872f0..e128dd7b 100644 --- a/app/Infrastructure/Sentry/BeforeSendFilter.php +++ b/app/Infrastructure/Sentry/BeforeSendFilter.php @@ -6,6 +6,7 @@ use ErrorException; use Illuminate\Database\QueryException; use Illuminate\Queue\MaxAttemptsExceededException; +use Predis\Connection\Resource\Exception\StreamInitException; use Sentry\Event; use Sentry\EventHint; use Symfony\Component\Console\Exception\RuntimeException as SymfonyConsoleRuntimeException; @@ -52,6 +53,29 @@ public static function filter(Event $event, ?EventHint $hint = null): ?Event return null; } + // Redis unreachable at CONNECT time — the same container-restart race as + // 08006 above, for the other datastore (deploy --force-recreate, redis + // recreate, DNS not yet resolving). Bursty, not chronic: the 100 most + // recent events of #957 fell into three windows (2026-07-21, -07-22, + // -08-03) with days of silence between, and 808 events since 2026-07-01 + // buried the rest of the fleetq project. + // + // Scoped to StreamInitException on purpose — Predis throws it only from + // StreamFactory during stream init, so a read/write error on an already + // established connection (a different defect class) still reports. + // A SUSTAINED outage stays visible: HealthController pings Redis and + // flips /api/v1/health to 503 `degraded`. + if ($e instanceof StreamInitException) { + return null; + } + + // Same restart race reaching the /metrics scrape — the prometheus client + // wraps the Redis connect failure in its own storage exception, so the + // class check above can't catch it. (#1082) + if (str_contains($msg, "Can't connect to Redis server")) { + return null; + } + // Anthropic auth_error / Prism 401 — FallbackAiGateway::isAuthError already // routes around it without recording a circuit-breaker failure. if (str_contains($msg, 'authentication_error') && str_contains($msg, 'x-api-key')) { diff --git a/tests/Unit/Infrastructure/Sentry/BeforeSendFilterWebhookTest.php b/tests/Unit/Infrastructure/Sentry/BeforeSendFilterWebhookTest.php index d0e45b9c..5476bddb 100644 --- a/tests/Unit/Infrastructure/Sentry/BeforeSendFilterWebhookTest.php +++ b/tests/Unit/Infrastructure/Sentry/BeforeSendFilterWebhookTest.php @@ -5,6 +5,10 @@ use App\Domain\Shared\Exceptions\AiAccessUnavailableException; use App\Infrastructure\Sentry\BeforeSendFilter; use PHPUnit\Framework\Attributes\Test; +use Predis\Connection\ConnectionException; +use Predis\Connection\Parameters; +use Predis\Connection\Resource\Exception\StreamInitException; +use Predis\Connection\StreamConnection; use RuntimeException; use Sentry\Event; use Sentry\EventHint; @@ -82,6 +86,48 @@ public function it_drops_ai_access_unavailable_from_queue_workers(): void $this->assertNull($result); } + #[Test] + public function it_drops_redis_connect_failures_from_the_restart_race(): void + { + $result = BeforeSendFilter::filter( + Event::createEvent(), + $this->hintFor(new StreamInitException('Connection refused [tcp://agent-fleet-redis:6379]')), + ); + + $this->assertNull($result); + } + + #[Test] + public function it_drops_prometheus_redis_connect_failures(): void + { + $result = BeforeSendFilter::filter( + Event::createEvent(), + $this->hintFor(new RuntimeException( + "Can't connect to Redis server. php_network_getaddresses: getaddrinfo for agent-fleet-redis failed", + )), + ); + + $this->assertNull($result); + } + + #[Test] + public function it_keeps_redis_read_errors_on_established_connections(): void + { + // The restart-race filter must not swallow this class — a read error on + // an open connection is a different defect (fixed separately in 16c2b9f2). + $event = Event::createEvent(); + + $result = BeforeSendFilter::filter( + $event, + $this->hintFor(new ConnectionException( + new StreamConnection(new Parameters(['host' => 'agent-fleet-redis'])), + 'read error on connection to agent-fleet-redis:6379', + )), + ); + + $this->assertSame($event, $result); + } + private function hintFor(\Throwable $e): EventHint { $hint = new EventHint;