From 8c4e41d7f9bf70017bc359e0819189dad3768171 Mon Sep 17 00:00:00 2001
From: Taylor Baldwin <taylorbaldwin@gmail.com>
Date: Fri, 9 Jan 2026 19:01:53 +0000
Subject: [PATCH] make drones/proxies reestablish connection with controller
 every ~1 hour

---
 common/src/util.rs                  | 11 +++++++++++
 plane/src/drone/mod.rs              |  7 +++++++
 plane/src/proxy/proxy_connection.rs |  7 +++++++
 3 files changed, 25 insertions(+)

diff --git a/common/src/util.rs b/common/src/util.rs
index 0bfda8890..1c2512958 100644
--- a/common/src/util.rs
+++ b/common/src/util.rs
@@ -22,3 +22,14 @@ pub fn random_token() -> String {
 pub fn random_prefixed_string(prefix: &str) -> String {
     format!("{}-{}", prefix, random_string())
 }
+
+/// Returns a duration of approximately 1 hour, with random jitter of +/- 10 minutes.
+pub fn random_reconnect_interval() -> std::time::Duration {
+    const BASE_SECONDS: u64 = 60 * 60; // 1 hour
+    const JITTER_SECONDS: u64 = 10 * 60; // 10 minutes
+
+    let jitter: i64 =
+        rand::thread_rng().gen_range(-(JITTER_SECONDS as i64)..=(JITTER_SECONDS as i64));
+    let total_seconds = (BASE_SECONDS as i64 + jitter) as u64;
+    std::time::Duration::from_secs(total_seconds)
+}
diff --git a/plane/src/drone/mod.rs b/plane/src/drone/mod.rs
index 22cadbefe..f768d1110 100644
--- a/plane/src/drone/mod.rs
+++ b/plane/src/drone/mod.rs
@@ -19,6 +19,7 @@ use plane_common::{
     },
     typed_socket::{client::TypedSocketConnector, TypedSocketSender},
     types::{BackendState, ClusterName, DronePoolName},
+    util::random_reconnect_interval,
     PlaneClient,
 };
 use runtime::docker::DockerRuntime;
@@ -98,8 +99,14 @@ pub async fn drone_loop(
         log_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
         let mut message_counts: HashMap<&'static str, u64> = HashMap::new();
 
+        let reconnect_at = tokio::time::Instant::now() + random_reconnect_interval();
+
         loop {
             tokio::select! {
+                _ = tokio::time::sleep_until(reconnect_at) => {
+                    tracing::info!("Periodic reconnect triggered.");
+                    break;
+                }
                 _ = log_interval.tick() => {
                     let (outgoing, incoming) = socket.channel_depths();
                     tracing::info!(
diff --git a/plane/src/proxy/proxy_connection.rs b/plane/src/proxy/proxy_connection.rs
index 80b9da759..cdda9cc75 100644
--- a/plane/src/proxy/proxy_connection.rs
+++ b/plane/src/proxy/proxy_connection.rs
@@ -3,6 +3,7 @@ use plane_common::{
     names::ProxyName,
     protocol::{MessageFromProxy, MessageToProxy, RouteInfoRequest},
     types::ClusterName,
+    util::random_reconnect_interval,
     PlaneClient,
 };
 use std::{collections::HashMap, sync::Arc, time::Duration};
@@ -62,8 +63,14 @@ impl ProxyConnection {
                     log_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
                     let mut message_counts: HashMap<&'static str, u64> = HashMap::new();
 
+                    let reconnect_at = tokio::time::Instant::now() + random_reconnect_interval();
+
                     loop {
                         tokio::select! {
+                            _ = tokio::time::sleep_until(reconnect_at) => {
+                                tracing::info!("Periodic reconnect triggered.");
+                                break;
+                            }
                             _ = log_interval.tick() => {
                                 let (outgoing, incoming) = conn.channel_depths();
                                 tracing::info!(