make drones/proxies reestablish connection with controller every ~1 hour

common/src/util.rs

@@ -22,3 +22,14 @@ pub fn random_token() -> String {
 pub fn random_prefixed_string(prefix: &str) -> String {
     format!("{}-{}", prefix, random_string())
 }
+
+/// Returns a duration of approximately 1 hour, with random jitter of +/- 10 minutes.
+pub fn random_reconnect_interval() -> std::time::Duration {
+    const BASE_SECONDS: u64 = 60 * 60; // 1 hour
+    const JITTER_SECONDS: u64 = 10 * 60; // 10 minutes
+
+    let jitter: i64 =
+        rand::thread_rng().gen_range(-(JITTER_SECONDS as i64)..=(JITTER_SECONDS as i64));
+    let total_seconds = (BASE_SECONDS as i64 + jitter) as u64;
+    std::time::Duration::from_secs(total_seconds)
+}

plane/src/drone/mod.rs

@@ -19,6 +19,7 @@ use plane_common::{
     },
     typed_socket::{client::TypedSocketConnector, TypedSocketSender},
     types::{BackendState, ClusterName, DronePoolName},
+    util::random_reconnect_interval,
     PlaneClient,
 };
 use runtime::docker::DockerRuntime;
@@ -98,8 +99,14 @@ pub async fn drone_loop(
         log_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
         let mut message_counts: HashMap<&'static str, u64> = HashMap::new();
 
+        let reconnect_at = tokio::time::Instant::now() + random_reconnect_interval();
+
         loop {
             tokio::select! {
+                _ = tokio::time::sleep_until(reconnect_at) => {
+                    tracing::info!("Periodic reconnect triggered.");
+                    break;
+                }
                 _ = log_interval.tick() => {
                     let (outgoing, incoming) = socket.channel_depths();
                     tracing::info!(

plane/src/proxy/proxy_connection.rs

@@ -3,6 +3,7 @@ use plane_common::{
     names::ProxyName,
     protocol::{MessageFromProxy, MessageToProxy, RouteInfoRequest},
     types::ClusterName,
+    util::random_reconnect_interval,
     PlaneClient,
 };
 use std::{collections::HashMap, sync::Arc, time::Duration};
@@ -62,8 +63,14 @@ impl ProxyConnection {
                     log_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
                     let mut message_counts: HashMap<&'static str, u64> = HashMap::new();
 
+                    let reconnect_at = tokio::time::Instant::now() + random_reconnect_interval();
+
                     loop {
                         tokio::select! {
+                            _ = tokio::time::sleep_until(reconnect_at) => {
+                                tracing::info!("Periodic reconnect triggered.");
+                                break;
+                            }
                             _ = log_interval.tick() => {
                                 let (outgoing, incoming) = conn.channel_depths();
                                 tracing::info!(