fix(installation-cleanup): don't mark live agents stale (Telegram bridge was 403ing)

backend/__tests__/unit/services/agentInstallationCleanup.staleness.test.js

@@ -0,0 +1,118 @@
+/**
+ * markStaleInstallations — liveness signal regression tests.
+ *
+ * Guards the fix for a live outage (2026-07-18): the Telegram bridge agents
+ * (telegram-app, yunus) were marked stale by this cron while actively
+ * relaying messages, which made every send 403 with
+ * "Agent token not authorized for this pod" — pod authorization requires
+ * status 'active'.
+ *
+ * Root cause was a liveness blind spot, not a bug in the marking logic:
+ *   - their tokens are long-lived file-based tokens with NO expiresAt, and
+ *     hasValidRuntimeToken deliberately treats null-expiry as invalid;
+ *   - posting a pod message emits NO AgentEvent, so a send-only integration
+ *     shows zero event activity no matter how much traffic it pushes.
+ * Both "dead" signals fired for an agent that was demonstrably alive.
+ *
+ * The surviving signal is agentRuntimeTokens[].lastUsedAt, which
+ * agentRuntimeAuth writes on every authenticated request.
+ */
+
+jest.mock('node-cron', () => ({ schedule: jest.fn() }));
+
+jest.mock('../../../models/AgentRegistry', () => ({
+  AgentInstallation: { find: jest.fn(), updateMany: jest.fn(), deleteMany: jest.fn() },
+}));
+jest.mock('../../../models/AgentEvent', () => ({ findOne: jest.fn() }));
+jest.mock('../../../models/User', () => ({ findOne: jest.fn() }));
+jest.mock('../../../models/Pod', () => ({ updateOne: jest.fn() }));
+jest.mock('../../../services/agentIdentityService', () => ({
+  buildAgentUsername: (name, instanceId) => `${name}__${instanceId}`,
+}));
+
+const { AgentInstallation } = require('../../../models/AgentRegistry');
+const AgentEvent = require('../../../models/AgentEvent');
+const User = require('../../../models/User');
+const { markStaleInstallations } = require('../../../services/agentInstallationCleanupService');
+
+const DAY = 24 * 60 * 60 * 1000;
+
+/** One active installation for the agent under test. */
+function givenOneActiveInstall() {
+  AgentInstallation.find.mockReturnValue({
+    select: () => ({ lean: async () => [{ _id: 'i1', agentName: 'telegram-app', instanceId: 'default', podId: 'p1' }] }),
+  });
+}
+
+/** The owning user's token state: what expiry it carries and when it was last used. */
+function givenToken({ expiresAt = null, lastUsedAt = null }) {
+  User.findOne.mockReturnValue({
+    select: () => ({ lean: async () => ({ agentRuntimeTokens: [{ expiresAt, lastUsedAt }] }) }),
+  });
+}
+
+/** No AgentEvent has ever been recorded — true for send-only integrations. */
+function givenNoEvents() {
+  AgentEvent.findOne.mockReturnValue({
+    select: () => ({ sort: () => ({ lean: async () => null }) }),
+  });
+}
+
+beforeEach(() => {
+  jest.clearAllMocks();
+  AgentInstallation.updateMany.mockResolvedValue({ modifiedCount: 1 });
+  givenOneActiveInstall();
+  givenNoEvents();
+});
+
+describe('markStaleInstallations liveness signals', () => {
+  it('spares a non-expiring token that was used recently (the outage case)', async () => {
+    givenToken({ expiresAt: null, lastUsedAt: new Date(Date.now() - 1 * DAY) });
+
+    const { marked } = await markStaleInstallations(7);
+
+    expect(marked).toBe(0);
+    expect(AgentInstallation.updateMany).not.toHaveBeenCalled();
+  });
+
+  it('still marks a non-expiring token unused past the cutoff', async () => {
+    givenToken({ expiresAt: null, lastUsedAt: new Date(Date.now() - 30 * DAY) });
+
+    const { marked } = await markStaleInstallations(7);
+
+    expect(marked).toBe(1);
+    expect(AgentInstallation.updateMany).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'active' }),
+      expect.objectContaining({ $set: expect.objectContaining({ status: 'stale' }) }),
+    );
+  });
+
+  it('still marks a token that was never used at all', async () => {
+    givenToken({ expiresAt: null, lastUsedAt: null });
+
+    const { marked } = await markStaleInstallations(7);
+
+    expect(marked).toBe(1);
+  });
+
+  it('does not mark installations younger than the staleness window', async () => {
+    // A freshly provisioned agent reads "dead" on both signals purely because
+    // nobody has @-mentioned it yet. The createdAt floor must exclude it.
+    givenToken({ expiresAt: null, lastUsedAt: null });
+
+    await markStaleInstallations(7);
+
+    expect(AgentInstallation.updateMany).toHaveBeenCalledWith(
+      expect.objectContaining({ createdAt: { $lt: expect.any(Date) } }),
+      expect.anything(),
+    );
+  });
+
+  it('spares an unexpired token regardless of use (pre-existing behavior)', async () => {
+    givenToken({ expiresAt: new Date(Date.now() + 30 * DAY), lastUsedAt: null });
+
+    const { marked } = await markStaleInstallations(7);
+
+    expect(marked).toBe(0);
+  });
+});

backend/services/agentInstallationCleanupService.ts

@@ -65,6 +65,40 @@ function hasValidRuntimeToken(user: { agentRuntimeTokens?: Array<{ expiresAt?: D
   });
 }
 
+/**
+ * Returns true if any of the user's runtime tokens was actually USED since
+ * `cutoff`.
+ *
+ * Why this exists: `hasValidRuntimeToken` only asks "is a token unexpired?",
+ * and deliberately treats a null `expiresAt` as invalid. That is right for
+ * abandoned stragglers, but it misreads long-lived file-based tokens (the
+ * Telegram bridge's `~/.commonly/tokens/<name>.json`), which legitimately
+ * carry no expiry. The AgentEvent check doesn't rescue them either: posting a
+ * message emits NO AgentEvent, so a send-only integration looks dead no
+ * matter how much traffic it pushes.
+ *
+ * Measured 2026-07-18: telegram-app and yunus were both marked stale by this
+ * cron while actively relaying, which 403'd every send
+ * ("Agent token not authorized for this pod") because pod authorization
+ * requires status 'active'. Both were ~10 days from outright deletion by
+ * pruneStaleInstallations.
+ *
+ * `lastUsedAt` is the honest liveness signal: agentRuntimeAuth writes it on
+ * EVERY authenticated request. The user doc is already loaded here, so this
+ * costs no extra query — we were holding the answer and not reading it.
+ */
+function hasRecentTokenUse(
+  user: { agentRuntimeTokens?: Array<{ lastUsedAt?: Date | null }> } | null,
+  cutoff: Date,
+): boolean {
+  if (!user || !Array.isArray(user.agentRuntimeTokens)) return false;
+  return user.agentRuntimeTokens.some((t) => {
+    if (!t || !t.lastUsedAt) return false;
+    const used = t.lastUsedAt instanceof Date ? t.lastUsedAt.getTime() : new Date(t.lastUsedAt as any).getTime();
+    return Number.isFinite(used) && used > cutoff.getTime();
+  });
+}
+
 /**
  * Scan every active AgentInstallation and mark it stale when the owning agent
  * user has no valid runtime tokens AND the latest AgentEvent for
@@ -119,6 +153,13 @@ export async function markStaleInstallations(
         continue;
       }
 
+      // Token has no future expiry — but was it recently USED? Send-only
+      // integrations (Telegram bridge) hold non-expiring tokens and emit no
+      // AgentEvents, so this is the only signal that they're alive.
+      if (hasRecentTokenUse(user, cutoff)) {
+        continue;
+      }
+
       // Check most recent AgentEvent for this (agentName, instanceId)
       const latestEvent = await AgentEvent.findOne({ agentName, instanceId })
         .select('createdAt')
@@ -152,8 +193,16 @@ export async function markStaleInstallations(
     return { agentName, instanceId };
   });
 
+  // Never mark an installation younger than the staleness window. Without this
+  // floor, a freshly provisioned agent is stale on the very first nightly run:
+  // its token has never been used and it has emitted no events, so both
+  // liveness checks read "dead" purely because it hasn't been given work yet.
+  // That is a normal state — departments get provisioned before anyone
+  // @-mentions them, and lazily-started daemons don't run until mentioned.
+  // Observed 2026-07-18: dev-jr, installed the same afternoon, was marked
+  // stale minutes later and would have been deleted 14 days on.
   const result = await AgentInstallation.updateMany(
-    { status: 'active', $or: orClauses },
+    { status: 'active', createdAt: { $lt: cutoff }, $or: orClauses },
     { $set: { status: 'stale', staleSince: now } },
   );