diff --git a/cli/src/ai/telemetry.ts b/cli/src/ai/telemetry.ts new file mode 100644 index 0000000000..4113d79d6a --- /dev/null +++ b/cli/src/ai/telemetry.ts @@ -0,0 +1,86 @@ +import { sendEvent } from '../utils.js' + +export type AiAnalysisChoice = 'capgo_ai' | 'local_ai' | 'skip' | 'auto_upload' +export type AiAnalysisTriggeredBy = 'menu' | 'ci_flag' +export type AiAnalysisResult = 'success' | 'already_analyzed' | 'too_big' | 'error' + +export interface TrackAiAnalysisChoiceInput { + apikey: string + orgId: string + appId: string + platform: 'ios' | 'android' + jobId: string + choice: AiAnalysisChoice + triggeredBy: AiAnalysisTriggeredBy +} + +export interface TrackAiAnalysisResultInput { + apikey: string + orgId: string + appId: string + platform: 'ios' | 'android' + jobId: string + result: AiAnalysisResult + errorStatus?: number +} + +/** + * Emit `CLI AI Build Analysis Choice` for every branch the user (or CI flag) selected. + * + * Privacy boundary: only closed-enum choice + triggered_by metadata is sent. The + * AI diagnosis text is never observed at this stage. + */ +export async function trackAiAnalysisChoice(input: TrackAiAnalysisChoiceInput): Promise { + try { + await sendEvent(input.apikey, { + event: 'CLI AI Build Analysis Choice', + channel: 'build-lifecycle', + icon: '🤖', + notify: false, + user_id: input.orgId, + tags: { + app_id: input.appId, + platform: input.platform, + job_id: input.jobId, + choice: input.choice, + triggered_by: input.triggeredBy, + }, + }) + } + catch { + // Telemetry must never break the build flow. + } +} + +/** + * Emit `CLI AI Build Analysis Result` only for paths that actually hit the server + * (capgo_ai or auto_upload). + * + * Privacy boundary: the AI analysis text (`result.analysis` in PostAnalyzeResult) + * MUST NEVER appear in any tag here. Only the closed-enum `result` and optional + * `error_status` cross the boundary. + */ +export async function trackAiAnalysisResult(input: TrackAiAnalysisResultInput): Promise { + const tags: Record = { + app_id: input.appId, + platform: input.platform, + job_id: input.jobId, + result: input.result, + } + if (input.result === 'error' && typeof input.errorStatus === 'number' && Number.isFinite(input.errorStatus)) + tags.error_status = String(input.errorStatus) + + try { + await sendEvent(input.apikey, { + event: 'CLI AI Build Analysis Result', + channel: 'build-lifecycle', + icon: '🤖', + notify: false, + user_id: input.orgId, + tags, + }) + } + catch { + // Telemetry must never break the build flow. + } +} diff --git a/cli/src/build/onboarding/android/types.ts b/cli/src/build/onboarding/android/types.ts index 0b7bf90ec6..c5f4aae1ef 100644 --- a/cli/src/build/onboarding/android/types.ts +++ b/cli/src/build/onboarding/android/types.ts @@ -51,6 +51,12 @@ export type AndroidOnboardingStep | 'build-complete' | 'error' +export type AndroidOnboardingErrorCategory + = | 'keystore_invalid' + | 'google_oauth_failed' + | 'play_account_id_invalid' + | 'unknown' + export type KeystoreMethod = 'existing' | 'generate' export interface KeystoreReady { diff --git a/cli/src/build/onboarding/android/ui/app.tsx b/cli/src/build/onboarding/android/ui/app.tsx index 9049e22af0..452e31ba54 100644 --- a/cli/src/build/onboarding/android/ui/app.tsx +++ b/cli/src/build/onboarding/android/ui/app.tsx @@ -2,6 +2,7 @@ import type { FC } from 'react' import type { BuildLogger } from '../../../request.js' import type { GcpProject } from '../gcp-api.js' import type { + AndroidOnboardingErrorCategory, AndroidOnboardingProgress, AndroidOnboardingStep, AndroidPackageChoice, @@ -22,14 +23,16 @@ import { Alert, ProgressBar, Select } from '@inkjs/ui' import { Box, Newline, Text, useApp, useInput, useStdout } from 'ink' // src/build/onboarding/android/ui/app.tsx import React, { useCallback, useEffect, useRef, useState } from 'react' -import { findSavedKey } from '../../../../utils.js' +import { createSupabaseClient, findSavedKey, findSavedKeySilent, getOrganizationId } from '../../../../utils.js' import { loadSavedCredentials, updateSavedCredentials } from '../../../credentials.js' import { requestBuildInternal } from '../../../request.js' -import { createCiSecretEntries, detectCiSecretTargets, getCiSecretTargetLabel, listExistingCiSecretKeys, uploadCiSecrets } from '../../ci-secrets.js' import type { CiSecretEntry, CiSecretSetupAdvice, CiSecretTarget } from '../../ci-secrets.js' +import { createCiSecretEntries, detectCiSecretTargets, getCiSecretTargetLabel, listExistingCiSecretKeys, uploadCiSecrets } from '../../ci-secrets.js' +import { mapAndroidOnboardingError } from '../../error-categories.js' import { canUseFilePicker, openKeystorePicker } from '../../file-picker.js' -import { findAndroidApplicationIds } from '../gradle-parser.js' +import { trackBuilderOnboardingStep } from '../../telemetry.js' import { Divider, ErrorLine, FilteredTextInput, Header, SpinnerLine, SuccessLine } from '../../ui/components.js' +import { findAndroidApplicationIds } from '../gradle-parser.js' import { ANDROIDPUBLISHER_API, createServiceAccountKey, @@ -119,8 +122,126 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir const [step, setStep] = useState( startStep === 'welcome' ? 'welcome' : startStep, ) + + // Telemetry: resolve org id once + emit per-step events + const stepTimingRef = useRef<{ step: AndroidOnboardingStep | null, startedAt: number }>({ + step: null, + startedAt: Date.now(), + }) + // Buffer of telemetry events that occurred before `resolvedOrgId` landed. + // Drained in order when the org id becomes available. Without this buffer, + // any step transitions during the async org-id resolution (which involves + // two HTTP round-trips: createSupabaseClient + getOrganizationId) would be + // dropped from the funnel. + const pendingTelemetryRef = useRef>([]) + const [resolvedOrgId, setResolvedOrgId] = useState(null) + const resolvedApiKeyRef = useRef(apikey ?? null) + const orgIdResolvedRef = useRef(false) + // Captures the mapped error category at handleError time so the telemetry + // useEffect can pass it through without re-mapping a reconstructed Error + // (which would have lost the .phase / instanceof discriminators). + const errorCategoryRef = useRef(undefined) + + useEffect(() => { + if (resolvedApiKeyRef.current) + return + const saved = findSavedKeySilent() + if (saved) + resolvedApiKeyRef.current = saved + }, []) + + useEffect(() => { + if (orgIdResolvedRef.current || !resolvedApiKeyRef.current) + return + orgIdResolvedRef.current = true + + let cancelled = false + void (async () => { + const supabase = await createSupabaseClient(resolvedApiKeyRef.current!, undefined, undefined, true) + .catch(() => null) + if (!supabase || cancelled) + return + const orgId = await getOrganizationId(supabase, appId).catch(() => null) + if (orgId && !cancelled) + setResolvedOrgId(orgId) + })() + + return () => { + cancelled = true + } + }, [appId]) + const [logLines, setLogLines] = useState([]) const [error, setError] = useState(null) + + // Emit telemetry on every step transition (including initial mount). + // Sequencing: + // 1. If `resolvedOrgId` just became available, drain the backlog first. + // 2. Skip same-step re-renders (orgId-lands triggers a re-fire — we don't + // want to re-emit the current step, only drain the backlog). + // 3. Otherwise compute the new event, then either emit immediately (orgId + // available) or queue it (orgId still loading). + useEffect(() => { + if (!resolvedApiKeyRef.current) + return + + const previous = stepTimingRef.current + const isDuplicateStep = previous.step !== null && previous.step === step && step !== 'error' + + // (1) Drain the backlog if org id is now available, even when the current + // step is a duplicate (e.g., this effect fired because resolvedOrgId moved + // from null to a real value, not because step changed). + if (resolvedOrgId && pendingTelemetryRef.current.length > 0) { + for (const queued of pendingTelemetryRef.current) { + void trackBuilderOnboardingStep({ + apikey: resolvedApiKeyRef.current, + appId, + orgId: resolvedOrgId, + platform: 'android', + ...queued, + }) + } + pendingTelemetryRef.current = [] + } + + // (2) Now safely skip the duplicate-step path. + if (isDuplicateStep) + return + + const now = Date.now() + // Initial step (previous.step === null) and same-step error re-entries have + // no meaningful previous-step duration. + const durationMs = previous.step === null || previous.step === step + ? undefined + : now - previous.startedAt + + const eventPayload = { + step, + durationMs, + errorCategory: step === 'error' ? errorCategoryRef.current : undefined, + } + + stepTimingRef.current = { step, startedAt: now } + + // (3) Either fire immediately or buffer. + if (resolvedOrgId) { + void trackBuilderOnboardingStep({ + apikey: resolvedApiKeyRef.current, + appId, + orgId: resolvedOrgId, + platform: 'android', + ...eventPayload, + }) + } + else { + pendingTelemetryRef.current.push(eventPayload) + } + }, [step, appId, resolvedOrgId, error]) + const [retryCount, setRetryCount] = useState(0) const [retryStep, setRetryStep] = useState(null) const exitRequestedRef = useRef(false) @@ -351,6 +472,10 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir const handleError = useCallback( (err: unknown, failedStep: AndroidOnboardingStep) => { + // Capture the mapped category BEFORE we collapse err to a string. + // The telemetry useEffect will read this ref instead of re-mapping a + // reconstructed `new Error(message)` (which has no discriminators). + errorCategoryRef.current = mapAndroidOnboardingError(err) const message = err instanceof Error ? err.message : String(err) if (retryCount === 0) { setError(message) @@ -2067,6 +2192,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir onChange={(value) => { if (value === 'retry') { setError(null) + errorCategoryRef.current = undefined const target = retryStep setRetryStep(null) setStep(target) diff --git a/cli/src/build/onboarding/error-categories.ts b/cli/src/build/onboarding/error-categories.ts new file mode 100644 index 0000000000..41ab2d49b1 --- /dev/null +++ b/cli/src/build/onboarding/error-categories.ts @@ -0,0 +1,84 @@ +import type { AndroidOnboardingErrorCategory } from './android/types.js' +import type { OnboardingErrorCategory, OnboardingStep } from './types.js' +import { MissingScopesError } from './android/oauth-google.js' +import { CertificateLimitError } from './apple-api.js' + +interface MaybeStatus { + status?: unknown +} + +interface MaybePhase { + phase?: string +} + +function getStatus(error: unknown): number | undefined { + if (!error || typeof error !== 'object') + return undefined + const candidate = (error as MaybeStatus).status + return typeof candidate === 'number' ? candidate : undefined +} + +function getPhase(error: unknown): string | undefined { + if (!error || typeof error !== 'object') + return undefined + const candidate = (error as MaybePhase).phase + return typeof candidate === 'string' ? candidate : undefined +} + +export function mapIosOnboardingError( + error: unknown, + failedStep?: OnboardingStep, +): OnboardingErrorCategory { + // Structural discriminators take precedence so an ASC API error thrown + // during an import step (e.g. fetching a profile via the API) still maps + // to the correct category instead of being shadowed by the step-derived + // fallback below. + if (error instanceof CertificateLimitError) + return 'cert_limit_reached' + + const status = getStatus(error) + if (status === 401) + return 'apple_api_unauthorized' + if (status === 429) + return 'apple_api_rate_limited' + + const phase = getPhase(error) + if (phase === 'profile') + return 'profile_creation_failed' + if (phase === 'p8') + return 'p8_invalid' + + // Import-flow step-derived categories. The import path throws + // MacOSSigningError / generic Error without a `phase` or `status` + // discriminator, so we derive the category from the step at which the + // failure occurred. + if (failedStep === 'import-scanning') + return 'keychain_no_identities' + if (failedStep === 'import-compiling-helper') + return 'keychain_helper_compile_failed' + if (failedStep === 'import-exporting') + return 'keychain_export_failed' + if (failedStep === 'import-fetching-profile') + return 'profile_read_failed' + if (failedStep === 'import-pick-profile' || failedStep === 'import-no-match-recovery') + return 'profile_no_match' + + return 'unknown' +} + +export function mapAndroidOnboardingError(error: unknown): AndroidOnboardingErrorCategory { + // MissingScopesError has no `phase` property, so the instanceof check must + // precede the phase-based dispatching below. + if (error instanceof MissingScopesError) + return 'google_oauth_failed' + + const phase = getPhase(error) + if (phase === 'keystore') + return 'keystore_invalid' + if (phase === 'oauth') + return 'google_oauth_failed' + if (phase === 'play_account_id') + return 'play_account_id_invalid' + + return 'unknown' +} diff --git a/cli/src/build/onboarding/telemetry.ts b/cli/src/build/onboarding/telemetry.ts new file mode 100644 index 0000000000..dbf0a75905 --- /dev/null +++ b/cli/src/build/onboarding/telemetry.ts @@ -0,0 +1,52 @@ +import type { AndroidOnboardingErrorCategory, AndroidOnboardingStep } from './android/types.js' +import type { OnboardingErrorCategory, OnboardingStep, Platform } from './types.js' +import { sendEvent } from '../../utils.js' +import { mapAndroidOnboardingError, mapIosOnboardingError } from './error-categories.js' + +export interface TrackBuilderOnboardingStepInput { + apikey: string + appId: string + orgId: string + platform: Platform + step: OnboardingStep | AndroidOnboardingStep + durationMs?: number + /** Raw caught error — mapped via the platform's category mapper. Use this OR errorCategory, not both. */ + error?: unknown + /** Pre-computed category. Takes precedence over `error` if both are present. */ + errorCategory?: OnboardingErrorCategory | AndroidOnboardingErrorCategory +} + +export async function trackBuilderOnboardingStep(input: TrackBuilderOnboardingStepInput): Promise { + const tags: Record = { + step: input.step, + platform: input.platform, + app_id: input.appId, + } + + if (typeof input.durationMs === 'number' && Number.isFinite(input.durationMs)) + tags.duration_ms = String(Math.round(input.durationMs)) + + if (input.errorCategory !== undefined) { + tags.error_category = input.errorCategory + } + else if (input.error !== undefined) { + tags.error_category = input.platform === 'ios' + ? mapIosOnboardingError(input.error) + : mapAndroidOnboardingError(input.error) + } + + try { + await sendEvent(input.apikey, { + event: 'Builder Onboarding Step', + channel: 'builder-onboarding', + icon: '🧭', + notify: false, + user_id: input.orgId, + tags, + }) + } + catch { + // Telemetry must never break the wizard. sendEvent already swallows + // fetch failures internally; this catch covers anything else. + } +} diff --git a/cli/src/build/onboarding/types.ts b/cli/src/build/onboarding/types.ts index e214ce3986..ae61aa9b33 100644 --- a/cli/src/build/onboarding/types.ts +++ b/cli/src/build/onboarding/types.ts @@ -49,6 +49,20 @@ export type OnboardingStep | 'no-platform' | 'error' +export type OnboardingErrorCategory + = | 'apple_api_unauthorized' + | 'apple_api_rate_limited' + | 'cert_limit_reached' + | 'profile_creation_failed' + | 'p8_invalid' + // Import-existing flow (keychain / provisioning profile imports) + | 'keychain_no_identities' + | 'keychain_export_failed' + | 'keychain_helper_compile_failed' + | 'profile_no_match' + | 'profile_read_failed' + | 'unknown' + export interface ApiKeyData { keyId: string issuerId: string diff --git a/cli/src/build/onboarding/ui/app.tsx b/cli/src/build/onboarding/ui/app.tsx index 7334b092e2..2435e75bf0 100644 --- a/cli/src/build/onboarding/ui/app.tsx +++ b/cli/src/build/onboarding/ui/app.tsx @@ -1,7 +1,7 @@ import type { FC } from 'react' import type { BuildLogger } from '../../request.js' import type { DiscoveredProfile, IdentityProfileMatch, SigningIdentity } from '../macos-signing.js' -import type { ApiKeyData, CertificateData, OnboardingProgress, OnboardingStep, ProfileData } from '../types.js' +import type { ApiKeyData, CertificateData, OnboardingErrorCategory, OnboardingProgress, OnboardingStep, ProfileData } from '../types.js' import { handleCustomMsg } from '../../qr.js' import { spawn } from 'node:child_process' import { Buffer } from 'node:buffer' @@ -17,17 +17,19 @@ import open from 'open' import React, { useCallback, useEffect, useRef, useState } from 'react' import { writeOnboardingSupportBundle } from '../../../onboarding-support.js' import { formatRunnerCommand, splitRunnerCommand } from '../../../runner-command.js' -import { findSavedKeySilent, getPMAndCommand } from '../../../utils.js' +import { createSupabaseClient, findSavedKeySilent, getOrganizationId, getPMAndCommand } from '../../../utils.js' import { loadSavedCredentials, updateSavedCredentials } from '../../credentials.js' import { requestBuildInternal } from '../../request.js' import { CertificateLimitError, createCertificate, createProfile, deleteProfile, DuplicateProfileError, ensureBundleId, findCertIdBySha1, generateJwt, listProfilesForCert, revokeCertificate, verifyApiKey } from '../apple-api.js' import { createP12, DEFAULT_P12_PASSWORD, generateCsr } from '../csr.js' +import { mapIosOnboardingError } from '../error-categories.js' import { canUseFilePicker, openFilePicker } from '../file-picker.js' import { exportP12FromKeychain, isHelperCached, isMacOS, listSigningIdentities, matchIdentitiesToProfiles, precompileSwiftHelper, scanProvisioningProfiles } from '../macos-signing.js' import { deleteProgress, getImportEntryStep, getResumeStep, loadProgress, saveProgress } from '../progress.js' import { getBuildOnboardingRecoveryAdvice } from '../recovery.js' -import { createCiSecretEntries, detectCiSecretTargets, getCiSecretTargetLabel, listExistingCiSecretKeys, uploadCiSecrets } from '../ci-secrets.js' import type { CiSecretEntry, CiSecretSetupAdvice, CiSecretTarget } from '../ci-secrets.js' +import { createCiSecretEntries, detectCiSecretTargets, getCiSecretTargetLabel, listExistingCiSecretKeys, uploadCiSecrets } from '../ci-secrets.js' +import { trackBuilderOnboardingStep } from '../telemetry.js' import { getPhaseLabel, @@ -91,6 +93,59 @@ const OnboardingApp: FC = ({ appId, initialProgress, iosDir, apikey }) const startStep = getResumeStep(initialProgress) const [step, setStep] = useState(startStep === 'welcome' ? 'welcome' : startStep) + + // Telemetry: resolve org id once + emit per-step events + const stepTimingRef = useRef<{ step: OnboardingStep | null, startedAt: number }>({ + step: null, + startedAt: Date.now(), + }) + // Buffer of telemetry events that occurred before `resolvedOrgId` landed. + // Drained in order when the org id becomes available. Without this buffer, + // any step transitions during the async org-id resolution (which involves + // two HTTP round-trips: createSupabaseClient + getOrganizationId) would be + // dropped from the funnel. + const pendingTelemetryRef = useRef>([]) + const [resolvedOrgId, setResolvedOrgId] = useState(null) + const resolvedApiKeyRef = useRef(apikey ?? null) + const orgIdResolvedRef = useRef(false) + // Captures the mapped error category at handleError time so the telemetry + // useEffect can pass it through without re-mapping a reconstructed Error + // (which would have lost the .status / .phase / instanceof discriminators). + const errorCategoryRef = useRef(undefined) + + useEffect(() => { + if (resolvedApiKeyRef.current) + return + const saved = findSavedKeySilent() + if (saved) + resolvedApiKeyRef.current = saved + }, []) + + useEffect(() => { + if (orgIdResolvedRef.current || !resolvedApiKeyRef.current) + return + orgIdResolvedRef.current = true + + let cancelled = false + void (async () => { + const supabase = await createSupabaseClient(resolvedApiKeyRef.current!, undefined, undefined, true) + .catch(() => null) + if (!supabase || cancelled) + return + const orgId = await getOrganizationId(supabase, appId).catch(() => null) + if (orgId && !cancelled) + setResolvedOrgId(orgId) + })() + + return () => { + cancelled = true + } + }, [appId]) + const [log, setLog] = useState([]) const [error, setError] = useState(null) const [retryCount, setRetryCount] = useState(0) @@ -135,6 +190,71 @@ const OnboardingApp: FC = ({ appId, initialProgress, iosDir, apikey }) useEffect(() => { issuerIdRef.current = issuerId }, [issuerId]) + + // Emit telemetry on every step transition (including initial mount). + // Sequencing: + // 1. If `resolvedOrgId` just became available, drain the backlog first. + // 2. Skip same-step re-renders (orgId-lands triggers a re-fire — we don't + // want to re-emit the current step, only drain the backlog). + // 3. Otherwise compute the new event, then either emit immediately (orgId + // available) or queue it (orgId still loading). + useEffect(() => { + if (!resolvedApiKeyRef.current) + return + + const previous = stepTimingRef.current + const isDuplicateStep = previous.step !== null && previous.step === step && step !== 'error' + + // (1) Drain the backlog if org id is now available, even when the current + // step is a duplicate (e.g., this effect fired because resolvedOrgId moved + // from null to a real value, not because step changed). + if (resolvedOrgId && pendingTelemetryRef.current.length > 0) { + for (const queued of pendingTelemetryRef.current) { + void trackBuilderOnboardingStep({ + apikey: resolvedApiKeyRef.current, + appId, + orgId: resolvedOrgId, + platform: 'ios', + ...queued, + }) + } + pendingTelemetryRef.current = [] + } + + // (2) Now safely skip the duplicate-step path. + if (isDuplicateStep) + return + + const now = Date.now() + // Initial step (previous.step === null) and same-step error re-entries have + // no meaningful previous-step duration. + const durationMs = previous.step === null || previous.step === step + ? undefined + : now - previous.startedAt + + const eventPayload = { + step, + durationMs, + errorCategory: step === 'error' ? errorCategoryRef.current : undefined, + } + + stepTimingRef.current = { step, startedAt: now } + + // (3) Either fire immediately or buffer. + if (resolvedOrgId) { + void trackBuilderOnboardingStep({ + apikey: resolvedApiKeyRef.current, + appId, + orgId: resolvedOrgId, + platform: 'ios', + ...eventPayload, + }) + } + else { + pendingTelemetryRef.current.push(eventPayload) + } + }, [step, appId, resolvedOrgId, error]) + const [teamId, setTeamId] = useState(initialProgress?.completedSteps.certificateCreated?.teamId || '') const [certData, setCertData] = useState(initialProgress?.completedSteps.certificateCreated || null) const [profileData, setProfileData] = useState(initialProgress?.completedSteps.profileCreated || null) @@ -320,6 +440,10 @@ const OnboardingApp: FC = ({ appId, initialProgress, iosDir, apikey }) setStep('api-key-instructions') return } + // Capture the mapped category BEFORE we collapse err to a string. + // The telemetry useEffect will read this ref instead of re-mapping a + // reconstructed `new Error(message)` (which has no discriminators). + errorCategoryRef.current = mapIosOnboardingError(err, failedStep) const message = err instanceof Error ? err.message : String(err) const nextRetryCount = retryCount + 1 const bundlePath = writeOnboardingSupportBundle({ @@ -486,6 +610,7 @@ const OnboardingApp: FC = ({ appId, initialProgress, iosDir, apikey }) if (result.success && existsSync(join(process.cwd(), iosDir))) { addLog(`✔ Native iOS platform created with ${addIosCommand}`) setError(null) + errorCategoryRef.current = undefined setRetryCount(0) // Re-run the welcome → platform check inline rather than detouring // through the legacy platform-select step. @@ -2356,6 +2481,7 @@ const OnboardingApp: FC = ({ appId, initialProgress, iosDir, apikey }) onChange={async (value) => { if (value === 'retry') { setError(null) + errorCategoryRef.current = undefined pickerOpenedRef.current = false setStep(retryStep) } @@ -2377,6 +2503,7 @@ const OnboardingApp: FC = ({ appId, initialProgress, iosDir, apikey }) setCertData(null) setProfileData(null) setError(null) + errorCategoryRef.current = undefined setRetryCount(0) pickerOpenedRef.current = false setSupportBundlePath(null) diff --git a/cli/src/build/request.ts b/cli/src/build/request.ts index 7b64329837..f975c72502 100644 --- a/cli/src/build/request.ts +++ b/cli/src/build/request.ts @@ -53,12 +53,14 @@ import { startCaptureForJob, } from '../ai/log-capture' import { renderMarkdown } from '../ai/render-markdown' +import { trackAiAnalysisChoice, trackAiAnalysisResult } from '../ai/telemetry' import { assertCliPermission, canPromptInteractively, createSupabaseClient, findSavedKey, getConfig, getOrganizationId, sendEvent } from '../utils' import { mergeCredentials, MIN_OUTPUT_RETENTION_SECONDS, parseInAppUpdatePriority, parseOptionalBoolean, parseOutputRetentionSeconds } from './credentials' import { buildProvisioningMap } from './credentials-command' import { writeBuildOutputRecord } from './output-record' import { getPlatformDirFromCapacitorConfig } from './platform-paths' import { handleCustomMsg } from './qr.js' +import { trackBuilderUpload } from './telemetry.js' /** * Callback interface for build logging. @@ -1696,6 +1698,19 @@ export async function requestBuildInternal(appId: string, options: BuildRequestO // Upload using TUS protocol log.uploadProgress(0) + const uploadStartedAt = Date.now() + const buildModeForTelemetry = options.buildMode || 'release' + void trackBuilderUpload({ + apikey: options.apikey, + appId, + orgId, + platform, + buildMode: buildModeForTelemetry, + jobId: buildRequest.job_id, + sizeBytes: zipStats.size, + phase: 'started', + }) + await new Promise((resolve, reject) => { const upload = new tus.Upload(zipBuffer as any, { endpoint: buildRequest.upload_url, @@ -1725,7 +1740,19 @@ export async function requestBuildInternal(appId: string, options: BuildRequestO } }, // Callback for errors which cannot be fixed using retries - onError(error) { + async onError(error) { + await trackBuilderUpload({ + apikey: options.apikey, + appId, + orgId, + platform, + buildMode: buildModeForTelemetry, + jobId: buildRequest.job_id, + sizeBytes: zipStats.size, + phase: 'failed', + durationSeconds: (Date.now() - uploadStartedAt) / 1000, + error, + }) log.error(`Upload error: ${error.message}`) if (error instanceof tus.DetailedError) { const body = error.originalResponse?.getBody() @@ -1760,6 +1787,17 @@ export async function requestBuildInternal(appId: string, options: BuildRequestO }, // Callback for once the upload is completed onSuccess() { + void trackBuilderUpload({ + apikey: options.apikey, + appId, + orgId, + platform, + buildMode: buildModeForTelemetry, + jobId: buildRequest.job_id, + sizeBytes: zipStats.size, + phase: 'succeeded', + durationSeconds: (Date.now() - uploadStartedAt) / 1000, + }) log.uploadProgress(100) if (verbose) { log.success('TUS upload completed successfully') @@ -1948,7 +1986,29 @@ export async function requestBuildInternal(appId: string, options: BuildRequestO const AI_WARNING = '⚠ AI can make mistakes. Always verify the diagnosis against the full log before applying the suggested fix.' - const runCapgoAi = async (): Promise => { + // Closed-enum mapper for PostAnalyzeResult.kind → telemetry result tag. + // Never include the analysis text itself in telemetry. + const mapPostAnalyzeResultKind = (kind: PostAnalyzeResult['kind']): 'success' | 'already_analyzed' | 'too_big' | 'error' => { + if (kind === 'ok') + return 'success' + if (kind === 'already_analyzed') + return 'already_analyzed' + if (kind === 'too_big') + return 'too_big' + return 'error' + } + + const runCapgoAi = async (choice: 'capgo_ai' | 'auto_upload', triggeredBy: 'menu' | 'ci_flag'): Promise => { + await trackAiAnalysisChoice({ + apikey: options.apikey, + orgId, + appId, + platform, + jobId: capturedJobId!, + choice, + triggeredBy, + }) + const logsPath = `${process.env.CAPGO_AI_LOG_BASE_DIR || '/tmp/capgo-builds'}/${capturedJobId}.log` let logs = '' try { @@ -1987,6 +2047,18 @@ export async function requestBuildInternal(appId: string, options: BuildRequestO aiSpinner?.stop('Capgo AI finished') } + // Telemetry — closed-enum result only, never the analysis text. + const resultTag = mapPostAnalyzeResultKind(result.kind) + await trackAiAnalysisResult({ + apikey: options.apikey, + orgId, + appId, + platform, + jobId: capturedJobId!, + result: resultTag, + errorStatus: result.kind === 'error' ? result.status : undefined, + }) + if (result.kind === 'ok') { stream.write(`\n--- AI analysis ---\n${renderMarkdown(result.analysis, isInteractive)}\n\n${AI_WARNING}\n`) } @@ -2002,11 +2074,32 @@ export async function requestBuildInternal(appId: string, options: BuildRequestO } const runLocalAi = async (): Promise => { + await trackAiAnalysisChoice({ + apikey: options.apikey, + orgId, + appId, + platform, + jobId: capturedJobId!, + choice: 'local_ai', + triggeredBy: 'menu', + }) const promptPath = await writeLocalAiFile(capturedJobId!) keepPromptFile = true process.stdout.write(`\nSaved prompt to ${promptPath}\nPoint your local AI (Claude, Codex, aider, etc.) at this file.\n${AI_WARNING}\n`) } + const emitSkipChoice = async (): Promise => { + await trackAiAnalysisChoice({ + apikey: options.apikey, + orgId, + appId, + platform, + jobId: capturedJobId!, + choice: 'skip', + triggeredBy: (behavior === 'auto_upload' || behavior === 'skip') ? 'ci_flag' : 'menu', + }) + } + async function showMenu(): Promise { if (await isLogTooBig(capturedJobId!)) { process.stdout.write('Log too big for AI analysis (>10 MB). Offering local AI instead.\n') @@ -2022,21 +2115,24 @@ export async function requestBuildInternal(appId: string, options: BuildRequestO ], }) if (choice === 'capgo') - await runCapgoAi() + await runCapgoAi('capgo_ai', 'menu') else if (choice === 'local') await runLocalAi() + else + await emitSkipChoice() } try { if (behavior === 'skip') { - // nothing + await emitSkipChoice() } else if (behavior === 'auto_upload') { if (await isLogTooBig(capturedJobId)) { process.stderr.write('Log too big for AI analysis (>10 MB), skipping\n') + await emitSkipChoice() } else { - await runCapgoAi() + await runCapgoAi('auto_upload', 'ci_flag') } } else { @@ -2045,6 +2141,7 @@ export async function requestBuildInternal(appId: string, options: BuildRequestO const wants = await confirm({ message: 'Build failed. Run AI analysis?' }) if (!wants || typeof wants === 'symbol') { // user cancelled or declined — skip + await emitSkipChoice() } else { await showMenu() diff --git a/cli/src/build/telemetry.ts b/cli/src/build/telemetry.ts new file mode 100644 index 0000000000..f14d1f0a82 --- /dev/null +++ b/cli/src/build/telemetry.ts @@ -0,0 +1,89 @@ +import { sendEvent } from '../utils.js' + +export type BuilderUploadFailureCategory + = | 'network_error' + | 'unauthorized' + | 'payload_too_large' + | 'storage_failure' + | 'unknown' + +type BuilderUploadPhase = 'started' | 'succeeded' | 'failed' + +export interface TrackBuilderUploadInput { + apikey: string + appId: string + orgId: string + platform: 'ios' | 'android' + buildMode: string + jobId: string + sizeBytes: number + phase: BuilderUploadPhase + durationSeconds?: number + error?: unknown +} + +interface MaybeTusResponse { + originalResponse?: { getStatus?: () => unknown } +} + +function getTusErrorStatus(error: unknown): number | undefined { + if (!error || typeof error !== 'object') + return undefined + const candidate = (error as MaybeTusResponse).originalResponse?.getStatus?.() + return typeof candidate === 'number' ? candidate : undefined +} + +export function mapBuilderUploadError(error: unknown): BuilderUploadFailureCategory { + const status = getTusErrorStatus(error) + if (status === 401 || status === 403) + return 'unauthorized' + if (status === 413) + return 'payload_too_large' + if (status !== undefined && status >= 500 && status < 600) + return 'storage_failure' + if (status === undefined || status === 0) + return 'network_error' + return 'unknown' +} + +const EVENT_NAME_BY_PHASE: Record = { + started: 'Builder Upload Started', + succeeded: 'Builder Upload Succeeded', + failed: 'Builder Upload Failed', +} + +const ICON_BY_PHASE: Record = { + started: '⬆️', + succeeded: '📦', + failed: '🚫', +} + +export async function trackBuilderUpload(input: TrackBuilderUploadInput): Promise { + const tags: Record = { + app_id: input.appId, + platform: input.platform, + build_mode: input.buildMode, + job_id: input.jobId, + upload_size_bytes: String(input.sizeBytes), + } + + if (typeof input.durationSeconds === 'number' && Number.isFinite(input.durationSeconds)) + tags.upload_duration_seconds = String(Math.round(input.durationSeconds)) + + if (input.phase === 'failed' && input.error !== undefined) + tags.failure_category = mapBuilderUploadError(input.error) + + try { + await sendEvent(input.apikey, { + event: EVENT_NAME_BY_PHASE[input.phase], + channel: 'build-lifecycle', + icon: ICON_BY_PHASE[input.phase], + notify: false, + user_id: input.orgId, + tags, + }) + } + catch { + // never throw — telemetry must not break the build flow + } +} diff --git a/docs/superpowers/specs/2026-05-18-capgo-builder-posthog-tracking-design.md b/docs/superpowers/specs/2026-05-18-capgo-builder-posthog-tracking-design.md new file mode 100644 index 0000000000..d5c7147af4 --- /dev/null +++ b/docs/superpowers/specs/2026-05-18-capgo-builder-posthog-tracking-design.md @@ -0,0 +1,249 @@ +# Capgo Builder onboarding + build PostHog tracking + +**Date:** 2026-05-18 +**Branch:** `feat/builder-tracking-posthog` +**Worktree:** `capgo-builder-tracking-wt` +**Scope:** changes confined to the `capgo` repo. The `capgo_builder` repo is **not** modified. + +## Goal + +Mirror the existing Capgo onboarding-progress PostHog tracking onto the **Capgo Builder** flow so we can see (a) where users drop off in the iOS / Android credential-setup wizard and (b) build-lifecycle outcomes. Privacy posture matches the existing CLI exception telemetry: no raw error strings, no file paths, no credentials — only categorized enums and stable identifiers. + +## Non-goals + +- No new tracking for runtime OTA updates (decision: out of scope). +- No tracking added inside the `capgo_builder` repo. `build_started` is derived server-side by the existing reconciliation cron, which already polls the builder. +- No new analytics dashboards; events feed the existing PostHog project. +- No removal or refactor of existing tracking. `sendEventToTracking` is reused as-is. + +## Event families + +### 1. Onboarding step events + +One event per CLI wizard step transition. Sent from the CLI through the existing `/private/events` endpoint so the existing dual-writer (LogSnag + PostHog) and org grouping apply automatically. + +**Event:** `Builder Onboarding Step` +**Channel:** `builder-onboarding` +**Icon:** `🧭` + +**Payload:** +```ts +{ + event: 'Builder Onboarding Step', + user_id: orgId, // org id used as user_id (existing convention, see on_app_create.ts:138) + channel: 'builder-onboarding', + icon: '🧭', + notify: false, + groups: { organization: orgId }, + tags: { + step: 'api-key-instructions', // value from OnboardingStep | AndroidOnboardingStep + platform: 'ios' | 'android', + app_id: 'com.example.app', + duration_ms: '1234', // ms spent on previous step; optional + error_category: 'apple_api_unauthorized', // ONLY when step === 'error' + }, +} +``` + +**Closed enum: `error_category`** + +iOS: +- `apple_api_unauthorized` — 401 from App Store Connect +- `apple_api_rate_limited` — 429 from App Store Connect +- `cert_limit_reached` — Apple cert quota hit +- `profile_creation_failed` — non-401/429 failure during profile creation +- `p8_invalid` — supplied P8 file unreadable or malformed +- `unknown` — anything that does not match an enum value above + +Android: +- `keystore_invalid` — supplied keystore unreadable or aliases missing +- `google_oauth_failed` — Google sign-in did not return a valid token +- `play_account_id_invalid` — pasted Play developer account ID rejected +- `unknown` — fallback + +The CLI maps caught exceptions to one of these enum values **before** building the payload. Raw error messages never leave the CLI. + +### 2. Builder upload events (project tarball → builder storage) + +Three events fired from the CLI around the TUS upload between `Build Requested` and `Build Started`. Until this set was added, the gap between "build job row inserted" and "builder picks it up" was an observability blind spot — a failed CLI-to-builder upload would never surface in PostHog. + +**Channel:** `build-lifecycle` + +| Event | Source | When | Icon | +| --- | --- | --- | --- | +| `Builder Upload Started` | `cli/src/build/request.ts` (just before `tus.Upload.start()`) | TUS handshake about to begin | ⬆️ | +| `Builder Upload Succeeded` | Same site, `onSuccess` callback | TUS upload completes; control passes to `/build/start/{job_id}` | 📦 | +| `Builder Upload Failed` | Same site, `onError` callback | TUS upload fatally fails | 🚫 | + +**Payload:** +```ts +{ + event: 'Builder Upload Started' | 'Builder Upload Succeeded' | 'Builder Upload Failed', + channel: 'build-lifecycle', + icon: /* see table */, + notify: false, + user_id: orgId, + groups: { organization: orgId }, + tags: { + app_id, + platform: 'ios' | 'android', + build_mode: string, + job_id, // builder job id from `Build Requested` (for correlation) + upload_size_bytes, // exact zip size from `zipStats.size` + upload_duration_seconds?, // succeeded/failed only — wall-clock from `tus.Upload.start()` to terminal callback + failure_category?, // failed only + }, +} +``` + +**Closed enum: `failure_category` for upload failures** + +- `network_error` — TUS error with no `originalResponse` (connection dropped, DNS, timeout) +- `unauthorized` — HTTP 401 or 403 from the upload endpoint +- `payload_too_large` — HTTP 413 +- `storage_failure` — HTTP 5xx from R2/S3 +- `unknown` — any other terminal status + +Mapping happens in the CLI helper via structural typing on `error.originalResponse?.getStatus?.()` (no hard import of `tus.DetailedError`). + +### 3. Build lifecycle events + +Fired entirely server-side. The `capgo_builder` repo is not modified — the reconciliation cron already polls the builder for status, so transition detection happens there. + +**Channel:** `build-lifecycle` + +| Event | Source file | When | Icon | +| ------------------ | ------------------------------------------------ | ------------------------------------------------------------------------------ | ---- | +| `Build Requested` | `public/build/request.ts` (after insert) | Build row successfully inserted into `build_requests` | 🛠️ | +| `Build Started` | `triggers/cron_reconcile_build_status.ts` | Status transitions from a non-running state into `running` | ⏳ | +| `Build Succeeded` | `triggers/cron_reconcile_build_status.ts` | Terminal status `success` reached for the first time (was non-terminal before) | ✅ | +| `Build Failed` | `triggers/cron_reconcile_build_status.ts` | Terminal status `failed` reached for the first time | ❌ | +| `Build Timed Out` | `triggers/cron_reconcile_build_status.ts` | `timeoutApplied === true` and was non-terminal before | ⏰ | + +**Payload:** +```ts +{ + event: 'Build Requested' | 'Build Started' | 'Build Succeeded' | 'Build Failed' | 'Build Timed Out', + user_id: orgId, + channel: 'build-lifecycle', + icon: /* see table */, + notify: false, + groups: { organization: orgId }, + tags: { + app_id, + platform: 'ios' | 'android', + build_mode: 'development' | 'production', + duration_seconds: '120', // terminal events only + failure_category: 'timeout' | 'builder_error' | 'validation_error' | 'unknown', // Failed / Timed Out only + }, +} +``` + +**Closed enum: `failure_category`** + +- `timeout` — `timeoutApplied` was set on this reconciliation pass +- `builder_error` — builder reported a terminal failure with a non-empty error +- `validation_error` — build_requests row marked failed before the builder accepted it (e.g., invalid `build_mode`, missing credentials) +- `unknown` — anything else + +Mapping happens in `cron_reconcile_build_status.ts` next to the existing status-update logic. + +## Architecture + +```text +ONBOARDING: + + CLI wizard step reducer + └─→ trackOnboardingStep(step, platform, appId, error?) [cli/src/build/onboarding/telemetry.ts] + └─→ POST /private/events [reuses existing endpoint] + └─→ backend validates body, resolves orgId via resolveTrackingUserId + └─→ sendEventToTracking(...) [supabase/functions/_backend/utils/tracking.ts] + ├─→ logsnag(c).track(...) + └─→ trackPosthogEvent(c, {...}) + +BUILDS: + + public/build/request.ts + └─ insert build_requests row succeeded + └─→ sendEventToTracking('Build Requested') + + triggers/cron_reconcile_build_status.ts (cron) + └─ for each stale build: + ├─ fetch latest builder status + ├─ compare to previous DB status + └─ on transition: + ├─ pending|queued → running : sendEventToTracking('Build Started') + ├─ * → success : sendEventToTracking('Build Succeeded') + ├─ * → failed : sendEventToTracking('Build Failed') + └─ timeoutApplied : sendEventToTracking('Build Timed Out') +``` + +### Why the CLI does not call PostHog directly + +The CLI already has `capgo/cli/src/posthog.ts`, but it is scoped to exception capture (`$exception` events with stack traces). Routing onboarding events through the backend gives us: + +- Org grouping for free (`groups: { organization: orgId }`) without the CLI having to know the org id +- Dual-write to LogSnag (existing convention) +- Auth-gated event source (anyone with a CLI token is a real user) +- Consistency with `on_app_create.ts` and the other backend trackers + +### Why reuse `/private/events` instead of a new endpoint + +The existing `/private/events` Hono handler (lines 79–162 of `events.ts`) already implements every concern the spec needed for a new endpoint: auth via `middlewareV2`, org resolution via `resolveTrackingUserId` (verifies the caller can post for that org), app_id permission check from `tags.app_id`, sendEventToTracking with org grouping. Adding a second endpoint would duplicate ~80 lines of working code. The CLI helper just POSTs with `event: 'Builder Onboarding Step'` and the new event flows through the same code path. + +### Why `build_started` does not need capgo_builder changes + +The existing reconciliation cron already fetches builder job status for every stale build. We can detect the queued → running transition by comparing the new builder status against the persisted `build_requests.status` before this pass overwrites it. The transition fires the event; the existing update writes the new status. + +## File changes + +All paths relative to the `capgo` repo root. + +### New files + +- `cli/src/build/onboarding/telemetry.ts` — Exposes `trackOnboardingStep(input)`. Best-effort `fetch` to the existing `/private/events` endpoint with `AbortController` timeout (1500ms, matches `posthog.ts`). Never throws. +- `tests/build-lifecycle-tracking.unit.test.ts` — Cron-side tests: transitions emit the right events, idempotency when re-running on the same build, `failure_category` mapping. + +### Modified files + +- `supabase/functions/_backend/public/build/request.ts` — After the successful insert (between the existing `Build job created` cloudlog at line 307 and the `c.json` return at line 316), emit `Build Requested`. Payload sourced from the just-inserted row. +- `supabase/functions/_backend/triggers/cron_reconcile_build_status.ts` — Capture `build.status` (the previous status) into a local before the `.update(...)` call. After the update, compare previous vs. `effectiveStatus` and emit the matching transition event. Wrap each emission in `backgroundTask(c, ...)` so the cron is not delayed by tracking I/O. +- `cli/src/build/onboarding/ui/app.tsx` — iOS state lives here (`useState` at line 91, ~20 `setStep(...)` call sites). Add **one** `useEffect(() => { ... }, [step])` near the top of the `OnboardingApp` component that fires `trackOnboardingStep({ step, platform: 'ios', app_id, duration_ms, error_category? })`. Use a `useRef<{ step, startedAt }>` to remember the previous step and compute `duration_ms = Date.now() - startedAt`. The effect updates the ref at the end so the next transition has a fresh baseline. +- `cli/src/build/onboarding/android/ui/app.tsx` — Same single-`useEffect` wiring with platform `'android'`. +- `cli/src/build/onboarding/types.ts` — Export the iOS `OnboardingErrorCategory` union for `telemetry.ts`. +- `cli/src/build/onboarding/android/types.ts` — Export the Android `OnboardingErrorCategory` union. + +### Not modified + +- `capgo_builder/` submodule — explicitly out of scope. +- `aliproxy/` — unrelated (Alibaba CDN proxy for the updater). +- `cloudflare_workers/` — no builder code lives here. +- `cli/src/posthog.ts` — kept as exception-only telemetry. Generic event tracking lives in the new `telemetry.ts` file to keep responsibilities separate. + +## Privacy posture + +- **Closed-enum error categories**: the CLI maps caught exceptions to a known string before sending. Raw error messages, paths, and credential material never leave the CLI process. +- **Reused sanitizer**: where any string field is unavoidable (e.g., during future extensions), `sanitizeTelemetryText` from `cli/src/posthog.ts` is the canonical pre-send filter. +- **No user_id fingerprinting**: `user_id` in the payload is the org id, matching `on_app_create.ts:138`. Individual users are not distinguished in PostHog. +- **No CLI opt-out env var in this PR**: this PR does not introduce a `CAPGO_DISABLE_TELEMETRY` or `CAPGO_DISABLE_POSTHOG` check in any new helper. The existing exception-capture helper (`cli/src/posthog.ts`, introduced in PR #2088) honors those vars, but the new helpers do not. Adding a unified opt-out at the `sendEvent` layer is deferred to a follow-up. +- **App id is sent**: the user explicitly chose to include `app_id` as a tag, matching existing `on_app_create.ts:141` behavior. Bundle IDs are not treated as PII in the existing tracking surface. + +## Error handling + +- **CLI helper**: `try { await fetch(...) } catch { /* swallow */ }`. AbortController with 1500ms timeout. Never blocks the wizard. Never logs to stdout (would pollute Ink UI). +- **Backend endpoint**: Returns 200 even when `sendEventToTracking` reports per-provider failures (already handled inside `sendEventToTracking`). Returns 400 only for schema validation errors. +- **Cron transitions**: emit inside `backgroundTask(c, ...)`. Per-build failures of tracking do not abort the reconciliation loop (already wrapped in `Promise.allSettled`). +- **Idempotency**: cron only processes stale (non-terminal) builds. Even so, the transition check uses the previous DB status explicitly — re-running the cron on the same build cannot double-fire. + +## Testing strategy + +- **Unit (CLI)**: mock `sendEvent`, assert payload shape, assert error-category mapping, assert error swallowing. +- **Unit (backend endpoint)**: mock `sendEventToTracking`, assert it is called with the expected `event`, `tags`, `groups`. Assert 401 without auth, 400 on bad payload. +- **Unit (cron)**: feed synthetic builder responses, assert correct transition events fire and only the expected ones. Re-run on the same build → no duplicate emission. +- **Existing test harness**: extends patterns in `tests/tracking.unit.test.ts`, `tests/posthog.unit.test.ts`, and `tests/on-error-posthog.unit.test.ts`. + +## Open items / explicit decisions + +- **No `Build Cancelled` event** for now. `public/build/cancel.ts` exists and could fire it, but cancellations were not in the user's scope. Easy to add later. +- **No per-org rate limit** on onboarding events at the reused `/private/events` endpoint. The wizard has fewer than 35 transitions per run; abuse risk is low. Revisit if we ever see > 1000 events/org/day. +- **Duration timing is wall-clock from CLI**. Users who walk away mid-wizard and return next day will produce one huge `duration_ms` value. We accept this — it is also signal (long pauses mean drop-off). diff --git a/supabase/functions/_backend/public/build/ai_analyze.ts b/supabase/functions/_backend/public/build/ai_analyze.ts index 72323d0bf5..7b329d89f2 100644 --- a/supabase/functions/_backend/public/build/ai_analyze.ts +++ b/supabase/functions/_backend/public/build/ai_analyze.ts @@ -1,9 +1,10 @@ import type { Context } from 'hono' import type { Database } from '../../utils/supabase.types.ts' import { quickError, simpleError } from '../../utils/hono.ts' -import { cloudlog, cloudlogErr } from '../../utils/logging.ts' +import { cloudlog, cloudlogErr, serializeError } from '../../utils/logging.ts' import { checkPermission } from '../../utils/rbac.ts' import { supabaseApikey } from '../../utils/supabase.ts' +import { sendEventToTracking } from '../../utils/tracking.ts' import { getEnv } from '../../utils/utils.ts' interface BuilderAnalysisResponse { @@ -11,6 +12,65 @@ interface BuilderAnalysisResponse { error?: string } +type AiAnalysisResult + = | 'success' + | 'already_analyzed' + | 'invalid_state' + | 'unauthorized' + | 'builder_error' + | 'config_error' + +interface EmitAiAnalysisResultInput { + appId: string + jobId: string + result: AiAnalysisResult + ownerOrg?: string + logsBytes: number + durationMs?: number +} + +/** + * Emit the `AI Build Analysis Result` event for an exit branch. + * + * Privacy boundary: the AI diagnosis text from the builder MUST NOT cross into any + * tag here. Only the closed-enum `result`, size/duration metadata, and stable + * identifiers are sent. Callers fire this before throwing (or before returning a + * successful response) so every exit branch produces exactly one Result event. + */ +async function emitAiAnalysisResult(c: Context, input: EmitAiAnalysisResultInput): Promise { + const tags: Record = { + app_id: input.appId, + job_id: input.jobId, + result: input.result, + logs_bytes: String(input.logsBytes), + } + if (input.durationMs !== undefined && Number.isFinite(input.durationMs)) + tags.duration_ms = String(Math.round(input.durationMs)) + + // Telemetry MUST NOT break the AI analyze flow. sendEventToTracking swallows + // per-provider errors internally, but defend against an unexpected throw at + // the orchestration layer (e.g. backgroundTask unavailable in tests). + try { + await sendEventToTracking(c, { + event: 'AI Build Analysis Result', + channel: 'build-lifecycle', + icon: '🤖', + notify: false, + user_id: input.ownerOrg, + groups: input.ownerOrg ? { organization: input.ownerOrg } : undefined, + tags, + }) + } + catch (error) { + cloudlogErr({ + requestId: c.get('requestId'), + message: 'AI Build Analysis Result telemetry failed', + result: input.result, + error: serializeError(error), + }) + } +} + export async function aiAnalyzeBuild( c: Context, jobId: string, @@ -18,6 +78,8 @@ export async function aiAnalyzeBuild( apikey: Database['public']['Tables']['apikeys']['Row'], logs: string, ): Promise { + const logsBytes = logs?.length ?? 0 + // 1. Permission check (reuse app.build_native — see design rationale) if (!(await checkPermission(c, 'app.build_native', { appId }))) { cloudlogErr({ @@ -27,6 +89,8 @@ export async function aiAnalyzeBuild( app_id: appId, user_id: apikey.user_id, }) + // No row yet — `ownerOrg` is unknown for this branch. + await emitAiAnalysisResult(c, { appId, jobId, result: 'unauthorized', logsBytes }) throw simpleError('unauthorized', 'You do not have permission to analyze this build') } @@ -34,7 +98,7 @@ export async function aiAnalyzeBuild( const supabase = supabaseApikey(c, apikey.key) const { data: row, error: selectErr } = await supabase .from('build_requests') - .select('app_id, status, ai_analyzed') + .select('app_id, status, ai_analyzed, owner_org') .eq('builder_job_id', jobId) .eq('app_id', appId) .maybeSingle() @@ -46,6 +110,7 @@ export async function aiAnalyzeBuild( job_id: jobId, error: selectErr.message, }) + await emitAiAnalysisResult(c, { appId, jobId, result: 'builder_error', logsBytes }) throw simpleError('internal_error', 'Failed to fetch build request') } @@ -57,28 +122,63 @@ export async function aiAnalyzeBuild( app_id: appId, user_id: apikey.user_id, }) + // Row is null — `ownerOrg` cannot be resolved for this branch. + await emitAiAnalysisResult(c, { appId, jobId, result: 'unauthorized', logsBytes }) throw simpleError('unauthorized', 'You do not have permission to analyze this build') } + const ownerOrg = row.owner_org + if (row.status !== 'failed') { + await emitAiAnalysisResult(c, { appId, jobId, result: 'invalid_state', ownerOrg, logsBytes }) throw simpleError('invalid_state', 'AI analysis only available for failed builds') } if (row.ai_analyzed === true) { + await emitAiAnalysisResult(c, { appId, jobId, result: 'already_analyzed', ownerOrg, logsBytes }) // 409 (not the simpleError default of 400) — CLI branches on res.status === 409 for this case throw quickError(409, 'already_analyzed', 'AI analysis already requested for this job') } + // Fire the Requested event only after structural guards pass. "Requested" means + // a structurally valid analysis attempt for a failed, not-yet-analyzed build + // was about to be made. Result events still fire at every exit branch. + // Telemetry MUST NOT break the AI analyze flow. + try { + await sendEventToTracking(c, { + event: 'AI Build Analysis Requested', + channel: 'build-lifecycle', + icon: '🤖', + notify: false, + user_id: ownerOrg, + groups: { organization: ownerOrg }, + tags: { + app_id: appId, + job_id: jobId, + logs_bytes: String(logsBytes), + }, + }) + } + catch (error) { + cloudlogErr({ + requestId: c.get('requestId'), + message: 'AI Build Analysis Requested telemetry failed', + error: serializeError(error), + }) + } + // 3. Proxy to capgo_builder const builderUrl = getEnv(c, 'BUILDER_URL') const builderApiKey = getEnv(c, 'BUILDER_API_KEY') if (!builderUrl || !builderApiKey) { + await emitAiAnalysisResult(c, { appId, jobId, result: 'config_error', ownerOrg, logsBytes }) throw simpleError('config_error', 'Builder service not configured') } // 60s timeout — matches the CLI's own request timeout. Without this, a hung // Workers AI call would hold the edge fn open until the platform's own // 150s wall-clock timeout, wasting compute and producing a vaguer error. + const builderStartedAt = Date.now() let builderResp: Response try { builderResp = await fetch(`${builderUrl}/jobs/${jobId}/ai-analyze`, { @@ -92,6 +192,7 @@ export async function aiAnalyzeBuild( }) } catch (err) { + const durationMs = Date.now() - builderStartedAt const isTimeout = err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError') cloudlogErr({ requestId: c.get('requestId'), @@ -99,10 +200,12 @@ export async function aiAnalyzeBuild( job_id: jobId, error: err instanceof Error ? err.message : String(err), }) + await emitAiAnalysisResult(c, { appId, jobId, result: 'builder_error', ownerOrg, logsBytes, durationMs }) throw simpleError('builder_error', isTimeout ? 'AI analysis timed out' : 'AI analysis request failed') } if (!builderResp.ok) { + const durationMs = Date.now() - builderStartedAt const errText = await builderResp.text().catch(() => '') cloudlogErr({ requestId: c.get('requestId'), @@ -111,19 +214,24 @@ export async function aiAnalyzeBuild( status: builderResp.status, error: errText, }) + await emitAiAnalysisResult(c, { appId, jobId, result: 'builder_error', ownerOrg, logsBytes, durationMs }) throw simpleError('builder_error', `AI analysis failed: ${errText}`) } const result = await builderResp.json() as BuilderAnalysisResponse if (!result || typeof result.analysis !== 'string') { + const durationMs = Date.now() - builderStartedAt cloudlogErr({ requestId: c.get('requestId'), message: 'Builder AI analyze returned malformed body', job_id: jobId, }) + await emitAiAnalysisResult(c, { appId, jobId, result: 'builder_error', ownerOrg, logsBytes, durationMs }) throw simpleError('builder_error', 'AI analysis returned malformed response') } + const durationMs = Date.now() - builderStartedAt + // 4. Flip the flag after the builder succeeds (idempotency) const { error: updateErr } = await supabase .from('build_requests') @@ -150,5 +258,7 @@ export async function aiAnalyzeBuild( user_id: apikey.user_id, }) + await emitAiAnalysisResult(c, { appId, jobId, result: 'success', ownerOrg, logsBytes, durationMs }) + return c.json({ analysis: result.analysis }, 200) } diff --git a/supabase/functions/_backend/public/build/request.ts b/supabase/functions/_backend/public/build/request.ts index aac6973706..ee18171819 100644 --- a/supabase/functions/_backend/public/build/request.ts +++ b/supabase/functions/_backend/public/build/request.ts @@ -1,9 +1,10 @@ import type { Context } from 'hono' import type { Database } from '../../utils/supabase.types.ts' import { quickError, simpleError } from '../../utils/hono.ts' -import { cloudlog, cloudlogErr } from '../../utils/logging.ts' +import { cloudlog, cloudlogErr, serializeError } from '../../utils/logging.ts' import { checkPermission } from '../../utils/rbac.ts' import { supabaseAdmin, supabaseApikey } from '../../utils/supabase.ts' +import { sendEventToTracking } from '../../utils/tracking.ts' import { getEnv } from '../../utils/utils.ts' export interface RequestBuildBody { @@ -313,6 +314,32 @@ export async function requestBuild( platform, }) + // Telemetry MUST NOT break the build request. sendEventToTracking swallows + // per-provider errors internally, but defend against an unexpected throw at + // the orchestration layer (e.g. backgroundTask unavailable in tests). + try { + await sendEventToTracking(c, { + event: 'Build Requested', + channel: 'build-lifecycle', + icon: '🛠️', + notify: false, + user_id: org_id, + groups: { organization: org_id }, + tags: { + app_id, + platform, + build_mode, + }, + }) + } + catch (error) { + cloudlogErr({ + requestId: c.get('requestId'), + message: 'Build Requested telemetry failed', + error: serializeError(error), + }) + } + return c.json({ build_request_id: buildRequestRow.id, job_id: builderJob.jobId, diff --git a/supabase/functions/_backend/public/build/start.ts b/supabase/functions/_backend/public/build/start.ts index 9dfc8e9aaa..ffc94c7357 100644 --- a/supabase/functions/_backend/public/build/start.ts +++ b/supabase/functions/_backend/public/build/start.ts @@ -1,6 +1,7 @@ import type { Context } from 'hono' import type { Database } from '../../utils/supabase.types.ts' import { HTTPException } from 'hono/http-exception' +import { emitBuildTransitionEvent } from '../../utils/build_tracking.ts' import { simpleError } from '../../utils/hono.ts' import { cloudlog, cloudlogErr } from '../../utils/logging.ts' import { checkPermission } from '../../utils/rbac.ts' @@ -88,7 +89,47 @@ async function markBuildAsFailed( ): Promise { // Access was already checked before starting the build. This trusted backend // status write uses service role because API-key RLS must stay read-only here. - const { error: updateError } = await supabaseAdmin(c) + // + // Fetch the row first to capture the fields we need for the lifecycle event + // (previousStatus for the CAS guard + platform/build_mode/owner_org for the + // payload). Without this, marking a build failed here would silently miss + // the `Build Failed` transition event, leaving the lifecycle funnel + // incomplete for the builder-rejection and outer-catch paths. + const adminClient = supabaseAdmin(c) + const { data: row, error: selectError } = await adminClient + .from('build_requests') + .select('status, platform, build_mode, owner_org') + .eq('builder_job_id', jobId) + .eq('app_id', appId) + .maybeSingle() + + if (selectError || !row) { + cloudlogErr({ + requestId: c.get('requestId'), + message: 'Failed to fetch build_request before marking as failed', + job_id: jobId, + error: selectError?.message ?? 'row not found', + }) + // Best-effort: still attempt the unguarded update so the user-facing status + // is correct even when we can't capture pre-transition context. + await adminClient + .from('build_requests') + .update({ + status: 'failed', + last_error: errorMessage, + updated_at: new Date().toISOString(), + }) + .eq('builder_job_id', jobId) + .eq('app_id', appId) + return + } + + const previousStatus = row.status + + // Optimistic concurrency-control: only one writer wins the transition. + // If another writer (cron, status poller, etc.) already advanced the row, + // the affected-row set is empty and we skip both the log and the emission. + const { data: updatedRows, error: updateError } = await adminClient .from('build_requests') .update({ status: 'failed', @@ -97,6 +138,8 @@ async function markBuildAsFailed( }) .eq('builder_job_id', jobId) .eq('app_id', appId) + .eq('status', previousStatus) + .select('id') if (updateError) { cloudlogErr({ @@ -105,14 +148,28 @@ async function markBuildAsFailed( job_id: jobId, error: updateError, }) + return } - else { + + if (updatedRows && updatedRows.length > 0) { cloudlog({ requestId: c.get('requestId'), message: 'Marked build_request as failed', job_id: jobId, error_message: errorMessage, }) + await emitBuildTransitionEvent(c, { + previousStatus, + effectiveStatus: 'failed', + timeoutApplied: false, + effectiveError: errorMessage, + build: { + app_id: appId, + platform: row.platform, + build_mode: row.build_mode, + owner_org: row.owner_org, + }, + }) } } @@ -157,7 +214,7 @@ export async function startBuild( const { data: buildRequest, error: buildRequestError } = await supabase .from('build_requests') - .select('id, app_id, owner_org') + .select('id, app_id, owner_org, status, platform, build_mode') .eq('builder_job_id', jobId) .eq('app_id', appId) .maybeSingle() @@ -239,7 +296,15 @@ export async function startBuild( // Update build_requests status to running. The builder response is trusted // backend data, and this write must not be exposed through API-key RLS. - const { error: updateError } = await supabaseAdmin(c) + // + // Optimistic concurrency-control (CAS) guard: `.eq('status', previousStatus)` + // ensures only one writer wins when concurrent start requests race. The + // `.select('id')` lets us detect whether this writer actually advanced the + // row; if `updatedRows` is empty, another writer already moved the status + // and emitted the transition event — skip emission to avoid double-firing. + const previousStatus = buildRequest.status + + const { data: updatedRows, error: updateError } = await supabaseAdmin(c) .from('build_requests') .update({ status: startedStatus, @@ -247,6 +312,8 @@ export async function startBuild( }) .eq('builder_job_id', jobId) .eq('app_id', boundAppId) + .eq('status', previousStatus) + .select('id') if (updateError) { cloudlogErr({ @@ -256,6 +323,21 @@ export async function startBuild( error: updateError.message, }) } + else if (updatedRows && updatedRows.length > 0) { + await emitBuildTransitionEvent(c, { + previousStatus, + effectiveStatus: startedStatus, + timeoutApplied: false, + build: { + app_id: buildRequest.app_id, + platform: buildRequest.platform, + build_mode: buildRequest.build_mode, + owner_org: buildRequest.owner_org, + }, + }) + } + // else: another writer already advanced the status (or it never matched + // previousStatus) — skip emission to avoid double-firing. // Generate JWT token for direct log stream access const jwtSecret = getEnv(c, 'JWT_SECRET') diff --git a/supabase/functions/_backend/public/build/status.ts b/supabase/functions/_backend/public/build/status.ts index e63c916e12..0454ff129b 100644 --- a/supabase/functions/_backend/public/build/status.ts +++ b/supabase/functions/_backend/public/build/status.ts @@ -10,6 +10,7 @@ import { normalizeBuildTimeoutSeconds, shouldApplyBuildTimeout, } from '../../utils/build_timeout.ts' +import { emitBuildTransitionEvent } from '../../utils/build_tracking.ts' import { simpleError } from '../../utils/hono.ts' import { cloudlog, cloudlogErr } from '../../utils/logging.ts' import { checkPermission } from '../../utils/rbac.ts' @@ -96,7 +97,7 @@ export async function getBuildStatus( // This prevents cross-app access by mixing an allowed app_id with another app's job_id. const { data: buildRequest, error: buildRequestError } = await supabase .from('build_requests') - .select('app_id, owner_org, platform') + .select('app_id, owner_org, platform, status, build_mode') .eq('builder_job_id', job_id) .maybeSingle() @@ -208,7 +209,15 @@ export async function getBuildStatus( // Use admin client: access was already verified above (RLS SELECT + checkPermission). // The data written comes from the trusted builder API, not from user input. // An RLS UPDATE policy would let API-key holders forge status/build-time, so we bypass RLS here. - const { error: updateError } = await supabaseAdmin(c) + // + // Optimistic concurrency-control (CAS) guard: `.eq('status', previousStatus)` ensures + // only one writer wins when two concurrent pollers race on the same job. The + // `.select('id')` lets us detect whether this writer actually advanced the row; + // if `updatedRows` is empty, another writer already moved the status and has + // (or will) emit the lifecycle event — skip emission here to avoid double-firing. + const previousStatus = buildRequest.status + + const { data: updatedRows, error: updateError } = await supabaseAdmin(c) .from('build_requests') .update({ status: effectiveStatus, @@ -218,6 +227,8 @@ export async function getBuildStatus( }) .eq('builder_job_id', job_id) .eq('app_id', buildRequest.app_id) + .eq('status', previousStatus) + .select('id') if (updateError) { cloudlogErr({ @@ -227,6 +238,25 @@ export async function getBuildStatus( error: updateError.message, }) } + else if (updatedRows && updatedRows.length > 0) { + await emitBuildTransitionEvent(c, { + previousStatus, + effectiveStatus, + timeoutApplied, + effectiveError, + effectiveBuildTimeSeconds, + build: { + app_id: buildRequest.app_id, + platform: buildRequest.platform, + build_mode: buildRequest.build_mode, + owner_org: buildRequest.owner_org, + }, + }) + } + // else: another writer already advanced the status (or it never matched + // previousStatus) — skip emission to avoid double-firing. recordBuildTime + // below stays unconditional: it's idempotent at the DB layer, and skipping + // it would let billing miss a build. const shouldRecordBuildTime = !!builderJob.job.started_at && (timeoutApplied || ((effectiveStatus === 'succeeded' || effectiveStatus === 'failed') && !!builderJob.job.completed_at)) diff --git a/supabase/functions/_backend/triggers/cron_reconcile_build_status.ts b/supabase/functions/_backend/triggers/cron_reconcile_build_status.ts index 8f12c28eb4..f308511195 100644 --- a/supabase/functions/_backend/triggers/cron_reconcile_build_status.ts +++ b/supabase/functions/_backend/triggers/cron_reconcile_build_status.ts @@ -12,6 +12,7 @@ import { shouldApplyBuildTimeout, TERMINAL_BUILD_STATUSES, } from '../utils/build_timeout.ts' +import { emitBuildTransitionEvent } from '../utils/build_tracking.ts' import { BRES, middlewareAPISecret } from '../utils/hono.ts' import { cloudlog, cloudlogErr } from '../utils/logging.ts' import { recordBuildTime, supabaseAdmin } from '../utils/supabase.ts' @@ -68,7 +69,7 @@ app.post('/', middlewareAPISecret, async (c) => { const { data: staleBuilds, error: queryError } = await supabase .from('build_requests') - .select('id, builder_job_id, app_id, owner_org, requested_by, platform, status, created_at') + .select('id, builder_job_id, app_id, owner_org, requested_by, platform, build_mode, status, created_at') .not('status', 'in', `(${[...TERMINAL_BUILD_STATUSES].join(',')})`) .lt('updated_at', new Date(Date.now() - STALE_THRESHOLD_MINUTES * 60 * 1000).toISOString()) .order('updated_at', { ascending: true }) @@ -209,7 +210,16 @@ app.post('/', middlewareAPISecret, async (c) => { if (timeoutApplied) timedOut++ - const { error: updateError } = await supabase + const previousStatus = build.status + + // Optimistic concurrency-control (CAS) guard: `.eq('status', previousStatus)` + // ensures only one writer wins when the cron races with a CLI/dashboard + // poller on the same row. The `.select('id')` lets us detect whether this + // writer actually advanced the row; if `updatedRows` is empty, another + // writer already moved the status and has emitted the transition — skip + // emission here. The cron's per-build loop is inside Promise.allSettled, + // so a lost race must NOT throw: we just skip the event and continue. + const { data: updatedRows, error: updateError } = await supabase .from('build_requests') .update({ status: effectiveStatus, @@ -218,10 +228,17 @@ app.post('/', middlewareAPISecret, async (c) => { updated_at: new Date().toISOString(), }) .eq('id', build.id) + .eq('status', previousStatus) + .select('id') if (updateError) throw new Error(updateError.message) + const transitionApplied = !!updatedRows && updatedRows.length > 0 + + // recordBuildTime stays unconditional on terminal status: it's idempotent + // at the DB layer, and skipping it on the CAS-lost branch would let + // billing miss a build (worse than the rare duplicate). if ( (isTerminalBuildStatus(effectiveStatus) || timeoutApplied) && builderJob.job.started_at @@ -244,6 +261,24 @@ app.post('/', middlewareAPISecret, async (c) => { ) } } + + if (transitionApplied) { + await emitBuildTransitionEvent(c, { + previousStatus, + effectiveStatus, + timeoutApplied, + effectiveError, + effectiveBuildTimeSeconds, + build: { + app_id: build.app_id, + platform: build.platform, + build_mode: build.build_mode, + owner_org: build.owner_org, + }, + }) + } + // else: another writer (start.ts/status.ts, or another cron tick) already + // advanced this row and emitted — skip to avoid double-firing. }), ) diff --git a/supabase/functions/_backend/utils/build_tracking.ts b/supabase/functions/_backend/utils/build_tracking.ts new file mode 100644 index 0000000000..928310e051 --- /dev/null +++ b/supabase/functions/_backend/utils/build_tracking.ts @@ -0,0 +1,154 @@ +import type { Context } from 'hono' +import { TERMINAL_BUILD_STATUSES } from './build_timeout.ts' +import { cloudlogErr, serializeError } from './logging.ts' +import { sendEventToTracking } from './tracking.ts' + +export type BuildTransition = 'started' | 'succeeded' | 'failed' | 'timed_out' +export type BuildFailureCategory = 'timeout' | 'builder_error' | 'validation_error' | 'unknown' + +// Substring hints — `'missing credential'` matches both singular and plural; `'validation'` is intentionally broad. +const VALIDATION_HINTS = ['invalid build_mode', 'missing credential', 'validation'] + +interface ClassifyInput { + previous: string + next: string + timeoutApplied: boolean +} + +export function classifyBuildTransition(input: ClassifyInput): BuildTransition | null { + if (TERMINAL_BUILD_STATUSES.has(input.previous)) + return null + + // Timeout overrides the no-change check: a stale snapshot with the same + // previous/next must still emit `timed_out` when the cron applied a timeout. + if (input.timeoutApplied) + return 'timed_out' + + if (input.previous === input.next) + return null + + if (input.next === 'running') + return 'started' + + if (input.next === 'succeeded') + return 'succeeded' + + if (input.next === 'failed') + return 'failed' + + return null +} + +interface FailureInput { + timeoutApplied: boolean + errorMessage: string | null | undefined +} + +export function mapBuildFailureCategory(input: FailureInput): BuildFailureCategory { + if (input.timeoutApplied) + return 'timeout' + + const message = (input.errorMessage ?? '').toLowerCase() + if (!message) + return 'unknown' + + for (const hint of VALIDATION_HINTS) { + if (message.includes(hint)) + return 'validation_error' + } + + return 'builder_error' +} + +interface BuildRowForTracking { + app_id: string + platform: string + build_mode: string + owner_org: string +} + +export interface EmitBuildTransitionInput { + previousStatus: string + effectiveStatus: string + timeoutApplied: boolean + effectiveError?: string | null + effectiveBuildTimeSeconds?: number | null + build: BuildRowForTracking +} + +const EVENT_NAME_BY_TRANSITION: Record = { + started: 'Build Started', + succeeded: 'Build Succeeded', + failed: 'Build Failed', + timed_out: 'Build Timed Out', +} + +const ICON_BY_TRANSITION: Record = { + started: '⏳', + succeeded: '✅', + failed: '❌', + timed_out: '⏰', +} + +/** + * Emit the appropriate Build * lifecycle event for a status transition, or no-op when + * `classifyBuildTransition` returns null (already-terminal previous status, or no change). + * + * Shared by: + * - the cron reconcile path (stale / abandoned builds), and + * - the public/build/start.ts + public/build/status.ts happy paths. + * + * The terminal-status idempotency guard in `classifyBuildTransition` means re-calls on + * already-terminal rows are safe no-ops. + */ +export async function emitBuildTransitionEvent(c: Context, input: EmitBuildTransitionInput): Promise { + const transition = classifyBuildTransition({ + previous: input.previousStatus, + next: input.effectiveStatus, + timeoutApplied: input.timeoutApplied, + }) + if (!transition) + return + + const tags: Record = { + app_id: input.build.app_id, + platform: input.build.platform, + build_mode: input.build.build_mode, + } + if ( + input.effectiveBuildTimeSeconds !== null + && input.effectiveBuildTimeSeconds !== undefined + && (transition === 'succeeded' || transition === 'failed' || transition === 'timed_out') + ) { + tags.duration_seconds = String(input.effectiveBuildTimeSeconds) + } + if (transition === 'failed' || transition === 'timed_out') { + tags.failure_category = mapBuildFailureCategory({ + timeoutApplied: input.timeoutApplied, + errorMessage: input.effectiveError ?? null, + }) + } + + // Telemetry MUST NOT break the build flow. sendEventToTracking already swallows + // each provider's failure individually, but defend against an unexpected throw + // at the orchestration layer (e.g. backgroundTask unavailable in tests). + try { + await sendEventToTracking(c, { + event: EVENT_NAME_BY_TRANSITION[transition], + channel: 'build-lifecycle', + icon: ICON_BY_TRANSITION[transition], + notify: false, + user_id: input.build.owner_org, + groups: { organization: input.build.owner_org }, + tags, + }) + } + catch (error) { + cloudlogErr({ + requestId: c.get('requestId'), + message: 'emitBuildTransitionEvent failed', + transition, + error: serializeError(error), + }) + } +} diff --git a/tests/ai-analysis-telemetry.unit.test.ts b/tests/ai-analysis-telemetry.unit.test.ts new file mode 100644 index 0000000000..d0b3863cc1 --- /dev/null +++ b/tests/ai-analysis-telemetry.unit.test.ts @@ -0,0 +1,159 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { trackAiAnalysisChoice, trackAiAnalysisResult } from '../cli/src/ai/telemetry.ts' + +const sendEventMock = vi.hoisted(() => vi.fn()) + +vi.mock('../cli/src/utils.ts', () => ({ + sendEvent: sendEventMock, +})) + +describe('trackAiAnalysisChoice', () => { + beforeEach(() => { + sendEventMock.mockReset() + sendEventMock.mockResolvedValue(undefined) + }) + + it.each([ + ['capgo_ai', 'menu'] as const, + ['local_ai', 'menu'] as const, + ['skip', 'menu'] as const, + ['auto_upload', 'ci_flag'] as const, + ])('emits the expected payload for choice=%s triggeredBy=%s', async (choice, triggeredBy) => { + await trackAiAnalysisChoice({ + apikey: 'cap_test_key', + orgId: 'org-uuid-1', + appId: 'com.example.app', + platform: 'ios', + jobId: 'job-abc', + choice, + triggeredBy, + }) + + expect(sendEventMock).toHaveBeenCalledTimes(1) + const [calledKey, payload] = sendEventMock.mock.calls[0] + expect(calledKey).toBe('cap_test_key') + expect(payload).toMatchObject({ + event: 'CLI AI Build Analysis Choice', + channel: 'build-lifecycle', + icon: '🤖', + notify: false, + user_id: 'org-uuid-1', + tags: { + app_id: 'com.example.app', + platform: 'ios', + job_id: 'job-abc', + choice, + triggered_by: triggeredBy, + }, + }) + }) + + it('swallows errors thrown by sendEvent', async () => { + sendEventMock.mockRejectedValueOnce(new Error('network down')) + await expect(trackAiAnalysisChoice({ + apikey: 'cap_test_key', + orgId: 'org-uuid-1', + appId: 'com.example.app', + platform: 'android', + jobId: 'job-abc', + choice: 'skip', + triggeredBy: 'menu', + })).resolves.toBeUndefined() + }) +}) + +describe('trackAiAnalysisResult', () => { + beforeEach(() => { + sendEventMock.mockReset() + sendEventMock.mockResolvedValue(undefined) + }) + + it.each([ + 'success', + 'already_analyzed', + 'too_big', + ] as const)('emits the expected payload for result=%s without error_status', async (result) => { + await trackAiAnalysisResult({ + apikey: 'cap_test_key', + orgId: 'org-uuid-1', + appId: 'com.example.app', + platform: 'ios', + jobId: 'job-abc', + result, + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload).toMatchObject({ + event: 'CLI AI Build Analysis Result', + channel: 'build-lifecycle', + icon: '🤖', + notify: false, + user_id: 'org-uuid-1', + tags: { + app_id: 'com.example.app', + platform: 'ios', + job_id: 'job-abc', + result, + }, + }) + expect(payload.tags.error_status).toBeUndefined() + }) + + it('emits error with error_status when provided', async () => { + await trackAiAnalysisResult({ + apikey: 'cap_test_key', + orgId: 'org-uuid-1', + appId: 'com.example.app', + platform: 'android', + jobId: 'job-abc', + result: 'error', + errorStatus: 503, + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload.tags.result).toBe('error') + expect(payload.tags.error_status).toBe('503') + }) + + it('omits error_status when result is not error, even if errorStatus is provided', async () => { + await trackAiAnalysisResult({ + apikey: 'cap_test_key', + orgId: 'org-uuid-1', + appId: 'com.example.app', + platform: 'ios', + jobId: 'job-abc', + result: 'success', + errorStatus: 200, + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload.tags.error_status).toBeUndefined() + }) + + it('omits error_status when result is error but errorStatus is undefined (no status, e.g. network error)', async () => { + await trackAiAnalysisResult({ + apikey: 'cap_test_key', + orgId: 'org-uuid-1', + appId: 'com.example.app', + platform: 'ios', + jobId: 'job-abc', + result: 'error', + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload.tags.result).toBe('error') + expect(payload.tags.error_status).toBeUndefined() + }) + + it('swallows errors thrown by sendEvent', async () => { + sendEventMock.mockRejectedValueOnce(new Error('network down')) + await expect(trackAiAnalysisResult({ + apikey: 'cap_test_key', + orgId: 'org-uuid-1', + appId: 'com.example.app', + platform: 'ios', + jobId: 'job-abc', + result: 'success', + })).resolves.toBeUndefined() + }) +}) diff --git a/tests/build-ai-analyze.test.ts b/tests/build-ai-analyze.test.ts index 143d04d75c..95b354dddc 100644 --- a/tests/build-ai-analyze.test.ts +++ b/tests/build-ai-analyze.test.ts @@ -1,10 +1,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { aiAnalyzeBuild } from '../supabase/functions/_backend/public/build/ai_analyze' -const { mockSupabaseApikey, mockCheckPermission, mockGetEnv } = vi.hoisted(() => ({ +const { mockSupabaseApikey, mockCheckPermission, mockGetEnv, mockSendEventToTracking } = vi.hoisted(() => ({ mockSupabaseApikey: vi.fn(), mockCheckPermission: vi.fn(), mockGetEnv: vi.fn(), + mockSendEventToTracking: vi.fn(), })) vi.mock('../supabase/functions/_backend/utils/supabase.ts', () => ({ @@ -16,10 +17,14 @@ vi.mock('../supabase/functions/_backend/utils/rbac.ts', () => ({ vi.mock('../supabase/functions/_backend/utils/utils.ts', () => ({ getEnv: mockGetEnv, })) +vi.mock('../supabase/functions/_backend/utils/tracking.ts', () => ({ + sendEventToTracking: mockSendEventToTracking, +})) const requestId = 'req-ai-analyze-test' const jobId = 'job-abc' const appId = 'com.test.ai.analyze' +const orgId = 'org-test-1' const builderUrl = 'https://builder.capgo.test' const builderApiKey = 'builder-api-key' @@ -33,7 +38,14 @@ function createContext() { } as any } -function mockBuildRequestRow(row: { app_id: string, status: string, ai_analyzed: boolean } | null) { +interface RowShape { + app_id: string + status: string + ai_analyzed: boolean + owner_org: string +} + +function mockBuildRequestRow(row: RowShape | null) { const eqAppId = { maybeSingle: vi.fn().mockResolvedValue({ data: row, error: null }) } const eqJob = { eq: vi.fn().mockReturnValue(eqAppId) } const select = { eq: vi.fn().mockReturnValue(eqJob) } @@ -55,6 +67,8 @@ beforeEach(() => { mockSupabaseApikey.mockReset() mockCheckPermission.mockReset() mockGetEnv.mockReset() + mockSendEventToTracking.mockReset() + mockSendEventToTracking.mockResolvedValue(undefined) mockGetEnv.mockImplementation((_: unknown, key: string) => { if (key === 'BUILDER_URL') return builderUrl @@ -65,42 +79,74 @@ beforeEach(() => { globalThis.fetch = vi.fn() }) +function trackingCallsByEvent(eventName: string) { + return mockSendEventToTracking.mock.calls.filter(([, payload]) => payload.event === eventName) +} + describe('aiAnalyzeBuild', () => { - it('throws unauthorized when checkPermission denies', async () => { + it('throws unauthorized when checkPermission denies, fires Result-only with no owner_org', async () => { mockCheckPermission.mockResolvedValue(false) await expect(aiAnalyzeBuild(createContext(), jobId, appId, apikey, 'logs')) .rejects .toThrow(/permission to analyze/i) + + expect(trackingCallsByEvent('AI Build Analysis Requested')).toHaveLength(0) + const results = trackingCallsByEvent('AI Build Analysis Result') + expect(results).toHaveLength(1) + const [, payload] = results[0] + expect(payload.tags.result).toBe('unauthorized') + expect(payload.user_id).toBeUndefined() + expect(payload.groups).toBeUndefined() }) - it('throws unauthorized when build_request row not found', async () => { + it('throws unauthorized when build_request row not found, fires Result-only with no owner_org', async () => { mockCheckPermission.mockResolvedValue(true) mockBuildRequestRow(null) await expect(aiAnalyzeBuild(createContext(), jobId, appId, apikey, 'logs')) .rejects .toThrow(/permission to analyze/i) + + expect(trackingCallsByEvent('AI Build Analysis Requested')).toHaveLength(0) + const results = trackingCallsByEvent('AI Build Analysis Result') + expect(results).toHaveLength(1) + const [, payload] = results[0] + expect(payload.tags.result).toBe('unauthorized') + expect(payload.user_id).toBeUndefined() + expect(payload.groups).toBeUndefined() }) - it('throws invalid_state when status is not failed', async () => { + it('throws invalid_state when status is not failed; fires Result(invalid_state) only (no Requested)', async () => { mockCheckPermission.mockResolvedValue(true) - mockBuildRequestRow({ app_id: appId, status: 'succeeded', ai_analyzed: false }) + mockBuildRequestRow({ app_id: appId, status: 'succeeded', ai_analyzed: false, owner_org: orgId }) await expect(aiAnalyzeBuild(createContext(), jobId, appId, apikey, 'logs')) .rejects .toThrow(/only available for failed builds/i) + + expect(trackingCallsByEvent('AI Build Analysis Requested')).toHaveLength(0) + + const results = trackingCallsByEvent('AI Build Analysis Result') + expect(results).toHaveLength(1) + expect(results[0][1].tags.result).toBe('invalid_state') + expect(results[0][1].user_id).toBe(orgId) + expect(results[0][1].groups).toEqual({ organization: orgId }) }) - it('throws already_analyzed with HTTP 409 status when ai_analyzed is true', async () => { + it('throws already_analyzed when ai_analyzed is true; fires Result(already_analyzed) only (no Requested)', async () => { mockCheckPermission.mockResolvedValue(true) - mockBuildRequestRow({ app_id: appId, status: 'failed', ai_analyzed: true }) - // The CLI branches on res.status === 409 — verify both the message and the status code + mockBuildRequestRow({ app_id: appId, status: 'failed', ai_analyzed: true, owner_org: orgId }) await expect(aiAnalyzeBuild(createContext(), jobId, appId, apikey, 'logs')) .rejects .toMatchObject({ status: 409, message: expect.stringMatching(/already requested for this job/i) }) + + expect(trackingCallsByEvent('AI Build Analysis Requested')).toHaveLength(0) + const results = trackingCallsByEvent('AI Build Analysis Result') + expect(results).toHaveLength(1) + expect(results[0][1].tags.result).toBe('already_analyzed') }) - it('does NOT flip the flag when builder proxy returns non-2xx', async () => { + it('does NOT flip the flag when builder proxy returns non-2xx; fires Requested + Result(builder_error) with duration_ms', async () => { mockCheckPermission.mockResolvedValue(true) - const { updateEqApp } = mockBuildRequestRow({ app_id: appId, status: 'failed', ai_analyzed: false }) + const { updateEqApp } = mockBuildRequestRow({ app_id: appId, status: 'failed', ai_analyzed: false, owner_org: orgId }) ;(globalThis.fetch as any).mockResolvedValue(new Response('upstream broken', { status: 503 })) await expect(aiAnalyzeBuild(createContext(), jobId, appId, apikey, 'small logs')) @@ -108,11 +154,16 @@ describe('aiAnalyzeBuild', () => { .toThrow(/AI analysis failed/i) expect(updateEqApp).not.toHaveBeenCalled() + expect(trackingCallsByEvent('AI Build Analysis Requested')).toHaveLength(1) + const results = trackingCallsByEvent('AI Build Analysis Result') + expect(results).toHaveLength(1) + expect(results[0][1].tags.result).toBe('builder_error') + expect(results[0][1].tags.duration_ms).toBeDefined() }) - it('flips the flag and returns analysis on builder 200', async () => { + it('flips the flag, returns analysis on builder 200, fires Requested + Result(success); does NOT leak analysis text in tags', async () => { mockCheckPermission.mockResolvedValue(true) - const { updateEqApp } = mockBuildRequestRow({ app_id: appId, status: 'failed', ai_analyzed: false }) + const { updateEqApp } = mockBuildRequestRow({ app_id: appId, status: 'failed', ai_analyzed: false, owner_org: orgId }) ;(globalThis.fetch as any).mockResolvedValue( new Response(JSON.stringify({ analysis: '### Likely cause\nfoo' }), { status: 200, headers: { 'content-type': 'application/json' } }), ) @@ -128,5 +179,34 @@ describe('aiAnalyzeBuild', () => { expect(fetchCall[1].headers['x-api-key']).toBe(builderApiKey) expect(fetchCall[1].method).toBe('POST') expect(JSON.parse(fetchCall[1].body)).toEqual({ logs: 'small logs' }) + + // Telemetry assertions + expect(trackingCallsByEvent('AI Build Analysis Requested')).toHaveLength(1) + const results = trackingCallsByEvent('AI Build Analysis Result') + expect(results).toHaveLength(1) + expect(results[0][1].tags.result).toBe('success') + expect(results[0][1].tags.duration_ms).toBeDefined() + + // Privacy boundary: the analysis text must not appear in any tag. + for (const call of mockSendEventToTracking.mock.calls) { + const tagsString = JSON.stringify(call[1].tags || {}) + expect(tagsString).not.toContain('Likely cause') + expect(tagsString).not.toContain('### ') + } + }) + + it('fires Result(config_error) when BUILDER_URL is missing', async () => { + mockCheckPermission.mockResolvedValue(true) + mockBuildRequestRow({ app_id: appId, status: 'failed', ai_analyzed: false, owner_org: orgId }) + mockGetEnv.mockImplementation(() => '') + + await expect(aiAnalyzeBuild(createContext(), jobId, appId, apikey, 'logs')) + .rejects + .toThrow(/Builder service not configured/i) + + const results = trackingCallsByEvent('AI Build Analysis Result') + expect(results).toHaveLength(1) + expect(results[0][1].tags.result).toBe('config_error') + expect(results[0][1].user_id).toBe(orgId) }) }) diff --git a/tests/build-lifecycle-emit.unit.test.ts b/tests/build-lifecycle-emit.unit.test.ts new file mode 100644 index 0000000000..d54850c22b --- /dev/null +++ b/tests/build-lifecycle-emit.unit.test.ts @@ -0,0 +1,177 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const sendEventToTrackingMock = vi.hoisted(() => vi.fn()) + +vi.mock('../supabase/functions/_backend/utils/tracking.ts', () => ({ + sendEventToTracking: sendEventToTrackingMock, +})) + +const { emitBuildTransitionEvent } = await import('../supabase/functions/_backend/utils/build_tracking.ts') + +const baseBuild = { + app_id: 'com.example.app', + platform: 'ios', + build_mode: 'release', + owner_org: 'org-uuid-1', +} + +function fakeContext() { + return {} as any +} + +describe('emitBuildTransitionEvent', () => { + beforeEach(() => { + sendEventToTrackingMock.mockReset() + sendEventToTrackingMock.mockResolvedValue(undefined) + }) + + it('emits Build Started with no duration_seconds and no failure_category', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'pending', + effectiveStatus: 'running', + timeoutApplied: false, + build: baseBuild, + }) + + expect(sendEventToTrackingMock).toHaveBeenCalledTimes(1) + const [, payload] = sendEventToTrackingMock.mock.calls[0] + expect(payload).toMatchObject({ + event: 'Build Started', + channel: 'build-lifecycle', + icon: '⏳', + notify: false, + user_id: 'org-uuid-1', + groups: { organization: 'org-uuid-1' }, + tags: { + app_id: 'com.example.app', + platform: 'ios', + build_mode: 'release', + }, + }) + expect(payload.tags.duration_seconds).toBeUndefined() + expect(payload.tags.failure_category).toBeUndefined() + }) + + it('emits Build Succeeded with duration_seconds when provided', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'running', + effectiveStatus: 'succeeded', + timeoutApplied: false, + effectiveBuildTimeSeconds: 123, + build: baseBuild, + }) + + const [, payload] = sendEventToTrackingMock.mock.calls[0] + expect(payload).toMatchObject({ + event: 'Build Succeeded', + icon: '✅', + tags: { + duration_seconds: '123', + }, + }) + expect(payload.tags.failure_category).toBeUndefined() + }) + + it('emits Build Failed with failure_category=builder_error for a generic error message', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'running', + effectiveStatus: 'failed', + timeoutApplied: false, + effectiveError: 'gradle compile failed', + effectiveBuildTimeSeconds: 42, + build: baseBuild, + }) + + const [, payload] = sendEventToTrackingMock.mock.calls[0] + expect(payload).toMatchObject({ + event: 'Build Failed', + icon: '❌', + tags: { + failure_category: 'builder_error', + duration_seconds: '42', + }, + }) + }) + + it('emits Build Failed with failure_category=validation_error for validation-style messages', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'running', + effectiveStatus: 'failed', + timeoutApplied: false, + effectiveError: 'missing credentials', + build: baseBuild, + }) + + const [, payload] = sendEventToTrackingMock.mock.calls[0] + expect(payload.tags.failure_category).toBe('validation_error') + }) + + it('emits Build Timed Out with failure_category=timeout and capped duration', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'running', + effectiveStatus: 'failed', + timeoutApplied: true, + effectiveError: 'Build timed out after N seconds', + effectiveBuildTimeSeconds: 1800, + build: baseBuild, + }) + + const [, payload] = sendEventToTrackingMock.mock.calls[0] + expect(payload).toMatchObject({ + event: 'Build Timed Out', + icon: '⏰', + tags: { + failure_category: 'timeout', + duration_seconds: '1800', + }, + }) + }) + + it('does NOT call sendEventToTracking when previous status is already terminal', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'succeeded', + effectiveStatus: 'succeeded', + timeoutApplied: false, + build: baseBuild, + }) + + expect(sendEventToTrackingMock).not.toHaveBeenCalled() + }) + + it('does NOT call sendEventToTracking when previous === next and no timeout applied', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'running', + effectiveStatus: 'running', + timeoutApplied: false, + build: baseBuild, + }) + + expect(sendEventToTrackingMock).not.toHaveBeenCalled() + }) + + it('does NOT include duration_seconds for the started transition even when effectiveBuildTimeSeconds is set', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'pending', + effectiveStatus: 'running', + timeoutApplied: false, + effectiveBuildTimeSeconds: 7, + build: baseBuild, + }) + + const [, payload] = sendEventToTrackingMock.mock.calls[0] + expect(payload.tags.duration_seconds).toBeUndefined() + }) + + it('does NOT include duration_seconds when value is null', async () => { + await emitBuildTransitionEvent(fakeContext(), { + previousStatus: 'running', + effectiveStatus: 'succeeded', + timeoutApplied: false, + effectiveBuildTimeSeconds: null, + build: baseBuild, + }) + + const [, payload] = sendEventToTrackingMock.mock.calls[0] + expect(payload.tags.duration_seconds).toBeUndefined() + }) +}) diff --git a/tests/build-start-log-token.test.ts b/tests/build-start-log-token.test.ts index f823f359d5..15916c3468 100644 --- a/tests/build-start-log-token.test.ts +++ b/tests/build-start-log-token.test.ts @@ -2,12 +2,13 @@ import { jwtVerify } from 'jose' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { startBuild } from '../supabase/functions/_backend/public/build/start.ts' -const { mockSupabaseAdmin, mockSupabaseApikey, mockCheckPermission, mockGetEnv, mockReserveNativeBuildSlot } = vi.hoisted(() => ({ +const { mockSupabaseAdmin, mockSupabaseApikey, mockCheckPermission, mockGetEnv, mockReserveNativeBuildSlot, mockSendEventToTracking } = vi.hoisted(() => ({ mockSupabaseAdmin: vi.fn(), mockSupabaseApikey: vi.fn(), mockCheckPermission: vi.fn(), mockGetEnv: vi.fn(), mockReserveNativeBuildSlot: vi.fn(), + mockSendEventToTracking: vi.fn(), })) vi.mock('../supabase/functions/_backend/utils/supabase.ts', () => ({ @@ -27,6 +28,10 @@ vi.mock('../supabase/functions/_backend/utils/utils.ts', () => ({ getEnv: mockGetEnv, })) +vi.mock('../supabase/functions/_backend/utils/tracking.ts', () => ({ + sendEventToTracking: mockSendEventToTracking, +})) + describe('build start direct log token', () => { const requestId = 'req-build-start-log-token' const jobId = 'job-log-token-123' @@ -37,12 +42,36 @@ describe('build start direct log token', () => { const builderUrl = 'https://builder.capgo.test' const builderApiKey = 'builder-api-key' + // Mock the CAS update chain: .update(...).eq(...).eq(...).eq(...).select('id') + // Returns { data, error } from .select(); `data` shape decides whether + // emitBuildTransitionEvent fires. `mockReturnThis()` on .eq() lets the chain + // accept any number of guards (builder_job_id + app_id + status, currently). + function configureUpdateMock(selectResult: { data: Array<{ id: string }> | null, error: { message: string } | null }) { + const updateBuilder = { + eq: vi.fn().mockReturnThis(), + select: vi.fn().mockResolvedValue(selectResult), + } + + mockSupabaseAdmin.mockReturnValue({ + from: vi.fn().mockImplementation((table: string) => { + expect(table).toBe('build_requests') + return { + update: vi.fn().mockReturnValue(updateBuilder), + } + }), + }) + + return updateBuilder + } + beforeEach(() => { mockSupabaseAdmin.mockReset() mockSupabaseApikey.mockReset() mockCheckPermission.mockReset() mockGetEnv.mockReset() mockReserveNativeBuildSlot.mockReset() + mockSendEventToTracking.mockReset() + mockSendEventToTracking.mockResolvedValue(undefined) const selectBuilder = { eq: vi.fn().mockReturnThis(), @@ -51,17 +80,14 @@ describe('build start direct log token', () => { id: '3eb4f870-720d-46b9-843f-2e6d57d54000', app_id: appId, owner_org: '3eb4f870-720d-46b9-843f-2e6d57d54001', + status: 'pending', + platform: 'ios', + build_mode: 'release', }, error: null, }), } - const updateBuilder = { - eq: vi.fn() - .mockImplementationOnce(() => updateBuilder) - .mockResolvedValueOnce({ error: null }), - } - mockSupabaseApikey.mockReturnValue({ from: vi.fn().mockImplementation((table: string) => { expect(table).toBe('build_requests') @@ -71,14 +97,8 @@ describe('build start direct log token', () => { }), }) - mockSupabaseAdmin.mockReturnValue({ - from: vi.fn().mockImplementation((table: string) => { - expect(table).toBe('build_requests') - return { - update: vi.fn().mockReturnValue(updateBuilder), - } - }), - }) + // Default: CAS guard succeeds, one row returned, lifecycle event should fire. + configureUpdateMock({ data: [{ id: 'row-1' }], error: null }) mockCheckPermission.mockResolvedValue(true) mockReserveNativeBuildSlot.mockResolvedValue({ @@ -184,6 +204,146 @@ describe('build start direct log token', () => { appId, jobId, }) + + expect(mockSendEventToTracking).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ event: 'Build Started' }), + ) + } + finally { + fetchMock.mockRestore() + } + }) + + it('emits Build Failed when the builder rejects the start request', async () => { + // Builder rejection (start.ts:213) calls markBuildAsFailed, which now: + // 1. fetches the row to capture previousStatus + platform/build_mode/owner_org + // 2. updates status to 'failed' with a CAS guard + // 3. emits 'Build Failed' lifecycle event + // Without step 3 (the bug this guards against), the builder-rejection path + // would silently update status='failed' but never appear in the lifecycle funnel. + + // Override the admin mock to handle BOTH operations markBuildAsFailed performs: + // - `.from('build_requests').select(...)` to read the row + // - `.from('build_requests').update(...)` for the CAS write + const updateBuilder = { + eq: vi.fn().mockReturnThis(), + select: vi.fn().mockResolvedValue({ data: [{ id: 'row-1' }], error: null }), + } + const adminSelectChain = { + eq: vi.fn().mockReturnThis(), + maybeSingle: vi.fn().mockResolvedValue({ + data: { + status: 'pending', + platform: 'ios', + build_mode: 'release', + owner_org: '3eb4f870-720d-46b9-843f-2e6d57d54001', + }, + error: null, + }), + } + mockSupabaseAdmin.mockReturnValue({ + from: vi.fn().mockImplementation((table: string) => { + expect(table).toBe('build_requests') + return { + update: vi.fn().mockReturnValue(updateBuilder), + select: vi.fn().mockReturnValue(adminSelectChain), + } + }), + }) + + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('builder is offline', { + status: 500, + })) + + const context = { + get: vi.fn().mockImplementation((key: string) => { + if (key === 'requestId') + return requestId + return undefined + }), + json: (data: unknown, status = 200) => new Response(JSON.stringify(data), { + status, + headers: { + 'Content-Type': 'application/json', + }, + }), + } + + try { + await expect( + startBuild(context as any, jobId, appId, { key: 'cli-api-key', user_id: userId } as any), + ).rejects.toThrow() + + // Lifecycle funnel must include the terminal Build Failed transition. + expect(mockSendEventToTracking).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + event: 'Build Failed', + tags: expect.objectContaining({ + app_id: appId, + platform: 'ios', + build_mode: 'release', + failure_category: expect.any(String), + }), + }), + ) + } + finally { + fetchMock.mockRestore() + } + }) + + it('skips Build Started emission when CAS guard finds no matching row (lost race)', async () => { + // Override the default update mock: zero rows returned from .select('id') + // simulates another writer having already advanced the row's status before + // this request's UPDATE landed. The CAS guard `.eq('status', previousStatus)` + // matched no rows, so emitBuildTransitionEvent must NOT be called — the + // winning writer is responsible for emitting. + configureUpdateMock({ data: [], error: null }) + + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ + status: 'running', + }), { + status: 200, + headers: { + 'Content-Type': 'application/json', + }, + })) + + const context = { + get: vi.fn().mockImplementation((key: string) => { + if (key === 'requestId') + return requestId + return undefined + }), + json: (data: unknown, status = 200) => new Response(JSON.stringify(data), { + status, + headers: { + 'Content-Type': 'application/json', + }, + }), + } + + try { + const response = await startBuild( + context as any, + jobId, + appId, + { + key: 'cli-api-key', + user_id: userId, + } as any, + ) + + // Request still succeeds end-to-end — the CAS loss is silent. + expect(response.status).toBe(200) + const body = await response.json() as { status: string, job_id: string } + expect(body.status).toBe('running') + expect(body.job_id).toBe(jobId) + + // Lifecycle event must NOT fire on the CAS-lost branch. + expect(mockSendEventToTracking).not.toHaveBeenCalled() } finally { fetchMock.mockRestore() diff --git a/tests/build-tracking-helpers.unit.test.ts b/tests/build-tracking-helpers.unit.test.ts new file mode 100644 index 0000000000..e4c6ad1f7e --- /dev/null +++ b/tests/build-tracking-helpers.unit.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import { classifyBuildTransition, mapBuildFailureCategory } from '../supabase/functions/_backend/utils/build_tracking.ts' + +describe('classifyBuildTransition', () => { + it.concurrent('returns "started" when pending becomes running', () => { + expect(classifyBuildTransition({ previous: 'pending', next: 'running', timeoutApplied: false })).toBe('started') + }) + + it.concurrent('returns "started" when queued becomes running', () => { + expect(classifyBuildTransition({ previous: 'queued', next: 'running', timeoutApplied: false })).toBe('started') + }) + + it.concurrent('returns "succeeded" when any non-terminal becomes success', () => { + expect(classifyBuildTransition({ previous: 'running', next: 'succeeded', timeoutApplied: false })).toBe('succeeded') + expect(classifyBuildTransition({ previous: 'pending', next: 'succeeded', timeoutApplied: false })).toBe('succeeded') + }) + + it.concurrent('returns "failed" when any non-terminal becomes failed', () => { + expect(classifyBuildTransition({ previous: 'running', next: 'failed', timeoutApplied: false })).toBe('failed') + }) + + it.concurrent('returns "timed_out" when timeoutApplied is true', () => { + expect(classifyBuildTransition({ previous: 'running', next: 'failed', timeoutApplied: true })).toBe('timed_out') + expect(classifyBuildTransition({ previous: 'running', next: 'succeeded', timeoutApplied: true })).toBe('timed_out') + }) + + it.concurrent('returns null when previous status is already terminal (idempotency)', () => { + expect(classifyBuildTransition({ previous: 'succeeded', next: 'succeeded', timeoutApplied: false })).toBeNull() + expect(classifyBuildTransition({ previous: 'failed', next: 'failed', timeoutApplied: false })).toBeNull() + expect(classifyBuildTransition({ previous: 'cancelled', next: 'cancelled', timeoutApplied: false })).toBeNull() + expect(classifyBuildTransition({ previous: 'expired', next: 'expired', timeoutApplied: false })).toBeNull() + expect(classifyBuildTransition({ previous: 'released', next: 'released', timeoutApplied: false })).toBeNull() + }) + + it.concurrent('returns null when previous is terminal even if timeoutApplied is true', () => { + expect(classifyBuildTransition({ previous: 'failed', next: 'failed', timeoutApplied: true })).toBeNull() + }) + + it.concurrent('returns null when no state change happened (no transition)', () => { + expect(classifyBuildTransition({ previous: 'pending', next: 'pending', timeoutApplied: false })).toBeNull() + expect(classifyBuildTransition({ previous: 'running', next: 'running', timeoutApplied: false })).toBeNull() + }) + + it.concurrent('returns "timed_out" even when previous === next (timeout overrides no-change)', () => { + expect(classifyBuildTransition({ previous: 'running', next: 'running', timeoutApplied: true })).toBe('timed_out') + }) +}) + +describe('mapBuildFailureCategory', () => { + it.concurrent('returns timeout when the timeout flag is set', () => { + expect(mapBuildFailureCategory({ timeoutApplied: true, errorMessage: null })).toBe('timeout') + }) + + it.concurrent('returns validation_error for validation-style messages', () => { + expect(mapBuildFailureCategory({ timeoutApplied: false, errorMessage: 'Invalid build_mode value' })).toBe('validation_error') + expect(mapBuildFailureCategory({ timeoutApplied: false, errorMessage: 'missing credentials' })).toBe('validation_error') + }) + + it.concurrent('returns builder_error when there is any other non-empty error', () => { + expect(mapBuildFailureCategory({ timeoutApplied: false, errorMessage: 'gradle compile failed' })).toBe('builder_error') + }) + + it.concurrent('returns unknown when timeoutApplied is false and error is empty', () => { + expect(mapBuildFailureCategory({ timeoutApplied: false, errorMessage: null })).toBe('unknown') + expect(mapBuildFailureCategory({ timeoutApplied: false, errorMessage: '' })).toBe('unknown') + }) +}) diff --git a/tests/builder-onboarding-telemetry.unit.test.ts b/tests/builder-onboarding-telemetry.unit.test.ts new file mode 100644 index 0000000000..60a5dc9626 --- /dev/null +++ b/tests/builder-onboarding-telemetry.unit.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { trackBuilderOnboardingStep } from '../cli/src/build/onboarding/telemetry.ts' + +const sendEventMock = vi.hoisted(() => vi.fn()) + +vi.mock('../cli/src/utils.ts', () => ({ + sendEvent: sendEventMock, +})) + +describe('trackBuilderOnboardingStep', () => { + beforeEach(() => { + sendEventMock.mockReset() + sendEventMock.mockResolvedValue(undefined) + }) + + it('builds the expected payload and calls sendEvent once', async () => { + await trackBuilderOnboardingStep({ + apikey: 'cap_test_key', + step: 'api-key-instructions', + platform: 'ios', + appId: 'com.example.app', + orgId: 'org-uuid-1', + durationMs: 1234, + }) + + expect(sendEventMock).toHaveBeenCalledTimes(1) + const [calledKey, payload] = sendEventMock.mock.calls[0] + expect(calledKey).toBe('cap_test_key') + expect(payload).toMatchObject({ + event: 'Builder Onboarding Step', + channel: 'builder-onboarding', + icon: '🧭', + notify: false, + user_id: 'org-uuid-1', + tags: { + step: 'api-key-instructions', + platform: 'ios', + app_id: 'com.example.app', + duration_ms: '1234', + }, + }) + expect(payload.tags.error_category).toBeUndefined() + }) + + it('includes error_category only when an error is provided', async () => { + await trackBuilderOnboardingStep({ + apikey: 'cap_test_key', + step: 'error', + platform: 'ios', + appId: 'com.example.app', + orgId: 'org-uuid-1', + error: Object.assign(new Error('Unauthorized'), { status: 401 }), + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload.tags.error_category).toBe('apple_api_unauthorized') + }) + + it('uses the Android mapper when platform is android', async () => { + await trackBuilderOnboardingStep({ + apikey: 'cap_test_key', + step: 'error', + platform: 'android', + appId: 'com.example.app', + orgId: 'org-uuid-1', + error: Object.assign(new Error('Bad keystore'), { phase: 'keystore' }), + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload.tags.error_category).toBe('keystore_invalid') + }) + + it('swallows errors thrown by sendEvent', async () => { + sendEventMock.mockRejectedValueOnce(new Error('network down')) + await expect(trackBuilderOnboardingStep({ + apikey: 'cap_test_key', + step: 'welcome', + platform: 'ios', + appId: 'com.example.app', + orgId: 'org-uuid-1', + })).resolves.toBeUndefined() + }) + + it('does not include duration_ms when undefined', async () => { + await trackBuilderOnboardingStep({ + apikey: 'cap_test_key', + step: 'welcome', + platform: 'ios', + appId: 'com.example.app', + orgId: 'org-uuid-1', + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload.tags.duration_ms).toBeUndefined() + }) + + it('uses pre-computed errorCategory when provided (skipping the mapper)', async () => { + await trackBuilderOnboardingStep({ + apikey: 'cap_test_key', + step: 'error', + platform: 'ios', + appId: 'com.example.app', + orgId: 'org-uuid-1', + errorCategory: 'profile_creation_failed', + // error intentionally omitted + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload.tags.error_category).toBe('profile_creation_failed') + }) +}) diff --git a/tests/builder-upload-telemetry.unit.test.ts b/tests/builder-upload-telemetry.unit.test.ts new file mode 100644 index 0000000000..48a7707b0c --- /dev/null +++ b/tests/builder-upload-telemetry.unit.test.ts @@ -0,0 +1,137 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mapBuilderUploadError, trackBuilderUpload } from '../cli/src/build/telemetry.ts' + +const sendEventMock = vi.hoisted(() => vi.fn()) + +vi.mock('../cli/src/utils.ts', () => ({ + sendEvent: sendEventMock, +})) + +describe('mapBuilderUploadError', () => { + it.concurrent('maps HTTP 401 to unauthorized', () => { + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 401 } })).toBe('unauthorized') + }) + it.concurrent('maps HTTP 403 to unauthorized', () => { + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 403 } })).toBe('unauthorized') + }) + it.concurrent('maps HTTP 413 to payload_too_large', () => { + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 413 } })).toBe('payload_too_large') + }) + it.concurrent('maps HTTP 500-599 to storage_failure', () => { + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 500 } })).toBe('storage_failure') + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 502 } })).toBe('storage_failure') + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 599 } })).toBe('storage_failure') + }) + it.concurrent('maps no-response (connection-level) errors to network_error', () => { + expect(mapBuilderUploadError(new Error('ECONNRESET'))).toBe('network_error') + expect(mapBuilderUploadError({ originalResponse: undefined })).toBe('network_error') + expect(mapBuilderUploadError(null)).toBe('network_error') + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 0 } })).toBe('network_error') + }) + it.concurrent('maps other HTTP statuses to unknown', () => { + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 418 } })).toBe('unknown') + expect(mapBuilderUploadError({ originalResponse: { getStatus: () => 404 } })).toBe('unknown') + }) +}) + +describe('trackBuilderUpload', () => { + beforeEach(() => { + sendEventMock.mockReset() + sendEventMock.mockResolvedValue(undefined) + }) + + it('emits Builder Upload Started with size but no duration or failure_category', async () => { + await trackBuilderUpload({ + apikey: 'cap_test_key', + appId: 'com.example.app', + orgId: 'org-uuid-1', + platform: 'ios', + buildMode: 'release', + jobId: 'job-abc', + sizeBytes: 12_345_678, + phase: 'started', + }) + + expect(sendEventMock).toHaveBeenCalledTimes(1) + const [, payload] = sendEventMock.mock.calls[0] + expect(payload).toMatchObject({ + event: 'Builder Upload Started', + channel: 'build-lifecycle', + icon: '⬆️', + notify: false, + user_id: 'org-uuid-1', + tags: { + app_id: 'com.example.app', + platform: 'ios', + build_mode: 'release', + job_id: 'job-abc', + upload_size_bytes: '12345678', + }, + }) + expect(payload.tags.upload_duration_seconds).toBeUndefined() + expect(payload.tags.failure_category).toBeUndefined() + }) + + it('emits Builder Upload Succeeded with duration and size', async () => { + await trackBuilderUpload({ + apikey: 'cap_test_key', + appId: 'com.example.app', + orgId: 'org-uuid-1', + platform: 'android', + buildMode: 'release', + jobId: 'job-abc', + sizeBytes: 12_345_678, + phase: 'succeeded', + durationSeconds: 42.7, + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload).toMatchObject({ + event: 'Builder Upload Succeeded', + icon: '📦', + tags: { + platform: 'android', + upload_duration_seconds: '43', + }, + }) + }) + + it('emits Builder Upload Failed with failure_category from a 413', async () => { + await trackBuilderUpload({ + apikey: 'cap_test_key', + appId: 'com.example.app', + orgId: 'org-uuid-1', + platform: 'ios', + buildMode: 'release', + jobId: 'job-abc', + sizeBytes: 999_999, + phase: 'failed', + durationSeconds: 5, + error: { originalResponse: { getStatus: () => 413 } }, + }) + + const [, payload] = sendEventMock.mock.calls[0] + expect(payload).toMatchObject({ + event: 'Builder Upload Failed', + icon: '🚫', + tags: { + failure_category: 'payload_too_large', + upload_duration_seconds: '5', + }, + }) + }) + + it('swallows errors thrown by sendEvent', async () => { + sendEventMock.mockRejectedValueOnce(new Error('network down')) + await expect(trackBuilderUpload({ + apikey: 'cap_test_key', + appId: 'com.example.app', + orgId: 'org-uuid-1', + platform: 'ios', + buildMode: 'release', + jobId: 'job-abc', + sizeBytes: 1, + phase: 'started', + })).resolves.toBeUndefined() + }) +}) diff --git a/tests/onboarding-error-categories.unit.test.ts b/tests/onboarding-error-categories.unit.test.ts new file mode 100644 index 0000000000..487a7201df --- /dev/null +++ b/tests/onboarding-error-categories.unit.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest' +import { MissingScopesError } from '../cli/src/build/onboarding/android/oauth-google.ts' +import { CertificateLimitError } from '../cli/src/build/onboarding/apple-api.ts' +import { mapAndroidOnboardingError, mapIosOnboardingError } from '../cli/src/build/onboarding/error-categories.ts' + +describe('mapIosOnboardingError', () => { + it.concurrent('maps 401 from App Store Connect to apple_api_unauthorized', () => { + const err = Object.assign(new Error('Unauthorized'), { status: 401 }) + expect(mapIosOnboardingError(err)).toBe('apple_api_unauthorized') + }) + + it.concurrent('maps 429 to apple_api_rate_limited', () => { + const err = Object.assign(new Error('Too many'), { status: 429 }) + expect(mapIosOnboardingError(err)).toBe('apple_api_rate_limited') + }) + + it.concurrent('maps CertificateLimitError instances to cert_limit_reached', () => { + expect(mapIosOnboardingError(new CertificateLimitError([]))).toBe('cert_limit_reached') + }) + + it.concurrent('maps profile creation failures to profile_creation_failed', () => { + const err = Object.assign(new Error('Profile create failed'), { phase: 'profile' as const }) + expect(mapIosOnboardingError(err)).toBe('profile_creation_failed') + }) + + it.concurrent('maps P8 read errors to p8_invalid', () => { + const err = Object.assign(new Error('Cannot parse P8'), { phase: 'p8' as const }) + expect(mapIosOnboardingError(err)).toBe('p8_invalid') + }) + + it.concurrent('returns unknown for anything else', () => { + expect(mapIosOnboardingError(new Error('something else'))).toBe('unknown') + expect(mapIosOnboardingError('a string')).toBe('unknown') + expect(mapIosOnboardingError(undefined)).toBe('unknown') + }) + + it.concurrent('maps import-scanning failures to keychain_no_identities', () => { + expect(mapIosOnboardingError(new Error('no identities'), 'import-scanning')).toBe('keychain_no_identities') + }) + + it.concurrent('maps import-compiling-helper failures to keychain_helper_compile_failed', () => { + expect(mapIosOnboardingError(new Error('compile failed'), 'import-compiling-helper')).toBe('keychain_helper_compile_failed') + }) + + it.concurrent('maps import-exporting failures to keychain_export_failed', () => { + expect(mapIosOnboardingError(new Error('wrong password'), 'import-exporting')).toBe('keychain_export_failed') + }) + + it.concurrent('maps import-fetching-profile failures to profile_read_failed', () => { + expect(mapIosOnboardingError(new Error('fs error'), 'import-fetching-profile')).toBe('profile_read_failed') + }) + + it.concurrent('maps import-pick-profile and import-no-match-recovery to profile_no_match', () => { + expect(mapIosOnboardingError(new Error('no match'), 'import-pick-profile')).toBe('profile_no_match') + expect(mapIosOnboardingError(new Error('no match'), 'import-no-match-recovery')).toBe('profile_no_match') + }) + + it.concurrent('structural discriminators take precedence over failedStep', () => { + // Even if failedStep is an import step, a 401 still maps to apple_api_unauthorized + // (e.g. the helper precompile or fetch could theoretically throw an ASC error). + const err = Object.assign(new Error('Unauthorized'), { status: 401 }) + expect(mapIosOnboardingError(err, 'import-scanning')).toBe('apple_api_unauthorized') + }) + + it.concurrent('returns unknown for non-import failedStep with no structural discriminator', () => { + expect(mapIosOnboardingError(new Error('???'), 'welcome')).toBe('unknown') + expect(mapIosOnboardingError(new Error('???'), 'creating-certificate')).toBe('unknown') + }) + + it.concurrent('returns unknown when no failedStep and no structural discriminator', () => { + expect(mapIosOnboardingError(new Error('something else'), undefined)).toBe('unknown') + }) +}) + +describe('mapAndroidOnboardingError', () => { + it.concurrent('maps MissingScopesError to google_oauth_failed', () => { + expect(mapAndroidOnboardingError(new MissingScopesError(['scope1'], ''))).toBe('google_oauth_failed') + }) + + it.concurrent('maps keystore parse failures to keystore_invalid', () => { + const err = Object.assign(new Error('Bad keystore'), { phase: 'keystore' as const }) + expect(mapAndroidOnboardingError(err)).toBe('keystore_invalid') + }) + + it.concurrent('maps oauth token failures to google_oauth_failed', () => { + const err = Object.assign(new Error('Token refresh failed'), { phase: 'oauth' as const }) + expect(mapAndroidOnboardingError(err)).toBe('google_oauth_failed') + }) + + it.concurrent('maps play account id failures to play_account_id_invalid', () => { + const err = Object.assign(new Error('Bad ID'), { phase: 'play_account_id' as const }) + expect(mapAndroidOnboardingError(err)).toBe('play_account_id_invalid') + }) + + it.concurrent('returns unknown for everything else', () => { + expect(mapAndroidOnboardingError(new Error('???'))).toBe('unknown') + expect(mapAndroidOnboardingError(null)).toBe('unknown') + }) +}) diff --git a/tsconfig.json b/tsconfig.json index 968d009518..f9290fc45b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -51,6 +51,10 @@ "benches", "vite.config.mts", "temp_cli_test", - "tests/device_comparison.test.ts" + "tests/device_comparison.test.ts", + "tests/ai-analysis-telemetry.unit.test.ts", + "tests/builder-onboarding-telemetry.unit.test.ts", + "tests/builder-upload-telemetry.unit.test.ts", + "tests/onboarding-error-categories.unit.test.ts" ] }