diff --git a/cli/package.json b/cli/package.json index f9990ec36f..9d82ea9bf8 100644 --- a/cli/package.json +++ b/cli/package.json @@ -77,6 +77,8 @@ "test:build-needed": "bun test/test-build-needed.mjs", "test:ci-prompts": "bun test/test-ci-prompts.mjs", "test:ci-secrets": "bun test/test-ci-secrets.mjs", + "test:android-onboarding-progress": "bun test/test-android-onboarding-progress.mjs", + "test:onboarding-telemetry": "bun test/test-onboarding-telemetry.mjs", "test:posthog-exception": "bun test/test-posthog-exception.mjs", "test:onboarding-recovery": "bun test/test-onboarding-recovery.mjs", "test:onboarding-progress": "bun test/test-onboarding-progress.mjs", @@ -94,7 +96,7 @@ "test:macos-signing": "bun test/test-macos-signing.mjs", "test:apple-api-import-helpers": "bun test/test-apple-api-import-helpers.mjs", "test:manifest-path-encoding": "bun test/test-manifest-path-encoding.mjs", - "test": "bun run build && bun run test:version-detection:setup && bun run test:bundle && bun run test:functional && bun run test:semver && bun run test:version-edge-cases && bun run test:regex && bun run test:upload && bun run test:credentials && bun run test:credentials-validation && bun run test:android-service-account-validation && bun run test:build-zip-filter && bun run test:checksum && bun run test:build-needed && bun run test:ci-prompts && bun run test:ci-secrets && bun run test:posthog-exception && bun run test:build-platform-selection && bun run test:onboarding-recovery && bun run test:onboarding-progress && bun run test:onboarding-run-targets && bun run test:run-device-command && bun run test:init-app-conflict && bun run test:init-guardrails && bun run test:prompt-preferences && bun run test:esm-sdk && bun run test:mcp && bun run test:version-detection && bun run test:platform-paths && bun run test:payload-split && bun run test:manifest-path-encoding && bun run test:macos-signing && bun run test:apple-api-import-helpers && bun run test:ai-log-capture && bun run test:ai-analyze-flow && bun run test:ai-render-markdown", + "test": "bun run build && bun run test:version-detection:setup && bun run test:bundle && bun run test:functional && bun run test:semver && bun run test:version-edge-cases && bun run test:regex && bun run test:upload && bun run test:credentials && bun run test:credentials-validation && bun run test:android-service-account-validation && bun run test:build-zip-filter && bun run test:checksum && bun run test:build-needed && bun run test:ci-prompts && bun run test:ci-secrets && bun run test:android-onboarding-progress && bun run test:onboarding-telemetry && bun run test:posthog-exception && bun run test:build-platform-selection && bun run test:onboarding-recovery && bun run test:onboarding-progress && bun run test:onboarding-run-targets && bun run test:run-device-command && bun run test:init-app-conflict && bun run test:init-guardrails && bun run test:prompt-preferences && bun run test:esm-sdk && bun run test:mcp && bun run test:version-detection && bun run test:platform-paths && bun run test:payload-split && bun run test:manifest-path-encoding && bun run test:macos-signing && bun run test:apple-api-import-helpers && bun run test:ai-log-capture && bun run test:ai-analyze-flow && bun run test:ai-render-markdown", "test:build-platform-selection": "bun test/test-build-platform-selection.mjs", "test:ai-log-capture": "bun test/test-ai-log-capture.mjs", "test:ai-analyze-flow": "bun test/test-ai-analyze-flow.mjs", diff --git a/cli/src/build/onboarding/android/progress.ts b/cli/src/build/onboarding/android/progress.ts index 287f73138e..8ce231957d 100644 --- a/cli/src/build/onboarding/android/progress.ts +++ b/cli/src/build/onboarding/android/progress.ts @@ -115,6 +115,16 @@ function keystoreResumeStep(progress: AndroidOnboardingProgress): AndroidOnboard return 'keystore-method-select' } +export function hasAnyOAuthProgress(progress: AndroidOnboardingProgress): boolean { + return !!( + progress.completedSteps.googleSignInComplete + || progress.completedSteps.playAccountChosen + || progress.completedSteps.gcpProjectChosen + || progress.completedSteps.androidPackageChosen + || progress._oauthRefreshToken + ) +} + /** * Determine the first incomplete step for the Android flow. * @@ -167,9 +177,17 @@ export function getAndroidResumeStep(progress: AndroidOnboardingProgress | null) // Backward compatibility: legacy progress files (created before the // service-account fork existed) never set `serviceAccountMethod`. Per the - // design contract those resume into the OAuth path they were already on — - // we must NOT route them to the new fork screen and force them to re-pick - // mid-flow. Only routes through the fork explicitly set the method. + // design contract those resume into the OAuth path they were already on. + // Fresh progress files that reached the fork carry `serviceAccountForkSeen`, + // so quitting before choosing can restore the method-select screen without + // changing legacy behavior. + if ( + progress.serviceAccountForkSeen + && progress.serviceAccountMethod === undefined + && !hasAnyOAuthProgress(progress) + ) { + return 'service-account-method-select' + } // Phase 2b — Google sign-in: marker + refresh token. We need the refresh // token to mint access tokens for the rest of the flow on subsequent diff --git a/cli/src/build/onboarding/android/types.ts b/cli/src/build/onboarding/android/types.ts index d4cd58a973..ecccc8341d 100644 --- a/cli/src/build/onboarding/android/types.ts +++ b/cli/src/build/onboarding/android/types.ts @@ -132,9 +132,15 @@ export interface AndroidOnboardingProgress { keystoreKeyPassword?: string keystoreCommonName?: string - // Service account fork — set at `service-account-method-select`. Absent on - // legacy progress files (pre-2026-05) → resume defaults to `generate` so - // existing in-flight onboardings continue on the OAuth path they started on. + // Set when a fresh run completes keystore setup and becomes eligible to + // show `service-account-method-select`. This lets resume return to the fork + // if the user quits before choosing while still letting legacy progress + // files (without the marker) default to OAuth. + serviceAccountForkSeen?: true + // Service account fork — set when the user chooses existing JSON or Google + // OAuth provisioning. Absent on legacy progress files (pre-2026-05) → resume + // defaults to `generate` so existing in-flight onboardings continue on the + // OAuth path they started on. serviceAccountMethod?: ServiceAccountMethod // Import path — path the user picked at `sa-json-existing-path` / // `sa-json-existing-picker`. The file is read fresh at validation time so diff --git a/cli/src/build/onboarding/android/ui/app.tsx b/cli/src/build/onboarding/android/ui/app.tsx index eafb3129c0..a84b381bb8 100644 --- a/cli/src/build/onboarding/android/ui/app.tsx +++ b/cli/src/build/onboarding/android/ui/app.tsx @@ -27,10 +27,11 @@ import { createSupabaseClient, findSavedKey, findSavedKeySilent, getOrganization import { loadSavedCredentials, updateSavedCredentials } from '../../../credentials.js' import { requestBuildInternal } from '../../../request.js' import type { CiSecretEntry, CiSecretSetupAdvice, CiSecretTarget } from '../../ci-secrets.js' +import type { BuilderOnboardingAction } from '../../telemetry.js' import { createCiSecretEntries, detectCiSecretTargets, getCiSecretTargetLabel, listExistingCiSecretKeys, uploadCiSecrets } from '../../ci-secrets.js' import { mapAndroidOnboardingError, mapSaValidationKindToCategory } from '../../error-categories.js' import { canUseFilePicker, openKeystorePicker, openServiceAccountJsonPicker } from '../../file-picker.js' -import { trackBuilderOnboardingStep } from '../../telemetry.js' +import { trackBuilderOnboardingAction, trackBuilderOnboardingStep } from '../../telemetry.js' import { Divider, ErrorLine, FilteredTextInput, Header, SpinnerLine, SuccessLine } from '../../ui/components.js' import { findAndroidApplicationIds } from '../gradle-parser.js' import { validateServiceAccountJson } from '../service-account-validation.js' @@ -69,7 +70,7 @@ import { inviteServiceAccount, PLAY_DEVELOPERS_URL, } from '../play-api.js' -import { deleteAndroidProgress, getAndroidResumeStep, loadAndroidProgress, saveAndroidProgress } from '../progress.js' +import { deleteAndroidProgress, getAndroidResumeStep, hasAnyOAuthProgress, loadAndroidProgress, saveAndroidProgress } from '../progress.js' import { ANDROID_STEP_PROGRESS, getAndroidPhaseLabel } from '../types.js' interface LogEntry { text: string, color?: string } @@ -139,6 +140,11 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir durationMs?: number errorCategory?: AndroidOnboardingErrorCategory }>>([]) + const pendingActionTelemetryRef = useRef + }>>([]) const [resolvedOrgId, setResolvedOrgId] = useState(null) const resolvedApiKeyRef = useRef(apikey ?? null) const orgIdResolvedRef = useRef(false) @@ -208,6 +214,18 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir } pendingTelemetryRef.current = [] } + if (resolvedOrgId && pendingActionTelemetryRef.current.length > 0) { + for (const queued of pendingActionTelemetryRef.current) { + void trackBuilderOnboardingAction({ + apikey: resolvedApiKeyRef.current, + appId, + orgId: resolvedOrgId, + platform: 'android', + ...queued, + }) + } + pendingActionTelemetryRef.current = [] + } // (2) Now safely skip the duplicate-step path. if (isDuplicateStep) @@ -252,6 +270,32 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir } }, [step, appId, resolvedOrgId, error]) + const trackAction = useCallback( + ( + action: BuilderOnboardingAction, + tags?: Record, + actionStep: AndroidOnboardingStep = step, + ): void => { + if (!resolvedApiKeyRef.current) + return + + const payload = { step: actionStep, action, tags } + if (resolvedOrgId) { + void trackBuilderOnboardingAction({ + apikey: resolvedApiKeyRef.current, + appId, + orgId: resolvedOrgId, + platform: 'android', + ...payload, + }) + } + else { + pendingActionTelemetryRef.current.push(payload) + } + }, + [appId, resolvedOrgId, step], + ) + const [retryCount, setRetryCount] = useState(0) const [retryStep, setRetryStep] = useState(null) const exitRequestedRef = useRef(false) @@ -752,6 +796,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir const base64 = jsonBytes.toString('base64') setServiceAccountKeyBase64(base64) setSaValidationResult({ ok: true }) + trackAction('android_sa_validation_result', { result: 'success' }, 'sa-json-validating') await persist((p) => ({ ...p, _serviceAccountKeyBase64: base64, @@ -764,9 +809,13 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir } setSaValidationResult(result) - // Stash the validation failure kind so the PostHog - // `sa-json-validation-failed` step event carries the dimension. - // Read by the telemetry useEffect on the upcoming step transition. + trackAction('android_sa_validation_result', { + result: 'failure', + validation_kind: result.kind, + }, 'sa-json-validating') + // Emit the immediate action event above, and stash the validation + // kind so the upcoming `sa-json-validation-failed` step event also + // carries the same failure category. errorCategoryRef.current = mapSaValidationKindToCategory(result.kind) // shape-error indicates the file itself is wrong — surface as a // banner log and route to the same recovery screen so the user @@ -895,6 +944,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir ...p, keystoreKeyPassword: keyPw, _keystoreBase64: base64, + serviceAccountForkSeen: true, completedSteps: { ...p.completedSteps, keystoreReady: ready }, })) addLog(`✔ Keystore loaded — ${keystoreExistingPath}`) @@ -907,14 +957,8 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir const fresh = await loadAndroidProgress(appId) if (cancelled) return - const hasAnyOAuthProgress = !!( - fresh?.completedSteps.googleSignInComplete - || fresh?.completedSteps.playAccountChosen - || fresh?.completedSteps.gcpProjectChosen - || fresh?.completedSteps.androidPackageChosen - || fresh?._oauthRefreshToken - ) - if (hasAnyOAuthProgress || fresh?.serviceAccountMethod !== undefined) + const hasOAuthProgress = fresh ? hasAnyOAuthProgress(fresh) : false + if (hasOAuthProgress || fresh?.serviceAccountMethod !== undefined) setStep(fresh ? getAndroidResumeStep(fresh) : 'service-account-method-select') else setStep('service-account-method-select') @@ -956,6 +1000,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir keystoreKeyPassword: keyPw, keystoreCommonName: cn, _keystoreBase64: result.p12Base64, + serviceAccountForkSeen: true, completedSteps: { ...p.completedSteps, keystoreReady: ready }, })) addLog(`✔ Keystore generated — alias: ${result.alias}, valid until ${result.notAfter.getFullYear()}`) @@ -1726,6 +1771,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir ...p, keystoreKeyPassword: keyPw, _keystoreBase64: base64, + serviceAccountForkSeen: true, completedSteps: { ...p.completedSteps, keystoreReady: ready }, })) addLog(`✔ Keystore loaded — ${keystoreExistingPath}`) @@ -1734,14 +1780,8 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir // mid-flow), pick up where they left off; otherwise drop // them on the new fork. const fresh = await loadAndroidProgress(appId) - const hasAnyOAuthProgress = !!( - fresh?.completedSteps.googleSignInComplete - || fresh?.completedSteps.playAccountChosen - || fresh?.completedSteps.gcpProjectChosen - || fresh?.completedSteps.androidPackageChosen - || fresh?._oauthRefreshToken - ) - if (hasAnyOAuthProgress || fresh?.serviceAccountMethod !== undefined) + const hasOAuthProgress = fresh ? hasAnyOAuthProgress(fresh) : false + if (hasOAuthProgress || fresh?.serviceAccountMethod !== undefined) setStep(fresh ? getAndroidResumeStep(fresh) : 'service-account-method-select') else setStep('service-account-method-select') @@ -1882,6 +1922,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir selectFiredRef.current = true const method: 'existing' | 'generate' = value === 'existing' ? 'existing' : 'generate' setServiceAccountMethod(method) + trackAction('android_sa_method_selected', { method }) if (method === 'existing') { // Import path needs the package name first so validation can // probe edits.insert(packageName). The package-select step is @@ -2013,6 +2054,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir // sa-json-validation-failed exit point. errorCategoryRef.current = undefined if (value === 'retry') { + trackAction('android_sa_validation_recovery_selected', { recovery_action: 'retry' }) // Clear the saved path so the picker chooser shows fresh. setServiceAccountJsonPath('') setSaValidationResult(null) @@ -2024,6 +2066,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir return } if (value === 'save-anyway') { + trackAction('android_sa_validation_recovery_selected', { recovery_action: 'save_anyway' }) ;(async () => { try { if (!serviceAccountJsonPath) @@ -2046,6 +2089,7 @@ const AndroidOnboardingApp: FC = ({ appId, initialProgress, androidDir return } // oauth — fall back to the OAuth provisioning path. + trackAction('android_sa_validation_recovery_selected', { recovery_action: 'fallback_oauth' }) setServiceAccountMethod('generate') setSaValidationResult(null) persistAndStep( diff --git a/cli/src/build/onboarding/telemetry.ts b/cli/src/build/onboarding/telemetry.ts index dbf0a75905..fe86a52d6e 100644 --- a/cli/src/build/onboarding/telemetry.ts +++ b/cli/src/build/onboarding/telemetry.ts @@ -16,6 +16,21 @@ export interface TrackBuilderOnboardingStepInput { errorCategory?: OnboardingErrorCategory | AndroidOnboardingErrorCategory } +export type BuilderOnboardingAction + = | 'android_sa_method_selected' + | 'android_sa_validation_recovery_selected' + | 'android_sa_validation_result' + +export interface TrackBuilderOnboardingActionInput { + apikey: string + appId: string + orgId: string + platform: Platform + step: OnboardingStep | AndroidOnboardingStep + action: BuilderOnboardingAction + tags?: Record +} + export async function trackBuilderOnboardingStep(input: TrackBuilderOnboardingStepInput): Promise { const tags: Record = { step: input.step, @@ -50,3 +65,29 @@ export async function trackBuilderOnboardingStep(input: TrackBuilderOnboardingSt // fetch failures internally; this catch covers anything else. } } + +export async function trackBuilderOnboardingAction(input: TrackBuilderOnboardingActionInput): Promise { + const tags: Record = {} + + for (const [key, value] of Object.entries(input.tags ?? {})) + tags[key] = String(value) + + tags.step = input.step + tags.platform = input.platform + tags.app_id = input.appId + tags.action = input.action + + try { + await sendEvent(input.apikey, { + event: 'Builder Onboarding Action', + channel: 'builder-onboarding', + icon: '🧭', + notify: false, + user_id: input.orgId, + tags, + }) + } + catch { + // Telemetry must never break the wizard. + } +} diff --git a/cli/test/test-android-onboarding-progress.mjs b/cli/test/test-android-onboarding-progress.mjs new file mode 100644 index 0000000000..7c9d75bf3d --- /dev/null +++ b/cli/test/test-android-onboarding-progress.mjs @@ -0,0 +1,152 @@ +#!/usr/bin/env node +/** + * Focused resume-routing tests for Android onboarding progress. + */ +import process from 'node:process' + +console.log('🧪 Testing Android onboarding progress routing...\n') + +const { getAndroidResumeStep, hasAnyOAuthProgress } = await import('../src/build/onboarding/android/progress.ts') + +let testsPassed = 0 +let testsFailed = 0 + +async function test(name, fn) { + try { + console.log(`\n🔍 ${name}`) + await fn() + console.log(`✅ PASSED: ${name}`) + testsPassed++ + } + catch (error) { + console.error(`❌ FAILED: ${name}`) + console.error(` Error: ${error.message}`) + testsFailed++ + } +} + +function assertEquals(a, b, msg) { + if (a !== b) + throw new Error(msg || `Expected ${b}, got ${a}`) +} + +function keystoreReadyProgress(overrides = {}) { + return { + platform: 'android', + appId: 'com.example.app', + startedAt: '2026-05-22T00:00:00.000Z', + keystoreMethod: 'generate', + keystoreAlias: 'release', + keystoreStorePassword: 'store-pass', + _keystoreBase64: 'keystore-base64', + completedSteps: { + keystoreReady: { + keystorePath: 'android/app/release.p12', + alias: 'release', + isGenerated: true, + }, + }, + ...overrides, + } +} + +await test('fresh runs return to service-account method select if quit before choosing', async () => { + assertEquals( + getAndroidResumeStep(keystoreReadyProgress({ serviceAccountForkSeen: true })), + 'service-account-method-select', + ) +}) + +await test('legacy progress without fork marker still resumes OAuth path', async () => { + assertEquals(getAndroidResumeStep(keystoreReadyProgress()), 'google-sign-in') +}) + +await test('generate service-account path resumes OAuth sign-in', async () => { + assertEquals( + getAndroidResumeStep(keystoreReadyProgress({ + serviceAccountForkSeen: true, + serviceAccountMethod: 'generate', + })), + 'google-sign-in', + ) +}) + +await test('existing service-account path resumes package selection before package is known', async () => { + assertEquals( + getAndroidResumeStep(keystoreReadyProgress({ + serviceAccountForkSeen: true, + serviceAccountMethod: 'existing', + })), + 'android-package-select', + ) +}) + +await test('existing service-account path resumes validation after JSON path is saved', async () => { + assertEquals( + getAndroidResumeStep(keystoreReadyProgress({ + serviceAccountForkSeen: true, + serviceAccountMethod: 'existing', + serviceAccountJsonPath: '/tmp/service-account.json', + completedSteps: { + ...keystoreReadyProgress().completedSteps, + androidPackageChosen: { + packageName: 'com.example.app', + source: 'user-input', + }, + }, + })), + 'sa-json-validating', + ) +}) + +await test('existing service-account path resumes saving credentials after JSON is accepted', async () => { + assertEquals( + getAndroidResumeStep(keystoreReadyProgress({ + serviceAccountForkSeen: true, + serviceAccountMethod: 'existing', + _serviceAccountKeyBase64: 'service-account-json-base64', + })), + 'saving-credentials', + ) +}) + +for (const { label, patch } of [ + { + label: 'google sign-in marker', + patch: { completedSteps: { ...keystoreReadyProgress().completedSteps, googleSignInComplete: { email: 'user@example.com' } } }, + }, + { + label: 'Play account marker', + patch: { completedSteps: { ...keystoreReadyProgress().completedSteps, playAccountChosen: { accountId: '123456789' } } }, + }, + { + label: 'GCP project marker', + patch: { completedSteps: { ...keystoreReadyProgress().completedSteps, gcpProjectChosen: { projectId: 'capgo-test' } } }, + }, + { + label: 'Android package marker', + patch: { + completedSteps: { + ...keystoreReadyProgress().completedSteps, + androidPackageChosen: { packageName: 'com.example.app', source: 'user-input' }, + }, + }, + }, + { + label: 'OAuth refresh token', + patch: { _oauthRefreshToken: 'refresh-token' }, + }, +]) { + await test(`fresh fork marker with ${label} keeps legacy OAuth resume`, async () => { + const progress = keystoreReadyProgress({ + serviceAccountForkSeen: true, + ...patch, + }) + assertEquals(hasAnyOAuthProgress(progress), true) + assertEquals(getAndroidResumeStep(progress), 'google-sign-in') + }) +} + +console.log(`\n📊 Results: ${testsPassed} passed, ${testsFailed} failed`) +if (testsFailed > 0) + process.exit(1) diff --git a/cli/test/test-onboarding-telemetry.mjs b/cli/test/test-onboarding-telemetry.mjs new file mode 100644 index 0000000000..9078ce7cb0 --- /dev/null +++ b/cli/test/test-onboarding-telemetry.mjs @@ -0,0 +1,65 @@ +#!/usr/bin/env node +import assert from 'node:assert/strict' +import { trackBuilderOnboardingAction } from '../src/build/onboarding/telemetry.ts' + +console.log('🧪 Testing onboarding telemetry...\n') + +const originalFetch = globalThis.fetch + +try { + const requests = [] + globalThis.fetch = async (url, init) => { + requests.push({ init, url: String(url) }) + if (String(url).endsWith('/private/config')) + return new Response('', { status: 500 }) + return new Response('{}', { + headers: { 'Content-Type': 'application/json' }, + status: 200, + }) + } + + await trackBuilderOnboardingAction({ + action: 'android_sa_method_selected', + apikey: 'capgo-key', + appId: 'com.example.app', + orgId: 'org-id', + platform: 'android', + step: 'service-account-method-select', + tags: { + accepted: true, + action: 'caller_action', + app_id: 'caller_app', + attempt: 1, + method: 'existing', + platform: 'ios', + step: 'caller-step', + }, + }) + + const eventRequest = requests.find(request => request.url.endsWith('/private/events')) + assert.ok(eventRequest, 'Expected onboarding action telemetry request') + assert.equal(eventRequest.init.method, 'POST') + assert.equal(eventRequest.init.headers.capgkey, 'capgo-key') + assert.equal(eventRequest.init.headers['Content-Type'], 'application/json') + assert.equal(eventRequest.init.signal instanceof AbortSignal, true) + + const body = JSON.parse(eventRequest.init.body) + assert.equal(body.event, 'Builder Onboarding Action') + assert.equal(body.channel, 'builder-onboarding') + assert.equal(body.notify, false) + assert.equal(body.user_id, 'org-id') + assert.deepEqual(body.tags, { + accepted: 'true', + action: 'android_sa_method_selected', + app_id: 'com.example.app', + attempt: '1', + method: 'existing', + platform: 'android', + step: 'service-account-method-select', + }) + + console.log('✅ Onboarding telemetry tests passed') +} +finally { + globalThis.fetch = originalFetch +}