diff --git a/demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt b/demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt index a41125cd7..f1bb379c3 100644 --- a/demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt +++ b/demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt @@ -19,11 +19,14 @@ import android.app.Instrumentation import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.util.Log import androidx.test.platform.app.InstrumentationRegistry import androidx.test.uiautomator.UiDevice import com.google.maps.android.visualtesting.GeminiVisualTestHelper import org.junit.Assert.assertTrue +import org.junit.Assert.fail import java.io.File +import java.io.FileOutputStream abstract class BaseVisualTest { @@ -67,4 +70,57 @@ abstract class BaseVisualTest { e.printStackTrace() } } + + /** + * Verifies the screenshot against a golden image. + * If the golden matches, the test passes immediately. + * If it fails (or no golden exists), it falls back to Gemini for visual verification. + * If Gemini passes, it saves/updates the golden image and logs a warning to review it. + */ + protected suspend fun verifyScreenshotWithGoldenFallback( + testName: String, + screenshotBitmap: Bitmap, + prompt: String, + passCondition: (String) -> Boolean + ) { + val goldenFile = File(context.getExternalFilesDir(null), "goldens/$testName.png") + + var isPixelMatch = false + if (goldenFile.exists()) { + val goldenBitmap = BitmapFactory.decodeFile(goldenFile.absolutePath) + isPixelMatch = compareBitmaps(screenshotBitmap, goldenBitmap) + } + + if (isPixelMatch) { + Log.i("VisualTest", "Screenshot matched golden perfectly for '$testName'. Fast pass.") + return + } + + Log.w("VisualTest", "Screenshot did not match golden for '$testName'. Falling back to Gemini.") + val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey) + + requireNotNull(geminiResponse) { "Gemini response was null for test '$testName'" } + + val passed = passCondition(geminiResponse) + + if (passed) { + // Update golden since Gemini approved it + goldenFile.parentFile?.mkdirs() + FileOutputStream(goldenFile).use { out -> + screenshotBitmap.compress(Bitmap.CompressFormat.PNG, 100, out) + } + Log.w("VisualTest", "Gemini approved the new UI state for '$testName'. Golden image updated at ${goldenFile.absolutePath}. Please review this image manually to prevent baking in hallucinations.") + } else { + fail("Visual verification failed for '$testName'. Gemini response: $geminiResponse") + } + } + + private fun compareBitmaps(b1: Bitmap, b2: Bitmap): Boolean { + if (b1.width != b2.width || b1.height != b2.height) return false + val pixels1 = IntArray(b1.width * b1.height) + val pixels2 = IntArray(b2.width * b2.height) + b1.getPixels(pixels1, 0, b1.width, 0, 0, b1.width, b1.height) + b2.getPixels(pixels2, 0, b2.width, 0, 0, b2.width, b2.height) + return pixels1.contentEquals(pixels2) + } } diff --git a/demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt b/demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt index 032eeef31..8fc4c5c40 100644 --- a/demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt +++ b/demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt @@ -22,7 +22,6 @@ import androidx.test.uiautomator.By import androidx.test.uiautomator.Until import kotlinx.coroutines.runBlocking import org.junit.Assert.assertNotNull -import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test import org.junit.runner.RunWith @@ -54,12 +53,11 @@ class ClusteringVisualTest : BaseVisualTest() { // --- Perform a visual assertion on the new screen --- val prompt = "Does this image show a map with several markers clustered together? Answer only YES or NO." - val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey) - - println("Gemini's analysis after natural language click: $geminiResponse") - assertTrue( - "Visual verification failed. Gemini did not confirm the presence of a map with clusters.", - geminiResponse?.contains("YES", ignoreCase = true) == true + verifyScreenshotWithGoldenFallback( + testName = "Clustering_NaturalLanguageClick", + screenshotBitmap = screenshotBitmap, + prompt = prompt, + passCondition = { it.contains("YES", ignoreCase = true) } ) } @@ -98,15 +96,11 @@ class ClusteringVisualTest : BaseVisualTest() { If all three elements are present and legible, just confirm that the visual test has PASSED. If any element is missing or incorrect, please detail the discrepancy. """.trimIndent() - // --- STEP 3: Analyze the image using Gemini --- - val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey) - - // --- STEP 4: Assert on Gemini's response --- - println("Gemini's analysis: $geminiResponse") - // Example assertion: Check if Gemini confirms the presence of clusters - assertTrue( - "PASSED", - geminiResponse!!.contains("PASSED", ignoreCase = true) + verifyScreenshotWithGoldenFallback( + testName = "Clustering_ScreenContent", + screenshotBitmap = screenshotBitmap, + prompt = prompt, + passCondition = { it.contains("PASSED", ignoreCase = true) } ) } } \ No newline at end of file