diff --git a/src/mvt/android/artifacts/tombstone_crashes.py b/src/mvt/android/artifacts/tombstone_crashes.py index 9a878aa14..9c779e665 100644 --- a/src/mvt/android/artifacts/tombstone_crashes.py +++ b/src/mvt/android/artifacts/tombstone_crashes.py @@ -10,7 +10,7 @@ import betterproto2 from dateutil import parser -from mvt.android.parsers.proto.tombstone import Tombstone +from mvt.android.parsers.protobuf_parsers import parse_tombstone_record from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult from mvt.common.utils import convert_datetime_to_iso @@ -128,7 +128,7 @@ def parse_protobuf( self, file_name: str, file_timestamp: datetime.datetime, data: bytes ) -> None: """Parse Android tombstone crash files from a protobuf object.""" - tombstone_pb = Tombstone().parse(data) + tombstone_pb = parse_tombstone_record(data) tombstone_dict = tombstone_pb.to_dict( casing=betterproto2.Casing.SNAKE, include_default_values=True ) diff --git a/src/mvt/android/modules/androidqf/aqf_files.py b/src/mvt/android/modules/androidqf/aqf_files.py index de9b44a6c..382ad2eb1 100644 --- a/src/mvt/android/modules/androidqf/aqf_files.py +++ b/src/mvt/android/modules/androidqf/aqf_files.py @@ -14,6 +14,7 @@ from typing import Optional from mvt.android.modules.androidqf.base import AndroidQFModule +from mvt.android.parsers.protobuf_parsers import parse_files_records from mvt.common.module_types import ( ModuleAtomicResult, ModuleResults, @@ -125,6 +126,7 @@ def run(self) -> None: self.log.warning("Unable to determine device timezone, using UTC") device_timezone = zoneinfo.ZoneInfo("UTC") + data = [] for file in self._get_files_by_pattern("*/files.json"): rawdata = self._get_file_content(file).decode("utf-8", errors="ignore") try: @@ -136,24 +138,31 @@ def run(self) -> None: continue data.append(json.loads(line)) - for file_data in data: - for ts in ["access_time", "changed_time", "modified_time"]: - if ts in file_data: - utc_timestamp = datetime.datetime.fromtimestamp( - file_data[ts], tz=datetime.timezone.utc - ) - # Convert the UTC timestamp to local time on Android device's local timezone - local_timestamp = utc_timestamp.astimezone(device_timezone) - - # Preserve the device-local wall-clock time while using - # the project-wide ISO conversion helper. - local_timestamp = local_timestamp.replace( - tzinfo=datetime.timezone.utc - ) - file_data[ts] = convert_datetime_to_iso(local_timestamp) - - self.results.append(file_data) - - break # Only process the first matching file + if data == []: + for file in self._get_files_by_pattern("*/files.pb"): + try: + data = parse_files_records(self._get_file_content(file)) + except ValueError as exc: + self.log.error("Failed to parse files.pb: %s", exc) + return + break + + for file_data in data: + for ts in ["access_time", "changed_time", "modified_time"]: + if ts in file_data: + utc_timestamp = datetime.datetime.fromtimestamp( + file_data[ts], tz=datetime.timezone.utc + ) + # Convert the UTC timestamp to local time on Android device's local timezone + local_timestamp = utc_timestamp.astimezone(device_timezone) + + # Preserve the device-local wall-clock time while using + # the project-wide ISO conversion helper. + local_timestamp = local_timestamp.replace( + tzinfo=datetime.timezone.utc + ) + file_data[ts] = convert_datetime_to_iso(local_timestamp) + + self.results.append(file_data) self.log.info("Found a total of %d files", len(self.results)) diff --git a/src/mvt/android/modules/androidqf/aqf_packages.py b/src/mvt/android/modules/androidqf/aqf_packages.py index 294ad5302..8a4948ffa 100644 --- a/src/mvt/android/modules/androidqf/aqf_packages.py +++ b/src/mvt/android/modules/androidqf/aqf_packages.py @@ -10,6 +10,7 @@ from rich.progress import track +from mvt.android.parsers.protobuf_parsers import parse_packages_records from mvt.android.utils import ( BROWSER_INSTALLERS, PLAY_STORE_INSTALLERS, @@ -190,11 +191,18 @@ def check_virustotal(self, delay: int = 0) -> None: def run(self) -> None: packages = self._get_files_by_pattern("*/packages.json") - if not packages: - self.log.error( - "packages.json file not found in this androidqf bundle. Possibly malformed?" - ) + if packages: + self.results = json.loads(self._get_file_content(packages[0])) + self.log.info("Found %d packages in packages.json", len(self.results)) + return + + packages = self._get_files_by_pattern("*/packages.pb") + if packages: + self.results = parse_packages_records(self._get_file_content(packages[0])) + self.log.info("Found %d packages in packages.pb", len(self.results)) return - self.results = json.loads(self._get_file_content(packages[0])) - self.log.info("Found %d packages in packages.json", len(self.results)) + self.log.error( + "packages.json or packages.pb file not found in this androidqf bundle. Possibly malformed?" + ) + return diff --git a/src/mvt/android/modules/androidqf/mounts.py b/src/mvt/android/modules/androidqf/mounts.py index ea446c2b1..2e24477ba 100644 --- a/src/mvt/android/modules/androidqf/mounts.py +++ b/src/mvt/android/modules/androidqf/mounts.py @@ -8,6 +8,7 @@ from typing import Optional from mvt.android.artifacts.mounts import Mounts as MountsArtifact +from mvt.android.parsers.protobuf_parsers import parse_string_records from .base import AndroidQFModule @@ -34,6 +35,14 @@ def __init__( ) self.results: list = [] if results is None else results + def _load_json(self, file: str) -> list[str]: + data = self._get_file_content(file).decode("utf-8", errors="replace") + return json.loads(data) + + def _load_pb(self, file: str) -> list[str]: + data = self._get_file_content(file) + return parse_string_records(data) + def run(self) -> None: """ Run the mounts analysis module. @@ -42,30 +51,37 @@ def run(self) -> None: and analyzes them for suspicious configurations, particularly focusing on detecting root access indicators like /system mounted as read-write. """ - mount_files = self._get_files_by_pattern("*/mounts.json") - if not mount_files: - self.log.info("No mount information file found") - return + mount_data = [] - self.log.info("Found mount information file: %s", mount_files[0]) + mount_files = self._get_files_by_pattern("*/mounts.json") + if mount_files: + try: + mount_data = self._load_json(mount_files[0]) + self.log.info("Found mount information file: %s", mount_files[0]) + except Exception as exc: + self.log.error("Failed to parse JSON mount information: %s", exc) + return - try: - data = self._get_file_content(mount_files[0]).decode( - "utf-8", errors="replace" - ) - except Exception as exc: - self.log.error("Failed to read mount information file: %s", exc) + mount_files = self._get_files_by_pattern("*/mounts.pb") + if len(mount_data) == 0 and mount_files: + try: + mount_data = self._load_pb(mount_files[0]) + self.log.info("Found mount information file: %s", mount_files[0]) + except Exception as exc: + self.log.error("Failed to parse Protobuf mount information: %s", exc) + return + + if len(mount_data) == 0: + self.log.info("No mount information file found") return # Parse the mount data try: - json_data = json.loads(data) - - if isinstance(json_data, list): + if isinstance(mount_data, list): # AndroidQF format: array of strings like # "/dev/block/dm-12 on / type ext4 (ro,seclabel,noatime)" - mount_content = "\n".join(json_data) + mount_content = "\n".join(mount_data) else: self.log.error("Expected mounts.json to contain a list of mount lines") return diff --git a/src/mvt/android/modules/androidqf/root_binaries.py b/src/mvt/android/modules/androidqf/root_binaries.py index 81e8173b8..442f2e37d 100644 --- a/src/mvt/android/modules/androidqf/root_binaries.py +++ b/src/mvt/android/modules/androidqf/root_binaries.py @@ -7,6 +7,8 @@ import logging from typing import Optional +from mvt.android.parsers.protobuf_parsers import parse_string_records + from .base import AndroidQFModule @@ -58,26 +60,42 @@ def check_indicators(self) -> None: len(self.results), ) - def run(self) -> None: - """Run the root binaries analysis.""" - root_binaries_files = self._get_files_by_pattern("*/root_binaries.json") + def _load_json(self, file: str) -> list[str]: + data = self._get_file_content(file).decode("utf-8", errors="ignore") + return json.loads(data) - if not root_binaries_files: - self.log.info("No root_binaries.json file found") - return + def _load_pb(self, file: str) -> list[str]: + data = self._get_file_content(file) + return parse_string_records(data) - rawdata = self._get_file_content(root_binaries_files[0]).decode( - "utf-8", errors="ignore" - ) + def run(self) -> None: + """Run the root binaries analysis.""" + root_binary_paths = [] - try: - root_binary_paths = json.loads(rawdata) - except json.JSONDecodeError as e: - self.log.error("Failed to parse root_binaries.json: %s", e) + root_binaries_files = self._get_files_by_pattern("*/root_binaries.json") + if root_binaries_files: + try: + root_binary_paths = self._load_json(root_binaries_files[0]) + self.log.info("Found root_binaries.json file: %s", root_binaries_files[0]) + except Exception as exc: + self.log.error("Failed to parse JSON root_binaries.json: %s", exc) + return + + root_binaries_files = self._get_files_by_pattern("*/root_binaries.pb") + if len(root_binary_paths) == 0 and root_binaries_files: + try: + root_binary_paths = self._load_pb(root_binaries_files[0]) + self.log.info("Found root_binaries.pb file: %s", root_binaries_files[0]) + except Exception as exc: + self.log.error("Failed to parse Protobuf root_binaries.pb: %s", exc) + return + + if len(root_binary_paths) == 0: + self.log.info("No root_binaries file found") return if not isinstance(root_binary_paths, list): - self.log.error("Expected root_binaries.json to contain a list of paths") + self.log.error("Expected root_binaries.json or root_binaries.pb to contain a list of paths") return # Known root binary names that might be found and their descriptions diff --git a/src/mvt/android/parsers/proto/__init__.py b/src/mvt/android/parsers/proto/__init__.py index e69de29bb..675fab134 100644 --- a/src/mvt/android/parsers/proto/__init__.py +++ b/src/mvt/android/parsers/proto/__init__.py @@ -0,0 +1,603 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# sources: files_record.proto, packages_record.proto, string_record.proto, tombstone.proto +# plugin: python-betterproto2 +# This file has been @generated + +__all__ = ( + "Architecture", + "ArmMteMetadata", + "BacktraceFrame", + "Cause", + "CrashDetail", + "Fd", + "FilesRecord", + "HeapObject", + "LogBuffer", + "LogMessage", + "MemoryDump", + "MemoryError", + "MemoryErrorTool", + "MemoryErrorType", + "MemoryMapping", + "PackageCertificateRecord", + "PackageFileRecord", + "PackagesRecord", + "Register", + "Signal", + "StackHistoryBuffer", + "StackHistoryBufferEntry", + "StringRecord", + "Thread", + "Tombstone", +) + +from dataclasses import dataclass + +import betterproto2 + +from .message_pool import default_message_pool + +_COMPILER_VERSION = "0.9.0" +betterproto2.check_compiler_version(_COMPILER_VERSION) + + +class Architecture(betterproto2.Enum): + ARM32 = 0 + + ARM64 = 1 + + X86 = 2 + + X86_64 = 3 + + RISCV64 = 4 + + NONE = 5 + + +class MemoryErrorTool(betterproto2.Enum): + GWP_ASAN = 0 + + SCUDO = 1 + + +class MemoryErrorType(betterproto2.Enum): + UNKNOWN = 0 + + USE_AFTER_FREE = 1 + + DOUBLE_FREE = 2 + + INVALID_FREE = 3 + + BUFFER_OVERFLOW = 4 + + BUFFER_UNDERFLOW = 5 + + +@dataclass(eq=False, repr=False) +class ArmMteMetadata(betterproto2.Message): + memory_tags: "bytes" = betterproto2.field(1, betterproto2.TYPE_BYTES) + """ + One memory tag per granule (e.g. every 16 bytes) of regular memory. + """ + + +default_message_pool.register_message("", "ArmMTEMetadata", ArmMteMetadata) + + +@dataclass(eq=False, repr=False) +class BacktraceFrame(betterproto2.Message): + rel_pc: "int" = betterproto2.field(1, betterproto2.TYPE_UINT64) + + pc: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) + + sp: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) + + function_name: "str" = betterproto2.field(4, betterproto2.TYPE_STRING) + + function_offset: "int" = betterproto2.field(5, betterproto2.TYPE_UINT64) + + file_name: "str" = betterproto2.field(6, betterproto2.TYPE_STRING) + + file_map_offset: "int" = betterproto2.field(7, betterproto2.TYPE_UINT64) + + build_id: "str" = betterproto2.field(8, betterproto2.TYPE_STRING) + + +default_message_pool.register_message("", "BacktraceFrame", BacktraceFrame) + + +@dataclass(eq=False, repr=False) +class Cause(betterproto2.Message): + """ + + + Oneofs: + - details: + """ + + human_readable: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + memory_error: "MemoryError | None" = betterproto2.field( + 2, betterproto2.TYPE_MESSAGE, optional=True, group="details" + ) + + +default_message_pool.register_message("", "Cause", Cause) + + +@dataclass(eq=False, repr=False) +class CrashDetail(betterproto2.Message): + """ + NOTE TO OEMS: + If you add custom fields to this proto, do not use numbers in the reserved range. + """ + + name: "bytes" = betterproto2.field(1, betterproto2.TYPE_BYTES) + + data: "bytes" = betterproto2.field(2, betterproto2.TYPE_BYTES) + + +default_message_pool.register_message("", "CrashDetail", CrashDetail) + + +@dataclass(eq=False, repr=False) +class Fd(betterproto2.Message): + fd: "int" = betterproto2.field(1, betterproto2.TYPE_INT32) + + path: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) + + owner: "str" = betterproto2.field(3, betterproto2.TYPE_STRING) + + tag: "int" = betterproto2.field(4, betterproto2.TYPE_UINT64) + + +default_message_pool.register_message("", "FD", Fd) + + +@dataclass(eq=False, repr=False) +class FilesRecord(betterproto2.Message): + path: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + mtime: "float" = betterproto2.field(2, betterproto2.TYPE_DOUBLE) + """ + Unix epoch seconds (optional) + """ + + mode: "str" = betterproto2.field(3, betterproto2.TYPE_STRING) + """ + e.g. "0644" (optional) + """ + + size: "int" = betterproto2.field(4, betterproto2.TYPE_INT64) + """ + bytes (optional) + """ + + user: "str" = betterproto2.field(5, betterproto2.TYPE_STRING) + """ + owner name (optional) + """ + + group: "str" = betterproto2.field(6, betterproto2.TYPE_STRING) + """ + group name (optional) + """ + + +default_message_pool.register_message("", "FilesRecord", FilesRecord) + + +@dataclass(eq=False, repr=False) +class HeapObject(betterproto2.Message): + address: "int" = betterproto2.field(1, betterproto2.TYPE_UINT64) + + size: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) + + allocation_tid: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) + + allocation_backtrace: "list[BacktraceFrame]" = betterproto2.field( + 4, betterproto2.TYPE_MESSAGE, repeated=True + ) + + deallocation_tid: "int" = betterproto2.field(5, betterproto2.TYPE_UINT64) + + deallocation_backtrace: "list[BacktraceFrame]" = betterproto2.field( + 6, betterproto2.TYPE_MESSAGE, repeated=True + ) + + +default_message_pool.register_message("", "HeapObject", HeapObject) + + +@dataclass(eq=False, repr=False) +class LogBuffer(betterproto2.Message): + name: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + logs: "list[LogMessage]" = betterproto2.field( + 2, betterproto2.TYPE_MESSAGE, repeated=True + ) + + +default_message_pool.register_message("", "LogBuffer", LogBuffer) + + +@dataclass(eq=False, repr=False) +class LogMessage(betterproto2.Message): + timestamp: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + pid: "int" = betterproto2.field(2, betterproto2.TYPE_UINT32) + + tid: "int" = betterproto2.field(3, betterproto2.TYPE_UINT32) + + priority: "int" = betterproto2.field(4, betterproto2.TYPE_UINT32) + + tag: "str" = betterproto2.field(5, betterproto2.TYPE_STRING) + + message: "str" = betterproto2.field(6, betterproto2.TYPE_STRING) + + +default_message_pool.register_message("", "LogMessage", LogMessage) + + +@dataclass(eq=False, repr=False) +class MemoryDump(betterproto2.Message): + """ + + + Oneofs: + - metadata: + """ + + register_name: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + mapping_name: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) + + begin_address: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) + + memory: "bytes" = betterproto2.field(4, betterproto2.TYPE_BYTES) + + arm_mte_metadata: "ArmMteMetadata | None" = betterproto2.field( + 6, betterproto2.TYPE_MESSAGE, optional=True, group="metadata" + ) + + +default_message_pool.register_message("", "MemoryDump", MemoryDump) + + +@dataclass(eq=False, repr=False) +class MemoryError(betterproto2.Message): + """ + + + Oneofs: + - location: + """ + + tool: "MemoryErrorTool" = betterproto2.field( + 1, betterproto2.TYPE_ENUM, default_factory=lambda: MemoryErrorTool(0) + ) + + type: "MemoryErrorType" = betterproto2.field( + 2, betterproto2.TYPE_ENUM, default_factory=lambda: MemoryErrorType(0) + ) + + heap: "HeapObject | None" = betterproto2.field( + 3, betterproto2.TYPE_MESSAGE, optional=True, group="location" + ) + + +default_message_pool.register_message("", "MemoryError", MemoryError) + + +@dataclass(eq=False, repr=False) +class MemoryMapping(betterproto2.Message): + begin_address: "int" = betterproto2.field(1, betterproto2.TYPE_UINT64) + + end_address: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) + + offset: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) + + read: "bool" = betterproto2.field(4, betterproto2.TYPE_BOOL) + + write: "bool" = betterproto2.field(5, betterproto2.TYPE_BOOL) + + execute: "bool" = betterproto2.field(6, betterproto2.TYPE_BOOL) + + mapping_name: "str" = betterproto2.field(7, betterproto2.TYPE_STRING) + + build_id: "str" = betterproto2.field(8, betterproto2.TYPE_STRING) + + load_bias: "int" = betterproto2.field(9, betterproto2.TYPE_UINT64) + + +default_message_pool.register_message("", "MemoryMapping", MemoryMapping) + + +@dataclass(eq=False, repr=False) +class PackageCertificateRecord(betterproto2.Message): + md5: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + sha1: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) + + sha256: "str" = betterproto2.field(3, betterproto2.TYPE_STRING) + + valid_from: "str" = betterproto2.field(4, betterproto2.TYPE_STRING) + + valid_to: "str" = betterproto2.field(5, betterproto2.TYPE_STRING) + + issuer: "str" = betterproto2.field(6, betterproto2.TYPE_STRING) + + subject: "str" = betterproto2.field(7, betterproto2.TYPE_STRING) + + signature_algorithm: "str" = betterproto2.field(8, betterproto2.TYPE_STRING) + + serial_number: "str" = betterproto2.field(9, betterproto2.TYPE_STRING) + + +default_message_pool.register_message( + "", "PackageCertificateRecord", PackageCertificateRecord +) + + +@dataclass(eq=False, repr=False) +class PackageFileRecord(betterproto2.Message): + path: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + local_name: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) + + md5: "str" = betterproto2.field(3, betterproto2.TYPE_STRING) + + sha1: "str" = betterproto2.field(4, betterproto2.TYPE_STRING) + + sha256: "str" = betterproto2.field(5, betterproto2.TYPE_STRING) + + sha512: "str" = betterproto2.field(6, betterproto2.TYPE_STRING) + + suspicious: "bool" = betterproto2.field(7, betterproto2.TYPE_BOOL) + """ + suspiciousness, should APK be dumped? (optional) + """ + + certificates: "list[PackageCertificateRecord]" = betterproto2.field( + 8, betterproto2.TYPE_MESSAGE, repeated=True + ) + """ + certificates (optional) + """ + + infiles: "list[str]" = betterproto2.field( + 9, betterproto2.TYPE_STRING, repeated=True + ) + """ + files in APK (optional) + """ + + +default_message_pool.register_message("", "PackageFileRecord", PackageFileRecord) + + +@dataclass(eq=False, repr=False) +class PackagesRecord(betterproto2.Message): + name: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + installer: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) + + uid: "int" = betterproto2.field(3, betterproto2.TYPE_INT32) + + disabled: "bool" = betterproto2.field(4, betterproto2.TYPE_BOOL) + + system: "bool" = betterproto2.field(5, betterproto2.TYPE_BOOL) + + third_party: "bool" = betterproto2.field(6, betterproto2.TYPE_BOOL) + + files: "list[PackageFileRecord]" = betterproto2.field( + 7, betterproto2.TYPE_MESSAGE, repeated=True + ) + + +default_message_pool.register_message("", "PackagesRecord", PackagesRecord) + + +@dataclass(eq=False, repr=False) +class Register(betterproto2.Message): + name: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + + u64: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) + + +default_message_pool.register_message("", "Register", Register) + + +@dataclass(eq=False, repr=False) +class Signal(betterproto2.Message): + number: "int" = betterproto2.field(1, betterproto2.TYPE_INT32) + + name: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) + + code: "int" = betterproto2.field(3, betterproto2.TYPE_INT32) + + code_name: "str" = betterproto2.field(4, betterproto2.TYPE_STRING) + + has_sender: "bool" = betterproto2.field(5, betterproto2.TYPE_BOOL) + + sender_uid: "int" = betterproto2.field(6, betterproto2.TYPE_INT32) + + sender_pid: "int" = betterproto2.field(7, betterproto2.TYPE_INT32) + + has_fault_address: "bool" = betterproto2.field(8, betterproto2.TYPE_BOOL) + + fault_address: "int" = betterproto2.field(9, betterproto2.TYPE_UINT64) + + fault_adjacent_metadata: "MemoryDump | None" = betterproto2.field( + 10, betterproto2.TYPE_MESSAGE, optional=True + ) + """ + Note, may or may not contain the dump of the actual memory contents. Currently, on arm64, we + only include metadata, and not the contents. + """ + + +default_message_pool.register_message("", "Signal", Signal) + + +@dataclass(eq=False, repr=False) +class StackHistoryBuffer(betterproto2.Message): + tid: "int" = betterproto2.field(1, betterproto2.TYPE_UINT64) + + entries: "list[StackHistoryBufferEntry]" = betterproto2.field( + 2, betterproto2.TYPE_MESSAGE, repeated=True + ) + + +default_message_pool.register_message("", "StackHistoryBuffer", StackHistoryBuffer) + + +@dataclass(eq=False, repr=False) +class StackHistoryBufferEntry(betterproto2.Message): + addr: "BacktraceFrame | None" = betterproto2.field( + 1, betterproto2.TYPE_MESSAGE, optional=True + ) + + fp: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) + + tag: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) + + +default_message_pool.register_message( + "", "StackHistoryBufferEntry", StackHistoryBufferEntry +) + + +@dataclass(eq=False, repr=False) +class StringRecord(betterproto2.Message): + value: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) + """ + Root binaries: absolute path to a su/magisk-like binary. + Mounts: one line from `mount` output. + """ + + +default_message_pool.register_message("", "StringRecord", StringRecord) + + +@dataclass(eq=False, repr=False) +class Thread(betterproto2.Message): + id: "int" = betterproto2.field(1, betterproto2.TYPE_INT32) + + name: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) + + registers: "list[Register]" = betterproto2.field( + 3, betterproto2.TYPE_MESSAGE, repeated=True + ) + + backtrace_note: "list[str]" = betterproto2.field( + 7, betterproto2.TYPE_STRING, repeated=True + ) + + unreadable_elf_files: "list[str]" = betterproto2.field( + 9, betterproto2.TYPE_STRING, repeated=True + ) + + current_backtrace: "list[BacktraceFrame]" = betterproto2.field( + 4, betterproto2.TYPE_MESSAGE, repeated=True + ) + + memory_dump: "list[MemoryDump]" = betterproto2.field( + 5, betterproto2.TYPE_MESSAGE, repeated=True + ) + + tagged_addr_ctrl: "int" = betterproto2.field(6, betterproto2.TYPE_INT64) + + pac_enabled_keys: "int" = betterproto2.field(8, betterproto2.TYPE_INT64) + + +default_message_pool.register_message("", "Thread", Thread) + + +@dataclass(eq=False, repr=False) +class Tombstone(betterproto2.Message): + arch: "Architecture" = betterproto2.field( + 1, betterproto2.TYPE_ENUM, default_factory=lambda: Architecture(0) + ) + + guest_arch: "Architecture" = betterproto2.field( + 24, betterproto2.TYPE_ENUM, default_factory=lambda: Architecture(0) + ) + + build_fingerprint: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) + + revision: "str" = betterproto2.field(3, betterproto2.TYPE_STRING) + + timestamp: "str" = betterproto2.field(4, betterproto2.TYPE_STRING) + + pid: "int" = betterproto2.field(5, betterproto2.TYPE_UINT32) + + tid: "int" = betterproto2.field(6, betterproto2.TYPE_UINT32) + + uid: "int" = betterproto2.field(7, betterproto2.TYPE_UINT32) + + selinux_label: "str" = betterproto2.field(8, betterproto2.TYPE_STRING) + + command_line: "list[str]" = betterproto2.field( + 9, betterproto2.TYPE_STRING, repeated=True + ) + + process_uptime: "int" = betterproto2.field(20, betterproto2.TYPE_UINT32) + """ + Process uptime in seconds. + """ + + signal_info: "Signal | None" = betterproto2.field( + 10, betterproto2.TYPE_MESSAGE, optional=True + ) + + abort_message: "str" = betterproto2.field(14, betterproto2.TYPE_STRING) + + crash_details: "list[CrashDetail]" = betterproto2.field( + 21, betterproto2.TYPE_MESSAGE, repeated=True + ) + + causes: "list[Cause]" = betterproto2.field( + 15, betterproto2.TYPE_MESSAGE, repeated=True + ) + + threads: "dict[int, Thread]" = betterproto2.field( + 16, + betterproto2.TYPE_MAP, + map_meta=betterproto2.map_meta( + betterproto2.TYPE_UINT32, betterproto2.TYPE_MESSAGE + ), + ) + + guest_threads: "dict[int, Thread]" = betterproto2.field( + 25, + betterproto2.TYPE_MAP, + map_meta=betterproto2.map_meta( + betterproto2.TYPE_UINT32, betterproto2.TYPE_MESSAGE + ), + ) + + memory_mappings: "list[MemoryMapping]" = betterproto2.field( + 17, betterproto2.TYPE_MESSAGE, repeated=True + ) + + log_buffers: "list[LogBuffer]" = betterproto2.field( + 18, betterproto2.TYPE_MESSAGE, repeated=True + ) + + open_fds: "list[Fd]" = betterproto2.field( + 19, betterproto2.TYPE_MESSAGE, repeated=True + ) + + page_size: "int" = betterproto2.field(22, betterproto2.TYPE_UINT32) + + has_been_16kb_mode: "bool" = betterproto2.field(23, betterproto2.TYPE_BOOL) + + stack_history_buffer: "StackHistoryBuffer | None" = betterproto2.field( + 26, betterproto2.TYPE_MESSAGE, optional=True + ) + + +default_message_pool.register_message("", "Tombstone", Tombstone) diff --git a/src/mvt/android/parsers/proto/files_record.proto b/src/mvt/android/parsers/proto/files_record.proto new file mode 100644 index 000000000..60b5855da --- /dev/null +++ b/src/mvt/android/parsers/proto/files_record.proto @@ -0,0 +1,11 @@ +// Bugbane file metadata record, used by files.pb. +syntax = "proto3"; + +message FilesRecord { + string path = 1; + double mtime = 2; // Unix epoch seconds (optional) + string mode = 3; // e.g. "0644" (optional) + int64 size = 4; // bytes (optional) + string user = 5; // owner name (optional) + string group = 6; // group name (optional) +} diff --git a/src/mvt/android/parsers/proto/message_pool.py b/src/mvt/android/parsers/proto/message_pool.py new file mode 100644 index 000000000..cab002cf8 --- /dev/null +++ b/src/mvt/android/parsers/proto/message_pool.py @@ -0,0 +1,3 @@ +import betterproto2 + +default_message_pool = betterproto2.MessagePool() diff --git a/src/mvt/android/parsers/proto/packages_record.proto b/src/mvt/android/parsers/proto/packages_record.proto new file mode 100644 index 000000000..941aa7c32 --- /dev/null +++ b/src/mvt/android/parsers/proto/packages_record.proto @@ -0,0 +1,36 @@ +// Bugbane package inventory, used by packages.pb. +syntax = "proto3"; + +message PackageCertificateRecord { + string md5 = 1; + string sha1 = 2; + string sha256 = 3; + string valid_from = 4; + string valid_to = 5; + string issuer = 6; + string subject = 7; + string signature_algorithm = 8; + string serial_number = 9; +} + +message PackageFileRecord { + string path = 1; + string local_name = 2; + string md5 = 3; + string sha1 = 4; + string sha256 = 5; + string sha512 = 6; + bool suspicious = 7; // suspiciousness, should APK be dumped? (optional) + repeated PackageCertificateRecord certificates = 8; // certificates (optional) + repeated string infiles = 9; // files in APK (optional) +} + +message PackagesRecord { + string name = 1; + string installer = 2; + int32 uid = 3; + bool disabled = 4; + bool system = 5; + bool third_party = 6; + repeated PackageFileRecord files = 7; +} diff --git a/src/mvt/android/parsers/proto/py.typed b/src/mvt/android/parsers/proto/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/mvt/android/parsers/proto/string_record.proto b/src/mvt/android/parsers/proto/string_record.proto new file mode 100644 index 000000000..5f12be6d3 --- /dev/null +++ b/src/mvt/android/parsers/proto/string_record.proto @@ -0,0 +1,8 @@ +// Bugbane string record, used by root_binaries.pb and mounts.pb. +syntax = "proto3"; + +message StringRecord { + // Root binaries: absolute path to a su/magisk-like binary. + // Mounts: one line from `mount` output. + string value = 1; +} diff --git a/src/mvt/android/parsers/proto/tombstone.py b/src/mvt/android/parsers/proto/tombstone.py deleted file mode 100644 index f5d54e6cd..000000000 --- a/src/mvt/android/parsers/proto/tombstone.py +++ /dev/null @@ -1,207 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# sources: tombstone.proto -# plugin: python-betterproto2 -from dataclasses import dataclass - -import betterproto2 - - -class Architecture(betterproto2.Enum): - ARM32 = 0 - ARM64 = 1 - X86 = 2 - X86_64 = 3 - RISCV64 = 4 - NONE = 5 - - -class MemoryErrorTool(betterproto2.Enum): - GWP_ASAN = 0 - SCUDO = 1 - - -class MemoryErrorType(betterproto2.Enum): - UNKNOWN = 0 - USE_AFTER_FREE = 1 - DOUBLE_FREE = 2 - INVALID_FREE = 3 - BUFFER_OVERFLOW = 4 - BUFFER_UNDERFLOW = 5 - - -@dataclass(eq=False, repr=False) -class CrashDetail(betterproto2.Message): - """ - NOTE TO OEMS: If you add custom fields to this proto, do not use numbers in - the reserved range. - """ - - name: "bytes" = betterproto2.field(1, betterproto2.TYPE_BYTES) - data: "bytes" = betterproto2.field(2, betterproto2.TYPE_BYTES) - - -@dataclass(eq=False, repr=False) -class StackHistoryBufferEntry(betterproto2.Message): - addr: "BacktraceFrame | None" = betterproto2.field(1, betterproto2.TYPE_MESSAGE, optional=True) - fp: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) - tag: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) - - -@dataclass(eq=False, repr=False) -class StackHistoryBuffer(betterproto2.Message): - tid: "int" = betterproto2.field(1, betterproto2.TYPE_UINT64) - entries: "list[StackHistoryBufferEntry]" = betterproto2.field(2, betterproto2.TYPE_MESSAGE, repeated=True) - - -@dataclass(eq=False, repr=False) -class Tombstone(betterproto2.Message): - arch: "Architecture" = betterproto2.field(1, betterproto2.TYPE_ENUM, default_factory=lambda: Architecture(0)) - guest_arch: "Architecture" = betterproto2.field(24, betterproto2.TYPE_ENUM, default_factory=lambda: Architecture(0)) - build_fingerprint: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) - revision: "str" = betterproto2.field(3, betterproto2.TYPE_STRING) - timestamp: "str" = betterproto2.field(4, betterproto2.TYPE_STRING) - pid: "int" = betterproto2.field(5, betterproto2.TYPE_UINT32) - tid: "int" = betterproto2.field(6, betterproto2.TYPE_UINT32) - uid: "int" = betterproto2.field(7, betterproto2.TYPE_UINT32) - selinux_label: "str" = betterproto2.field(8, betterproto2.TYPE_STRING) - command_line: "list[str]" = betterproto2.field(9, betterproto2.TYPE_STRING, repeated=True) - # Process uptime in seconds. - process_uptime: "int" = betterproto2.field(20, betterproto2.TYPE_UINT32) - signal_info: "Signal | None" = betterproto2.field(10, betterproto2.TYPE_MESSAGE, optional=True) - abort_message: "str" = betterproto2.field(14, betterproto2.TYPE_STRING) - crash_details: "list[CrashDetail]" = betterproto2.field(21, betterproto2.TYPE_MESSAGE, repeated=True) - causes: "list[Cause]" = betterproto2.field(15, betterproto2.TYPE_MESSAGE, repeated=True) - threads: "dict[int, Thread]" = betterproto2.field( - 16, betterproto2.TYPE_MAP, map_meta=betterproto2.map_meta(betterproto2.TYPE_UINT32, betterproto2.TYPE_MESSAGE) - ) - guest_threads: "dict[int, Thread]" = betterproto2.field( - 25, betterproto2.TYPE_MAP, map_meta=betterproto2.map_meta(betterproto2.TYPE_UINT32, betterproto2.TYPE_MESSAGE) - ) - memory_mappings: "list[MemoryMapping]" = betterproto2.field(17, betterproto2.TYPE_MESSAGE, repeated=True) - log_buffers: "list[LogBuffer]" = betterproto2.field(18, betterproto2.TYPE_MESSAGE, repeated=True) - open_fds: "list[FD]" = betterproto2.field(19, betterproto2.TYPE_MESSAGE, repeated=True) - page_size: "int" = betterproto2.field(22, betterproto2.TYPE_UINT32) - has_been_16kb_mode: "bool" = betterproto2.field(23, betterproto2.TYPE_BOOL) - stack_history_buffer: "StackHistoryBuffer | None" = betterproto2.field(26, betterproto2.TYPE_MESSAGE, optional=True) - - -@dataclass(eq=False, repr=False) -class Signal(betterproto2.Message): - number: "int" = betterproto2.field(1, betterproto2.TYPE_INT32) - name: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) - code: "int" = betterproto2.field(3, betterproto2.TYPE_INT32) - code_name: "str" = betterproto2.field(4, betterproto2.TYPE_STRING) - has_sender: "bool" = betterproto2.field(5, betterproto2.TYPE_BOOL) - sender_uid: "int" = betterproto2.field(6, betterproto2.TYPE_INT32) - sender_pid: "int" = betterproto2.field(7, betterproto2.TYPE_INT32) - has_fault_address: "bool" = betterproto2.field(8, betterproto2.TYPE_BOOL) - fault_address: "int" = betterproto2.field(9, betterproto2.TYPE_UINT64) - # Note, may or may not contain the dump of the actual memory contents. - # Currently, on arm64, we only include metadata, and not the contents. - fault_adjacent_metadata: "MemoryDump | None" = betterproto2.field(10, betterproto2.TYPE_MESSAGE, optional=True) - - -@dataclass(eq=False, repr=False) -class HeapObject(betterproto2.Message): - address: "int" = betterproto2.field(1, betterproto2.TYPE_UINT64) - size: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) - allocation_tid: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) - allocation_backtrace: "list[BacktraceFrame]" = betterproto2.field(4, betterproto2.TYPE_MESSAGE, repeated=True) - deallocation_tid: "int" = betterproto2.field(5, betterproto2.TYPE_UINT64) - deallocation_backtrace: "list[BacktraceFrame]" = betterproto2.field(6, betterproto2.TYPE_MESSAGE, repeated=True) - - -@dataclass(eq=False, repr=False) -class MemoryError(betterproto2.Message): - tool: "MemoryErrorTool" = betterproto2.field(1, betterproto2.TYPE_ENUM, default_factory=lambda: MemoryErrorTool(0)) - type: "MemoryErrorType" = betterproto2.field(2, betterproto2.TYPE_ENUM, default_factory=lambda: MemoryErrorType(0)) - heap: "HeapObject | None" = betterproto2.field(3, betterproto2.TYPE_MESSAGE, optional=True, group="location") - - -@dataclass(eq=False, repr=False) -class Cause(betterproto2.Message): - human_readable: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) - memory_error: "MemoryError | None" = betterproto2.field(2, betterproto2.TYPE_MESSAGE, optional=True, group="details") - - -@dataclass(eq=False, repr=False) -class Register(betterproto2.Message): - name: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) - u64: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) - - -@dataclass(eq=False, repr=False) -class Thread(betterproto2.Message): - id: "int" = betterproto2.field(1, betterproto2.TYPE_INT32) - name: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) - registers: "list[Register]" = betterproto2.field(3, betterproto2.TYPE_MESSAGE, repeated=True) - backtrace_note: "list[str]" = betterproto2.field(7, betterproto2.TYPE_STRING, repeated=True) - unreadable_elf_files: "list[str]" = betterproto2.field(9, betterproto2.TYPE_STRING, repeated=True) - current_backtrace: "list[BacktraceFrame]" = betterproto2.field(4, betterproto2.TYPE_MESSAGE, repeated=True) - memory_dump: "list[MemoryDump]" = betterproto2.field(5, betterproto2.TYPE_MESSAGE, repeated=True) - tagged_addr_ctrl: "int" = betterproto2.field(6, betterproto2.TYPE_INT64) - pac_enabled_keys: "int" = betterproto2.field(8, betterproto2.TYPE_INT64) - - -@dataclass(eq=False, repr=False) -class BacktraceFrame(betterproto2.Message): - rel_pc: "int" = betterproto2.field(1, betterproto2.TYPE_UINT64) - pc: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) - sp: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) - function_name: "str" = betterproto2.field(4, betterproto2.TYPE_STRING) - function_offset: "int" = betterproto2.field(5, betterproto2.TYPE_UINT64) - file_name: "str" = betterproto2.field(6, betterproto2.TYPE_STRING) - file_map_offset: "int" = betterproto2.field(7, betterproto2.TYPE_UINT64) - build_id: "str" = betterproto2.field(8, betterproto2.TYPE_STRING) - - -@dataclass(eq=False, repr=False) -class ArmMTEMetadata(betterproto2.Message): - # One memory tag per granule (e.g. every 16 bytes) of regular memory. - memory_tags: "bytes" = betterproto2.field(1, betterproto2.TYPE_BYTES) - - -@dataclass(eq=False, repr=False) -class MemoryDump(betterproto2.Message): - register_name: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) - mapping_name: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) - begin_address: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) - memory: "bytes" = betterproto2.field(4, betterproto2.TYPE_BYTES) - arm_mte_metadata: "ArmMTEMetadata | None" = betterproto2.field(6, betterproto2.TYPE_MESSAGE, optional=True, group="metadata") - - -@dataclass(eq=False, repr=False) -class MemoryMapping(betterproto2.Message): - begin_address: "int" = betterproto2.field(1, betterproto2.TYPE_UINT64) - end_address: "int" = betterproto2.field(2, betterproto2.TYPE_UINT64) - offset: "int" = betterproto2.field(3, betterproto2.TYPE_UINT64) - read: "bool" = betterproto2.field(4, betterproto2.TYPE_BOOL) - write: "bool" = betterproto2.field(5, betterproto2.TYPE_BOOL) - execute: "bool" = betterproto2.field(6, betterproto2.TYPE_BOOL) - mapping_name: "str" = betterproto2.field(7, betterproto2.TYPE_STRING) - build_id: "str" = betterproto2.field(8, betterproto2.TYPE_STRING) - load_bias: "int" = betterproto2.field(9, betterproto2.TYPE_UINT64) - - -@dataclass(eq=False, repr=False) -class FD(betterproto2.Message): - fd: "int" = betterproto2.field(1, betterproto2.TYPE_INT32) - path: "str" = betterproto2.field(2, betterproto2.TYPE_STRING) - owner: "str" = betterproto2.field(3, betterproto2.TYPE_STRING) - tag: "int" = betterproto2.field(4, betterproto2.TYPE_UINT64) - - -@dataclass(eq=False, repr=False) -class LogBuffer(betterproto2.Message): - name: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) - logs: "list[LogMessage]" = betterproto2.field(2, betterproto2.TYPE_MESSAGE, repeated=True) - - -@dataclass(eq=False, repr=False) -class LogMessage(betterproto2.Message): - timestamp: "str" = betterproto2.field(1, betterproto2.TYPE_STRING) - pid: "int" = betterproto2.field(2, betterproto2.TYPE_UINT32) - tid: "int" = betterproto2.field(3, betterproto2.TYPE_UINT32) - priority: "int" = betterproto2.field(4, betterproto2.TYPE_UINT32) - tag: "str" = betterproto2.field(5, betterproto2.TYPE_STRING) - message: "str" = betterproto2.field(6, betterproto2.TYPE_STRING) diff --git a/src/mvt/android/parsers/protobuf_parsers.py b/src/mvt/android/parsers/protobuf_parsers.py new file mode 100644 index 000000000..00d694de9 --- /dev/null +++ b/src/mvt/android/parsers/protobuf_parsers.py @@ -0,0 +1,145 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Parse AndroidQF protobuf inventory artifacts into MVT result dictionaries.""" + +from __future__ import annotations + +import io +from typing import Any, TypeVar + +import betterproto2 + +from .proto import ( + FilesRecord, + PackageCertificateRecord, + PackageFileRecord, + PackagesRecord, + StringRecord, + Tombstone +) + +T = TypeVar("T", bound=betterproto2.Message) + + +def _parse_delimited_messages(data: bytes, message_type: type[T]) -> list[T]: + stream = io.BytesIO(data) + messages: list[T] = [] + while True: + if not stream.read(1): + break + stream.seek(stream.tell() - 1) + messages.append(message_type().load(stream, size=betterproto2.SIZE_DELIMITED)) + return messages + + +def parse_string_records(data: bytes) -> list[str]: + """Parse root_binaries.pb or mounts.pb into a list of string values.""" + return [record.value for record in _parse_delimited_messages(data, StringRecord)] + + +def files_record_to_dict(record: FilesRecord) -> dict[str, Any]: + """Map protobuf file metadata to the AndroidQF files.json result shape.""" + modified_time = record.mtime or 0.0 + return { + "path": record.path, + "size": record.size, + "mode": record.mode or "", + "user_id": 0, + "user_name": record.user or "", + "group_id": 0, + "group_name": record.group or "", + "changed_time": modified_time, + "modified_time": modified_time, + "access_time": modified_time, + "error": "", + "context": "", + "sha1": "", + "sha256": "", + "sha512": "", + "md5": "", + } + + +def parse_files_records(data: bytes) -> list[dict[str, Any]]: + """Parse files.pb into MVT file result dictionaries.""" + return [ + files_record_to_dict(record) + for record in _parse_delimited_messages(data, FilesRecord) + ] + + +def _certificate_to_androidqf_dict( + certificate: PackageCertificateRecord, +) -> dict[str, Any]: + cert: dict[str, Any] = {} + if certificate.md5: + cert["Md5"] = certificate.md5 + if certificate.sha1: + cert["Sha1"] = certificate.sha1 + if certificate.sha256: + cert["Sha256"] = certificate.sha256 + if certificate.valid_from: + cert["ValidFrom"] = certificate.valid_from + if certificate.valid_to: + cert["ValidTo"] = certificate.valid_to + if certificate.issuer: + cert["Issuer"] = certificate.issuer + if certificate.subject: + cert["Subject"] = certificate.subject + if certificate.signature_algorithm: + cert["SignatureAlgorithm"] = certificate.signature_algorithm + if certificate.serial_number: + try: + cert["SerialNumber"] = int(certificate.serial_number) + except ValueError: + cert["SerialNumber"] = certificate.serial_number + return cert + + +def package_file_record_to_dict(file_record: PackageFileRecord) -> dict[str, Any]: + """Map protobuf package file metadata to the AndroidQF packages.json file shape.""" + result: dict[str, Any] = { + "path": file_record.path, + "local_name": file_record.local_name or "", + "md5": file_record.md5 or "", + "sha1": file_record.sha1 or "", + "sha256": file_record.sha256 or "", + "sha512": file_record.sha512 or "", + "error": "", + "verified_certificate": False, + "certificate_error": "", + "trusted_certificate": False, + } + if file_record.certificates: + result["certificate"] = _certificate_to_androidqf_dict( + file_record.certificates[0] + ) + return result + + +def packages_record_to_dict(record: PackagesRecord) -> dict[str, Any]: + """Map protobuf package metadata to the AndroidQF packages.json result shape.""" + return { + "name": record.name, + "installer": record.installer or "null", + "uid": record.uid, + "disabled": record.disabled, + "system": record.system, + "third_party": record.third_party, + "files": [package_file_record_to_dict(file) for file in record.files], + } + + +def parse_packages_records(data: bytes) -> list[dict[str, Any]]: + """Parse packages.pb into MVT package result dictionaries.""" + return [ + packages_record_to_dict(record) + for record in _parse_delimited_messages(data, PackagesRecord) + ] + +def parse_tombstone_record(data: bytes) -> Tombstone: + """Parse tombstone.pb into a Tombstone record.""" + return Tombstone().parse(data) diff --git a/tests/android_androidqf/test_files.py b/tests/android_androidqf/test_files.py index c854473fb..9a3f5cd18 100644 --- a/tests/android_androidqf/test_files.py +++ b/tests/android_androidqf/test_files.py @@ -8,7 +8,7 @@ from mvt.android.modules.androidqf.aqf_files import AQFFiles from mvt.common.module import run_module - +from mvt.android.parsers.protobuf_parsers import parse_files_records from ..utils import get_android_androidqf, list_files @@ -23,3 +23,21 @@ def test_androidqf_files(self): assert len(m.results) == 3 assert len(m.timeline) == 6 assert len(m.alertstore.alerts) == 0 + + def test_androidqf_files_from_protobuf(self): + data_path = get_android_androidqf() + + # test protobuf parser per-se + data = (Path(data_path) / "files.pb").read_bytes() + records = parse_files_records(data) + assert len(records) == 3 + assert records[0]["path"] == "/sdcard/.profig.os" + assert records[0]["modified_time"] == 1593109532 + + # test module with protobuf file + m = AQFFiles(target_path=data_path, log=logging) + files = ["androidqf/files.pb"] + parent_path = Path(data_path).absolute().parent.as_posix() + m.from_dir(parent_path, files) + run_module(m) + assert len(m.results) == 3 diff --git a/tests/android_androidqf/test_mounts.py b/tests/android_androidqf/test_mounts.py index d66925d9c..fe08c21df 100644 --- a/tests/android_androidqf/test_mounts.py +++ b/tests/android_androidqf/test_mounts.py @@ -6,6 +6,7 @@ import logging from pathlib import Path +from mvt.android.modules.androidqf.mounts import Mounts from mvt.common.indicators import Indicator, IndicatorMatch from mvt.common.module import run_module @@ -112,11 +113,15 @@ def test_androidqf_module_no_mounts_file(self): When no `mounts.json` is present in the androidqf dataset, the module should not produce results nor detections. """ - from mvt.android.modules.androidqf.mounts import Mounts data_path = get_android_androidqf() m = Mounts(target_path=data_path, log=logging) - files = list_files(data_path) + # filter out valid mount files for the test + files = [ + path + for path in list_files(data_path) + if not path.endswith("mounts.json") and not path.endswith("mounts.pb") + ] parent_path = Path(data_path).absolute().parent.as_posix() m.from_dir(parent_path, files) @@ -130,3 +135,43 @@ def test_androidqf_module_no_mounts_file(self): assert len(m.alertstore.alerts) == 0, ( f"Expected no detections, got: {m.alertstore.alerts}" ) + + def test_androidqf_mounts_json(self): + data_path = get_android_androidqf() + m = Mounts(target_path=data_path, log=logging) + files = ["androidqf/mounts.json"] + parent_path = Path(data_path).absolute().parent.as_posix() + m.from_dir(parent_path, files) + run_module(m) + + assert len(m.results) == 3 + assert m.results[0]["device"] == "/dev/block/dm-12" + assert m.results[0]["mount_point"] == "/" + assert m.results[0]["filesystem_type"] == "ext4" + assert m.results[0]["mount_options"] == "ro,seclabel,noatime" + + def test_androidqf_mounts_pb(self): + data_path = get_android_androidqf() + m = Mounts(target_path=data_path, log=logging) + files = ["androidqf/mounts.pb"] + parent_path = Path(data_path).absolute().parent.as_posix() + m.from_dir(parent_path, files) + run_module(m) + + assert len(m.results) == 4 + assert m.results[0]["device"] == "/dev/block/dm-12" + assert m.results[0]["mount_point"] == "/" + assert m.results[0]["filesystem_type"] == "ext4" + assert m.results[0]["mount_options"] == "ro,seclabel,noatime" + + def test_androidqf_mounts_json_priority_over_pb(self): + data_path = get_android_androidqf() + m = Mounts(target_path=data_path, log=logging) + files = ["androidqf/mounts.json", "androidqf/mounts.pb"] + parent_path = Path(data_path).absolute().parent.as_posix() + m.from_dir(parent_path, files) + run_module(m) + + # We expect 3 results since the JSON should have priority over the PB. + assert len(m.results) == 3 + assert "/vendor" not in [result["mount_point"] for result in m.results] diff --git a/tests/android_androidqf/test_packages.py b/tests/android_androidqf/test_packages.py index a879e8805..b25ab9d03 100644 --- a/tests/android_androidqf/test_packages.py +++ b/tests/android_androidqf/test_packages.py @@ -13,7 +13,7 @@ from mvt.android.modules.androidqf.aqf_packages import AQFPackages from mvt.android.modules.androidqf import aqf_packages as aqf_packages_module from mvt.common.module import run_module - +from mvt.android.parsers.protobuf_parsers import parse_packages_records from ..utils import get_android_androidqf, list_files @@ -47,6 +47,26 @@ def test_packages_list(self, module): assert len(module.results) == 7 assert len(module.timeline) == 0 + def test_packages_from_protobuf(self): + data_path = get_android_androidqf() + + # test protobuf parser per-se + data = (Path(data_path) / "packages.pb").read_bytes() + records = parse_packages_records(data) + assert len(records) == 7 + assert records[0]["name"] == "com.whatsapp" + assert records[0]["files"][0]["certificate"]["Md5"] == "556c6019249bbc0cab70495178d3a9d1" + + # test module with protobuf file + m = AQFPackages(target_path=data_path, log=logging) + files = ["androidqf/packages.pb"] + parent_path = Path(data_path).absolute().parent.as_posix() + m.from_dir(parent_path, files) + run_module(m) + assert len(m.results) == 7 + assert m.results[0]["name"] == "com.whatsapp" + assert m.results[0]["files"][0]["certificate"]["Md5"] == "556c6019249bbc0cab70495178d3a9d1" + def test_non_appstore_warnings(self, caplog, module): run_module(module) diff --git a/tests/android_androidqf/test_root_binaries.py b/tests/android_androidqf/test_root_binaries.py index b014f580b..b6d2e93ee 100644 --- a/tests/android_androidqf/test_root_binaries.py +++ b/tests/android_androidqf/test_root_binaries.py @@ -114,3 +114,16 @@ def test_no_root_binaries_file(self, parent_data_path): assert len(m.results) == 0 assert len(m.alertstore.alerts) == 0 + + def test_root_binaries_from_protobuf(self, parent_data_path): + module = RootBinaries(target_path=parent_data_path, log=logging) + module.from_dir(parent_data_path, ["androidqf/root_binaries.pb"]) + run_module(module) + assert len(module.results) == 5 + + def test_root_binaries_json_priority_over_protobuf(self, parent_data_path): + module = RootBinaries(target_path=parent_data_path, log=logging) + module.from_dir(parent_data_path, ["androidqf/root_binaries.json", "androidqf/root_binaries.pb"]) + run_module(module) + assert len(module.results) == 4 + assert "/system/bin/magisk" not in [result["path"] for result in module.results] diff --git a/tests/artifacts/androidqf/files.pb b/tests/artifacts/androidqf/files.pb new file mode 100644 index 000000000..bd88cf8fe Binary files /dev/null and b/tests/artifacts/androidqf/files.pb differ diff --git a/tests/artifacts/androidqf/mounts.json b/tests/artifacts/androidqf/mounts.json new file mode 100644 index 000000000..78adc86e7 --- /dev/null +++ b/tests/artifacts/androidqf/mounts.json @@ -0,0 +1,5 @@ +[ + "/dev/block/dm-12 on / type ext4 (ro,seclabel,noatime)", + "/dev/block/dm-13 on /system type ext4 (ro,seclabel,noatime)", + "/dev/block/dm-14 on /data type ext4 (rw,seclabel,noatime)" +] diff --git a/tests/artifacts/androidqf/mounts.pb b/tests/artifacts/androidqf/mounts.pb new file mode 100644 index 000000000..153d7a7c7 --- /dev/null +++ b/tests/artifacts/androidqf/mounts.pb @@ -0,0 +1,5 @@ +7 +5/dev/block/dm-12 on / type ext4 (ro,seclabel,noatime)= +;/dev/block/dm-13 on /system type ext4 (ro,seclabel,noatime); +9/dev/block/dm-14 on /data type ext4 (rw,seclabel,noatime)= +;/dev/block/dm-15 on /vendor type ext4 (ro,seclabel,noatime) \ No newline at end of file diff --git a/tests/artifacts/androidqf/packages.pb b/tests/artifacts/androidqf/packages.pb new file mode 100644 index 000000000..fcaadc08e Binary files /dev/null and b/tests/artifacts/androidqf/packages.pb differ diff --git a/tests/artifacts/androidqf/root_binaries.pb b/tests/artifacts/androidqf/root_binaries.pb new file mode 100644 index 000000000..696aefeff --- /dev/null +++ b/tests/artifacts/androidqf/root_binaries.pb @@ -0,0 +1,6 @@ + +/system/bin/su +/system/xbin/busybox +/data/local/tmp/magisk +/system/bin/magiskhide +/system/bin/magisk \ No newline at end of file diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py index 4dbe5c0a4..6884708e8 100644 --- a/tests/common/test_utils.py +++ b/tests/common/test_utils.py @@ -62,7 +62,7 @@ def test_hash_from_file(self): def test_hash_from_folder(self): path = os.path.join(get_artifact_folder(), "androidqf") hashes = list(generate_hashes_from_path(path, logging)) - assert len(hashes) == 8 + assert len(hashes) == 13 # Sort the files to have reliable order for tests. hashes = sorted(hashes, key=lambda x: x["file_path"]) assert hashes[0]["file_path"] == os.path.join(path, "backup.ab")