[backport/1.4] Keep Ping port watcher running after unexpected errors

core/services/ping/portwatcher.py

@@ -79,20 +79,23 @@ async def start_watching(self) -> None:
         """Start watching for plugged/unplugged serial devices in the system."""
         # TODO: try https://pypi.org/project/inotify/
         while True:
-            ports = serial.tools.list_ports.comports()
-            ports_description = [f"{port.subsystem}:{port.name}" for port in ports]
-            logger.debug(f"Currently detected ports: {ports_description}")
-            found_ports = set()
-            for port in ports:
-                if self.port_should_be_probed(port):
-                    await self.probe_port(port)
-                found_ports.add(port)
+            try:
+                ports = serial.tools.list_ports.comports()
+                ports_description = [f"{port.subsystem}:{port.name}" for port in ports]
+                logger.debug(f"Currently detected ports: {ports_description}")
+                found_ports = set()
+                for port in ports:
+                    if self.port_should_be_probed(port):
+                        await self.probe_port(port)
+                    found_ports.add(port)
 
-            missing = self.known_ports - found_ports
-            for port in missing:
-                logger.info(f"Port lost: {port.hwid}")
-                self.known_ports.remove(port)
-                if self.port_lost_callback is not None:
-                    self.port_lost_callback(port)
-            await self.add_ping360()
+                missing = self.known_ports - found_ports
+                for port in missing:
+                    logger.info(f"Port lost: {port.hwid}")
+                    self.known_ports.remove(port)
+                    if self.port_lost_callback is not None:
+                        self.port_lost_callback(port)
+                await self.add_ping360()
+            except Exception as error:
+                logger.exception(f"Error while watching ports/devices: {error}")
             await asyncio.sleep(1)

core/services/ping/setup.py

@@ -24,6 +24,7 @@
         "anyio == 3.7.1",
         "fastapi-versioning == 0.9.1",
         "loguru == 0.5.3",
+        "psutil == 5.7.2",
         "pyserial == 3.5",
         "starlette == 0.27.0",
         "uvicorn == 0.13.4",

core/services/ping/test_portwatcher.py

@@ -0,0 +1,39 @@
+import asyncio
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+_PING_DIR = str(Path(__file__).resolve().parent)
+sys.path.insert(0, _PING_DIR)
+# Root pytest also finds other services' top-level modules (e.g. ardupilot_manager/exceptions.py).
+for _name in ("exceptions", "pingutils", "ping360_ethernet_prober", "portwatcher"):
+    _mod = sys.modules.get(_name)
+    if _mod is not None and not str(getattr(_mod, "__file__", "")).startswith(_PING_DIR):
+        del sys.modules[_name]
+
+import pytest
+
+from portwatcher import PortWatcher
+
+
+@pytest.mark.asyncio
+async def test_start_watching_continues_after_add_ping360_error() -> None:
+    watcher = PortWatcher(probe_callback=AsyncMock(), found_callback=AsyncMock())
+    calls = 0
+
+    async def flaky_add_ping360() -> None:
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise KeyError("uap0")
+        raise asyncio.CancelledError()
+
+    with (
+        patch.object(watcher, "add_ping360", side_effect=flaky_add_ping360),
+        patch("portwatcher.serial.tools.list_ports.comports", return_value=[]),
+        patch("portwatcher.asyncio.sleep", new_callable=AsyncMock),
+    ):
+        with pytest.raises(asyncio.CancelledError):
+            await watcher.start_watching()
+
+    assert calls == 2

pyproject.toml

@@ -24,6 +24,7 @@ pytest = "7.4.2"
 coverage = "7.5.1"
 pytest-cov = "4.1.0"
 pydantic = "1.10.12"
+psutil = "5.7.2"
 docker = "6.1.3"
 aiodocker = "0.21.0"
 aiohttp = "3.9.5"