web

selfdrive/carrot/carrot_man.py

@@ -52,7 +52,9 @@
 NAVI_IMAGE_BASE64_MAX_CHARS = 6 * 1024 * 1024
 NAVI_ROUTE_MAX_POINTS = 4096
 NAVI_ROUTE_SUMMARY_MAX_SCAN = 20000
-AUTO_ONROAD_DIAGNOSTICS = os.environ.get("CARROT_AUTO_ONROAD_DIAGNOSTICS", "0").strip().lower() in ("1", "true", "yes", "on")
+AUTO_ONROAD_DIAGNOSTICS = os.environ.get("CARROT_AUTO_ONROAD_DIAGNOSTICS", "1").strip().lower() in ("1", "true", "yes", "on")
+AUTO_ONROAD_TMUX_DELAY_SECONDS = float(os.environ.get("CARROT_AUTO_ONROAD_TMUX_DELAY_SECONDS", "60"))
+CARROT_EXCEPTION_UPLOAD_RETRY_SECONDS = 60.0
 
 
 def limit_route_points(points, max_points=NAVI_ROUTE_MAX_POINTS):
@@ -73,6 +75,40 @@ def limit_route_points(points, max_points=NAVI_ROUTE_MAX_POINTS):
         previous_index = source_index
     return limited
 
+_carrot_exception_tmux_send_lock = threading.Lock()
+_carrot_exception_tmux_send_queued = False
+
+
+def reset_carrot_exception_tmux_send_queue() -> None:
+  global _carrot_exception_tmux_send_queued
+
+  with _carrot_exception_tmux_send_lock:
+    _carrot_exception_tmux_send_queued = False
+
+
+def queue_carrot_exception_tmux_send(context: str = "") -> None:
+  global _carrot_exception_tmux_send_queued
+
+  with _carrot_exception_tmux_send_lock:
+    if _carrot_exception_tmux_send_queued:
+      return
+
+    try:
+      params = Params()
+      current = params.get("CarrotException")
+      if current in (None, "", b""):
+        put_nonblocking = getattr(params, "put_nonblocking", None)
+        if callable(put_nonblocking):
+          put_nonblocking("CarrotException", "tmux_send")
+        else:
+          params.put("CarrotException", "tmux_send")
+        _carrot_exception_tmux_send_queued = True
+        print(f"[carrot_man] CarrotException tmux_send queued: {context or 'exception'}")
+      elif current == "tmux_send":
+        _carrot_exception_tmux_send_queued = True
+    except Exception as e:
+      print(f"[carrot_man] failed to queue CarrotException tmux_send: {e}")
+
 ################ CarrotNavi
 ## 국가법령정보센터: 도로설계기준
 #V_CURVE_LOOKUP_BP = [0., 1./800., 1./670., 1./560., 1./440., 1./360., 1./265., 1./190., 1./135., 1./85., 1./55., 1./30., 1./15.]
@@ -362,12 +398,14 @@ def broadcast_version_info(self):
             self.connection = None
             print(f"##### broadcast_error...: {e}")
             traceback.print_exc()
+            queue_carrot_exception_tmux_send("broadcast_version_info")
 
         rk.keep_time()
         frame += 1
       except Exception as e:
         print(f"broadcast_version_info error...: {e}")
         traceback.print_exc()
+        queue_carrot_exception_tmux_send("broadcast_version_info")
         time.sleep(1)
 
 
@@ -641,6 +679,7 @@ def kisa_app_thread(self):
                   #print(json_obj)
                 except Exception as e:
                   traceback.print_exc()
+                  queue_carrot_exception_tmux_send("kisa_app_thread")
                   print(f"kisa_app_thread: json error...: {e}")
                   print(data)
 
@@ -667,58 +706,64 @@ def kisa_app_thread(self):
 
   def make_tmux_data(self):
     try:
-      subprocess.run("rm /data/media/tmux.log; tmux capture-pane -pq -S-1000 > /data/media/tmux.log", shell=True, capture_output=True, text=False)
+      subprocess.run("rm -f /data/media/tmux.log; tmux capture-pane -pq -S-1000 > /data/media/tmux.log", shell=True, capture_output=True, text=False, check=True)
       subprocess.run("/data/openpilot/selfdrive/apilot.py", shell=True, capture_output=True, text=False)
+      return True
     except Exception as e:
       print(f"TMUX creation error: {e}")
-      return
+      return False
 
   def send_tmux(self, ftp_password, tmux_why, send_settings=False):
     ftp_server = "shind0.synology.me"
     ftp_port = 8021
     ftp_username = "carrotpilot"
-    ftp = FTP()
-    ftp.connect(ftp_server, ftp_port)
-    ftp.login(ftp_username, ftp_password)
-    car_selected = Params().get("CarName")
-    if car_selected is None:
-      car_selected = "none"
-    else:
-      car_selected = car_selected
-
-    git_branch = Params().get("GitBranch").replace("/", "__")
+    ftp = FTP(timeout=10)
     try:
-      ftp.mkd(git_branch)
-    except Exception as e:
-      print(f"Directory creation failed: {e}")
-    ftp.cwd(git_branch)
+      ftp.connect(ftp_server, ftp_port, timeout=10)
+      ftp.login(ftp_username, ftp_password)
+      car_selected = Params().get("CarName") or "none"
 
-    directory = car_selected + " " + Params().get("DongleId")
-    current_time = datetime.now().strftime("%Y%m%d-%H%M%S")
-    filename = tmux_why + "-" + current_time + "-" + git_branch + ".txt"
+      git_branch = (Params().get("GitBranch") or "unknown").replace("/", "__")
+      try:
+        ftp.mkd(git_branch)
+      except Exception as e:
+        print(f"Directory creation failed: {e}")
+      ftp.cwd(git_branch)
 
-    try:
-      ftp.mkd(directory)
-    except Exception as e:
-      print(f"Directory creation failed: {e}")
-    ftp.cwd(directory)
+      directory = car_selected + " " + (Params().get("DongleId") or "unknown")
+      current_time = datetime.now().strftime("%Y%m%d-%H%M%S")
+      filename = tmux_why + "-" + current_time + "-" + git_branch + ".txt"
+
+      try:
+        ftp.mkd(directory)
+      except Exception as e:
+        print(f"Directory creation failed: {e}")
+      ftp.cwd(directory)
 
-    try:
       with open("/data/media/tmux.log", "rb") as file:
         ftp.storbinary(f'STOR {filename}', file)
-    except Exception as e:
-      print(f"ftp sending error...: {e}")
 
-    if send_settings:
-      self.save_toggle_values()
+      if send_settings:
+        self.save_toggle_values()
+        try:
+          #with open("/data/backup_params.json", "rb") as file:
+          with open("/data/toggle_values.json", "rb") as file:
+            ftp.storbinary(f'STOR toggles-{current_time}.json', file)
+        except Exception as e:
+          print(f"ftp params sending error...: {e}")
+      return True
+    except Exception as e:
+      print(f"ftp tmux sending error...: {e}")
+      traceback.print_exc()
+      return False
+    finally:
       try:
-        #with open("/data/backup_params.json", "rb") as file:
-        with open("/data/toggle_values.json", "rb") as file:
-          ftp.storbinary(f'STOR toggles-{current_time}.json', file)
-      except Exception as e:
-        print(f"ftp params sending error...: {e}")
-
-    ftp.quit()
+        ftp.quit()
+      except Exception:
+        try:
+          ftp.close()
+        except Exception:
+          pass
 
   def send_tmux_http(self, tmux_why, send_settings=False):
     def get_private_ip_by_iface(name="wlan0"):
@@ -752,18 +797,21 @@ def _pstr(key):
       "local_ip"          : get_private_ip_by_iface("wlan0"),
     }
 
-    files = [
-        ("files[0]", ("tmux.log", open("/data/media/tmux.log", "rb"), "text/plain")),
-    ]
-
-    if send_settings:
-      #self.save_toggle_values()
-      files.append(("files[1]",("toggle_values.json",open("/data/toggle_values.json", "rb"),"application/json")))
-
     params = {}
     headers = {}
+    files = []
 
     try:
+      files.append(("files[0]", ("tmux.log", open("/data/media/tmux.log", "rb"), "text/plain")))
+
+      if send_settings:
+        #self.save_toggle_values()
+        self.save_toggle_values()
+        try:
+          files.append(("files[1]",("toggle_values.json",open("/data/toggle_values.json", "rb"),"application/json")))
+        except Exception as e:
+          print(f"http params file open error...: {e}")
+
       response = requests.post(
           url,
           params=params,
@@ -774,6 +822,10 @@ def _pstr(key):
       )
       print(response.status_code, response.text)
       return response
+    except Exception as e:
+      print(f"http tmux sending error...: {e}")
+      traceback.print_exc()
+      return None
     finally:
       for _, fileinfo in files:
         fileobj = fileinfo[1]
@@ -864,10 +916,16 @@ def setup_socket():
     socket, poller = setup_socket()
     isOnroadCount = 0
     is_tmux_sent = False
+    onroad_start_at = None
+    onroad_tmux_captured = False
+    onroad_tmux_next_attempt_at = 0.0
+    pending_tmux_reason = None
+    pending_tmux_next_attempt_at = 0.0
 
     print("#########carrot_cmd_zmq: thread started...")
     while True:
       try:
+        now = time.monotonic()
         socks = dict(poller.poll(100))
 
         if socket in socks and socks[socket] == zmq.POLLIN:
@@ -878,27 +936,69 @@ def setup_socket():
           json_obj = None
 
         if json_obj is None:
-          isOnroadCount = isOnroadCount + 1 if self.params.get_bool("IsOnroad") else 0
-          if isOnroadCount == 0:
+          is_onroad = self.params.get_bool("IsOnroad")
+          if is_onroad:
+            if onroad_start_at is None:
+              onroad_start_at = now
+              isOnroadCount = 1
+              is_tmux_sent = False
+              onroad_tmux_captured = False
+              onroad_tmux_next_attempt_at = 0.0
+              if AUTO_ONROAD_DIAGNOSTICS:
+                self.show_panda_debug = True
+            else:
+              isOnroadCount += 1
+          else:
+            isOnroadCount = 0
+            onroad_start_at = None
             is_tmux_sent = False
-          if AUTO_ONROAD_DIAGNOSTICS and isOnroadCount == 1:
-            self.show_panda_debug = True
+            onroad_tmux_captured = False
+            onroad_tmux_next_attempt_at = 0.0
 
           network_type = self.sm['deviceState'].networkType # if not force_wifi else NetworkType.wifi
           networkConnected = False if network_type == NetworkType.none else True
 
-          if AUTO_ONROAD_DIAGNOSTICS and isOnroadCount == 500:
-            self.make_tmux_data()
-          if AUTO_ONROAD_DIAGNOSTICS and isOnroadCount > 500 and not is_tmux_sent and networkConnected:
-            self.send_tmux("Ekdrmsvkdlffjt7710", "onroad", send_settings = True)
-            self.send_tmux_http("onroad", send_settings = True)
-            is_tmux_sent = True
+          if AUTO_ONROAD_DIAGNOSTICS and onroad_start_at is not None and not is_tmux_sent:
+            onroad_elapsed = now - onroad_start_at
+            if not onroad_tmux_captured and onroad_elapsed >= AUTO_ONROAD_TMUX_DELAY_SECONDS and now >= onroad_tmux_next_attempt_at:
+              if self.make_tmux_data():
+                onroad_tmux_captured = True
+                onroad_tmux_next_attempt_at = 0.0
+                print(f"[carrot_man] onroad tmux captured after {onroad_elapsed:.1f}s; waiting for network upload")
+              else:
+                onroad_tmux_next_attempt_at = now + CARROT_EXCEPTION_UPLOAD_RETRY_SECONDS
+
+            if onroad_tmux_captured and networkConnected and now >= onroad_tmux_next_attempt_at:
+              ftp_ok = self.send_tmux("Ekdrmsvkdlffjt7710", "onroad", send_settings = True)
+              http_response = self.send_tmux_http("onroad", send_settings = True)
+              http_ok = http_response is not None and getattr(http_response, "ok", False)
+              if ftp_ok or http_ok:
+                print(f"[carrot_man] onroad tmux upload complete: ftp_ok={ftp_ok}, http_ok={http_ok}")
+                is_tmux_sent = True
+              else:
+                onroad_tmux_next_attempt_at = now + CARROT_EXCEPTION_UPLOAD_RETRY_SECONDS
           carrot_exception = self.params.get("CarrotException")
-          if carrot_exception in ["exception", "log", "tmux_send"] and networkConnected:
-            self.params.put("CarrotException", "")
-            self.make_tmux_data()
-            self.send_tmux("Ekdrmsvkdlffjt7710", carrot_exception)
-            self.send_tmux_http(carrot_exception, send_settings = False)
+          if carrot_exception in ["exception", "log", "tmux_send"] and pending_tmux_reason is None and now >= pending_tmux_next_attempt_at:
+            if self.make_tmux_data():
+              pending_tmux_reason = carrot_exception
+              pending_tmux_next_attempt_at = 0.0
+              print(f"[carrot_man] tmux captured for {carrot_exception}; waiting for network upload")
+            else:
+              pending_tmux_next_attempt_at = now + CARROT_EXCEPTION_UPLOAD_RETRY_SECONDS
+              reset_carrot_exception_tmux_send_queue()
+
+          if pending_tmux_reason is not None and networkConnected and now >= pending_tmux_next_attempt_at:
+            ftp_ok = self.send_tmux("Ekdrmsvkdlffjt7710", pending_tmux_reason)
+            http_response = self.send_tmux_http(pending_tmux_reason, send_settings = False)
+            http_ok = http_response is not None and getattr(http_response, "ok", False)
+            if ftp_ok or http_ok:
+              print(f"[carrot_man] tmux upload complete for {pending_tmux_reason}: ftp_ok={ftp_ok}, http_ok={http_ok}")
+              self.params.put("CarrotException", "")
+              pending_tmux_reason = None
+              pending_tmux_next_attempt_at = 0.0
+              reset_carrot_exception_tmux_send_queue()
+            else:
+              pending_tmux_next_attempt_at = now + CARROT_EXCEPTION_UPLOAD_RETRY_SECONDS
         elif 'echo_cmd' in json_obj:
           try:
             result = subprocess.run(json_obj['echo_cmd'], shell=True, capture_output=True, text=False)
@@ -916,10 +1016,12 @@ def setup_socket():
           #print(echo)
           socket.send(echo.encode())
         elif 'tmux_send' in json_obj:
-          self.make_tmux_data()
-          self.send_tmux(json_obj['tmux_send'], "tmux_send")
-          self.send_tmux_http("tmux_send")
-          echo = json.dumps({"tmux_send": json_obj['tmux_send'], "result": "success"})
+          tmux_created = self.make_tmux_data()
+          ftp_ok = self.send_tmux(json_obj['tmux_send'], "tmux_send") if tmux_created else False
+          http_response = self.send_tmux_http("tmux_send") if tmux_created else None
+          http_ok = http_response is not None and getattr(http_response, "ok", False)
+          result = "success" if ftp_ok or http_ok else "failed"
+          echo = json.dumps({"tmux_send": json_obj['tmux_send'], "result": result, "ftp_ok": ftp_ok, "http_ok": http_ok})
           socket.send(echo.encode())
       except Exception as e:
         print(f"carrot_cmd_zmq error: {e}")
@@ -1810,6 +1912,7 @@ def _safe_dispatch_handler(self, label: str, handler: Any, *args: Any):
     except Exception as e:
       print(f"navi {label} handler error: {e}")
       traceback.print_exc()
+      queue_carrot_exception_tmux_send(f"navi {label} handler")
       return None
 
   def carrot_navi_http_thread(self):
@@ -1819,6 +1922,7 @@ def carrot_navi_http_thread(self):
       except Exception as e:
         print(f"navi http server error: {e}")
         traceback.print_exc()
+        queue_carrot_exception_tmux_send("navi http server")
         time.sleep(2)
 
   def carrot_navi_tcp_server(self, port: int = 7712):
@@ -1911,6 +2015,7 @@ async def carrot_http_post(self, request: web.Request):
     except Exception as e:
       print(f"[HTTP] dispatch error: {e}")
       traceback.print_exc()
+      queue_carrot_exception_tmux_send("navi http dispatch")
       return web.json_response({
         "ok": False,
         "error": str(e),
@@ -1975,6 +2080,7 @@ def main():
     except Exception as e:
       print(f"carrot_man error...: {e}")
       traceback.print_exc()
+      queue_carrot_exception_tmux_send("carrot_man_thread")
       time.sleep(10)
 
 

selfdrive/carrot/server/features/params.py

@@ -87,7 +87,7 @@ async def api_param_set(request: web.Request) -> web.Response:
     pass
 
   try:
-    set_param_value(name, value)
+    set_param_value(name, value, p)
     return web.json_response({"ok": True, "name": name, "value": value, "has_params": HAS_PARAMS})
   except Exception as e:
     return web.json_response({"ok": False, "error": str(e)}, status=500)