"""Uploads locally-queued trip/GPS tracking rows and dashcam files to the remote server (see server/ in this repo) over HTTP, with retry/backoff. Tracking-log uploads always run when configured. Dashcam file uploads can be paused independently via a persisted toggle (settings table in the same SQLite db used for tracking) so a user can stop shipping large video files over a slow/metered connection while trip logs keep flowing. Network path: normal WiFi/Ethernet (via `requests`) is always tried first for everything. If a tracking-batch upload fails with a connection-level error (no route/DNS/timeout - not an HTTP error status, which means the server WAS reached) and the SIM868 cellular fallback is configured, the same small JSON batch is retried over the modem's AT-command HTTP client (see modules/app_cellular.py). Dashcam uploads never use cellular - only WiFi/Ethernet - since video files are far too large/slow for a GPRS/3G link; they simply wait for connectivity to return, same as before. *** The cellular fallback path is UNTESTED (no SIM card available) - see *** modules/app_cellular.py's docstring. It's opt-in/feature-flagged off by *** default (YARIS_CELLULAR_ENABLED unset) so this has zero effect on the *** already-verified WiFi upload path unless explicitly configured. Credentials/server URL are loaded from db/upload_creds.env (gitignored, never commit real values - mirrors the db/creds.env pattern already used by app_db.py for MongoDB). """ import hashlib import os import time from threading import Thread import requests from dotenv import load_dotenv from modules import app_sql, app_dash, app_cellular _BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) load_dotenv(os.path.join(_BASE_DIR, "db", "upload_creds.env")) SERVER_URL = os.getenv("YARIS_SERVER_URL", "").rstrip("/") DEVICE_NAME = os.getenv("YARIS_DEVICE_NAME", "") API_KEY = os.getenv("YARIS_API_KEY", "") UPLOAD_INTERVAL = 30 # seconds between upload attempts once caught up CATCHUP_INTERVAL = 1 # seconds between batches while draining a backlog TRACKING_BATCH_SIZE = 500 BACKOFF_INITIAL = 5 # seconds, doubles on each consecutive failure BACKOFF_MAX = 300 DASHCAM_UPLOAD_SETTING_KEY = "dashcam_upload_enabled" _upload_thread = None _last_error = None _last_success_at = None _last_network_path = None # "wifi" or "cellular" - last path a batch actually went out on def _headers(): return {"X-Device-Name": DEVICE_NAME, "X-Api-Key": API_KEY} def is_configured(): return bool(SERVER_URL and DEVICE_NAME and API_KEY) def is_dashcam_upload_enabled(): """Defaults to enabled - the user opts out, not in.""" return app_sql.get_setting(DASHCAM_UPLOAD_SETTING_KEY, "1") == "1" def set_dashcam_upload_enabled(enabled): app_sql.set_setting(DASHCAM_UPLOAD_SETTING_KEY, "1" if enabled else "0") def get_status(): """Snapshot used by the /api/upload/status endpoint for the web UI.""" return { "configured": is_configured(), "dashcam_upload_enabled": is_dashcam_upload_enabled(), "pending_tracking_rows": app_sql.count_unsent_tracking_rows(), "last_error": _last_error, "last_success_at": _last_success_at, "last_network_path": _last_network_path, "cellular_fallback_configured": app_cellular.is_configured(), } def _sha256_file(path): hasher = hashlib.sha256() with open(path, "rb") as f: while True: chunk = f.read(1024 * 1024) if not chunk: break hasher.update(chunk) return hasher.hexdigest() def _upload_tracking_batch(): """Send up to TRACKING_BATCH_SIZE not-yet-uploaded rows. Returns count sent. Tries WiFi/Ethernet (`requests`) first. If that fails at the connection level (no route, DNS failure, timeout - as opposed to the server responding with an HTTP error, which means it WAS reached) and the SIM868 cellular fallback is configured, retries the same batch over the modem's AT-command HTTP client before giving up. """ global _last_network_path rows = app_sql.get_unsent_tracking_rows(limit=TRACKING_BATCH_SIZE) if not rows: return 0 points = [ { "recorded_at": row["datetime"], "gps_lat": row["gpsx"], "gps_lon": row["gpsy"], "speed": row["speed"], "odometer": row["odometer"], "vehicle_name": row["vn"], "driver_name": row["name"], } for row in rows ] row_ids = [row["id"] for row in rows] try: resp = requests.post( f"{SERVER_URL}/api/v1/tracking/batch", json={"points": points}, headers=_headers(), timeout=15, ) resp.raise_for_status() _last_network_path = "wifi" except requests.exceptions.ConnectionError as wifi_err: if not app_cellular.is_configured(): raise print(f"WiFi/Ethernet upload unreachable ({wifi_err}), trying cellular fallback...") import json as _json body_bytes = _json.dumps({"points": points}).encode() try: status_code, _body = app_cellular.http_post_json( f"{SERVER_URL}/api/v1/tracking/batch", body_bytes, _headers() ) except app_cellular.CellularError as cell_err: # Neither path worked - raise the original WiFi error so the # normal backoff/retry logic in _upload_loop handles it. print(f"Cellular fallback also failed: {cell_err}") raise wifi_err if status_code >= 400: raise app_cellular.CellularError( f"cellular upload got HTTP {status_code}" ) _last_network_path = "cellular" app_sql.mark_tracking_rows_uploaded(row_ids) return len(rows) def _upload_dashcam_files(): """Upload any not-yet-uploaded dashcam files, one at a time. Returns count sent. Skipped entirely (returns 0) while the dashcam-upload toggle is off - tracking-log uploads are unaffected by this toggle. """ if not is_dashcam_upload_enabled(): return 0 uploaded_count = 0 for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION): sha256 = _sha256_file(abs_path) if app_sql.dashcam_file_marked_uploaded(rel_path, sha256): continue with open(abs_path, "rb") as f: resp = requests.post( f"{SERVER_URL}/api/v1/dashcam/upload", data={"relative_path": rel_path, "sha256": sha256}, files={"file": f}, headers=_headers(), timeout=300, ) resp.raise_for_status() app_sql.mark_dashcam_file_uploaded(rel_path, sha256) uploaded_count += 1 # Re-check the toggle between files so flipping it off mid-run stops # further dashcam uploads promptly without waiting for the next cycle. if not is_dashcam_upload_enabled(): break return uploaded_count def _upload_loop(): global _last_error, _last_success_at import datetime as _dt backoff = BACKOFF_INITIAL while True: if not is_configured(): time.sleep(UPLOAD_INTERVAL) continue try: sent = _upload_tracking_batch() uploaded = _upload_dashcam_files() if sent or uploaded: print(f"Upload: sent {sent} tracking rows, {uploaded} dashcam files") _last_error = None _last_success_at = _dt.datetime.now().isoformat() backoff = BACKOFF_INITIAL # A full batch means there's likely more backlog queued up (the # tracking table logs a new row every 5s, so a stale/offline # device can build up thousands of rows quickly) - keep draining # back-to-back instead of idling for the full interval. Only # settle into the slow/idle cadence once a batch comes back # short, i.e. we're actually caught up. if sent >= TRACKING_BATCH_SIZE: time.sleep(CATCHUP_INTERVAL) else: time.sleep(UPLOAD_INTERVAL) except (requests.RequestException, app_cellular.CellularError) as e: _last_error = str(e) print(f"Upload failed ({e}), retrying in {backoff}s") time.sleep(backoff) backoff = min(backoff * 2, BACKOFF_MAX) def start_uploading(): """Start the background upload thread, if the server is configured.""" global _upload_thread if not is_configured(): print( "Upload client not configured (set YARIS_SERVER_URL/YARIS_DEVICE_NAME/" "YARIS_API_KEY in db/upload_creds.env) - uploads disabled." ) return if _upload_thread is None or not _upload_thread.is_alive(): _upload_thread = Thread(target=_upload_loop, daemon=True) _upload_thread.start() print("Upload-client thread started.") else: print("Upload-client thread already running.")