From 975526667e529e6189e69beacdd59a84f261ce25 Mon Sep 17 00:00:00 2001 From: Mathieu Moran Date: Sat, 8 Aug 2026 20:16:31 -0400 Subject: [PATCH] Optimize backlog upload speed for tracking rows The upload loop previously always slept the full 30s UPLOAD_INTERVAL between batches, even when a full batch was just sent (meaning more rows were likely still queued). With GPS/OBD rows logged every 5s locally, an offline/stale device could build up a large backlog that took a very long time to drain at 100 rows/30s. - Bump TRACKING_BATCH_SIZE 100 -> 500 (cuts round-trips per backlog). - Only sleep the full UPLOAD_INTERVAL once actually caught up (a short batch came back); while a full batch keeps coming back, drain back-to-back with a short CATCHUP_INTERVAL (1s) instead. - server/app.py: add a sane MAX_TRACKING_BATCH_POINTS (5000) cap on the tracking/batch endpoint as a defensive guard against a runaway client, independent of the Pi's own batch size. Verified against the real deployed server: a simulated 1250-row backlog now drains in 3 batches (~0.35s server round-trip time) instead of 13 batches spread across ~6.5 minutes of idle sleeping. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- modules/app_upload.py | 17 ++++++++++++++--- server/app.py | 8 ++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/modules/app_upload.py b/modules/app_upload.py index bd38cac..f6c158d 100644 --- a/modules/app_upload.py +++ b/modules/app_upload.py @@ -27,8 +27,9 @@ SERVER_URL = os.getenv("YARIS_SERVER_URL", "").rstrip("/") DEVICE_NAME = os.getenv("YARIS_DEVICE_NAME", "") API_KEY = os.getenv("YARIS_API_KEY", "") -UPLOAD_INTERVAL = 30 # seconds between upload attempts when idle/healthy -TRACKING_BATCH_SIZE = 100 +UPLOAD_INTERVAL = 30 # seconds between upload attempts once caught up +CATCHUP_INTERVAL = 1 # seconds between batches while draining a backlog +TRACKING_BATCH_SIZE = 500 BACKOFF_INITIAL = 5 # seconds, doubles on each consecutive failure BACKOFF_MAX = 300 @@ -160,7 +161,17 @@ def _upload_loop(): _last_error = None _last_success_at = _dt.datetime.now().isoformat() backoff = BACKOFF_INITIAL - time.sleep(UPLOAD_INTERVAL) + + # A full batch means there's likely more backlog queued up (the + # tracking table logs a new row every 5s, so a stale/offline + # device can build up thousands of rows quickly) - keep draining + # back-to-back instead of idling for the full interval. Only + # settle into the slow/idle cadence once a batch comes back + # short, i.e. we're actually caught up. + if sent >= TRACKING_BATCH_SIZE: + time.sleep(CATCHUP_INTERVAL) + else: + time.sleep(UPLOAD_INTERVAL) except requests.RequestException as e: _last_error = str(e) print(f"Upload failed ({e}), retrying in {backoff}s") diff --git a/server/app.py b/server/app.py index deed615..ea4072a 100644 --- a/server/app.py +++ b/server/app.py @@ -17,6 +17,10 @@ STORAGE_DIR = os.environ.get("DASHCAM_STORAGE_DIR", "/data/dashcam") MAX_DASHCAM_UPLOAD_MB = int(os.environ.get("MAX_DASHCAM_UPLOAD_MB", "2048")) app.config["MAX_CONTENT_LENGTH"] = MAX_DASHCAM_UPLOAD_MB * 1024 * 1024 +# Sanity cap on tracking points per request - the Pi client batches in +# groups of 500, this just guards against a runaway/misbehaving client. +MAX_TRACKING_BATCH_POINTS = 5000 + @app.get("/health") def health(): @@ -39,6 +43,10 @@ def upload_tracking_batch(): points = body.get("points") if not isinstance(points, list) or not points: return jsonify(error="expected a non-empty 'points' list"), 400 + if len(points) > MAX_TRACKING_BATCH_POINTS: + return jsonify( + error=f"batch too large - max {MAX_TRACKING_BATCH_POINTS} points per request" + ), 400 for p in points: if "recorded_at" not in p: