From 42e679782e76a4715b87dbf44f4c77a02797ef4d Mon Sep 17 00:00:00 2001 From: Mathieu Moran Date: Sat, 8 Aug 2026 20:05:42 -0400 Subject: [PATCH] Add Pi-side upload client for Phase 9 remote server - app_sql.py: add UPLOADED column + migration on tracking table, add settings and dashcam_uploads tables, add helpers for unsent-row queries, marking rows uploaded, and dashcam upload dedupe tracking. - app_upload.py: new background upload client. Periodically batches unsent tracking rows to POST /api/v1/tracking/batch and uploads new dashcam files (sha256-verified) to POST /api/v1/dashcam/upload, with exponential backoff on failure. Exposes get_status() and a dashcam-upload enable/disable toggle backed by the settings table. - app.py: start the upload client thread at startup, add GET /api/upload/status and POST /api/upload/dashcam_toggle routes. - dash.html/dash.js: add a "Remote server upload" card with upload status and a toggle switch to pause dashcam video uploads while trip/GPS log uploads keep running (per user request - useful on a slow/metered connection). - db/upload_creds.env: new gitignore-exempted template (mirrors db/creds.env) for YARIS_SERVER_URL/YARIS_DEVICE_NAME/YARIS_API_KEY. - requirements.txt: add requests. Tested against the real deployed unraid server: tracking batch upload + dedupe, dashcam upload + dedupe, and toggle-off correctly pausing dashcam uploads while tracking uploads continue. Also verified the full Flask app imports/runs cleanly and the new API routes + dash.html toggle render correctly in a headless browser. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 1 + app.py | 17 +++- db/upload_creds.env | 12 +++ modules/app_sql.py | 120 ++++++++++++++++++++++++++- modules/app_upload.py | 185 ++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 +- static/js/dash.js | 51 ++++++++++++ templates/dash.html | 21 ++++- 8 files changed, 405 insertions(+), 5 deletions(-) create mode 100644 db/upload_creds.env create mode 100644 modules/app_upload.py diff --git a/.gitignore b/.gitignore index e9dd643..9e9b148 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ static/vendor/tiles/ .env *.env !db/creds.env +!db/upload_creds.env # Local SQLite trip-tracking databases (per-device data, not source) db/sql/ diff --git a/app.py b/app.py index 592442c..a04bad6 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,5 @@ -from flask import Flask, render_template, make_response, json, jsonify -from modules import app_gps, app_events, app_car, app_account, app_gps,app_obd, app_sql, app_dash +from flask import Flask, render_template, make_response, json, jsonify, request +from modules import app_gps, app_events, app_car, app_account, app_gps,app_obd, app_sql, app_dash, app_upload import time from gevent.pywsgi import WSGIServer @@ -12,6 +12,7 @@ obd = app_obd.OBD() car = app_car.Car() gps = app_gps.start_gps() app_sql.start_logging() +app_upload.start_uploading() @app.route('/') def index(): @@ -37,6 +38,18 @@ def dash_import(): summary = app_dash.import_dashcam() return jsonify(summary) +@app.route('/api/upload/status') +def upload_status(): + return jsonify(app_upload.get_status()) + +@app.route('/api/upload/dashcam_toggle', methods=['POST']) +def upload_dashcam_toggle(): + body = request.get_json(silent=True) or {} + if "enabled" not in body: + return jsonify(error="expected JSON body {'enabled': true|false}"), 400 + app_upload.set_dashcam_upload_enabled(bool(body["enabled"])) + return jsonify(app_upload.get_status()) + # not sure if im gonna use this @app.route('/api/obd_speed/') def data(): diff --git a/db/upload_creds.env b/db/upload_creds.env new file mode 100644 index 0000000..1a54721 --- /dev/null +++ b/db/upload_creds.env @@ -0,0 +1,12 @@ +## +## yaris-obd remote server upload client credentials +## +## Real values are set locally on the Pi only - never commit real credentials +## here. Get YARIS_DEVICE_NAME/YARIS_API_KEY by running +## `docker compose exec api python3 scripts/create_device.py ` on the +## server (see server/README.md). +## + +YARIS_SERVER_URL = "" +YARIS_DEVICE_NAME = "" +YARIS_API_KEY = "" diff --git a/modules/app_sql.py b/modules/app_sql.py index 61d73f5..02562c0 100644 --- a/modules/app_sql.py +++ b/modules/app_sql.py @@ -33,7 +33,8 @@ SQL_CREATE_TRACKING_TABLE = """CREATE TABLE IF NOT EXISTS tracking ( GPSX float, GPSY float, SPEED float, - ODOMETER float + ODOMETER float, + UPLOADED integer DEFAULT 0 );""" SQL_CREATE_DTC_TABLE = """CREATE TABLE IF NOT EXISTS dtc ( @@ -43,6 +44,19 @@ SQL_CREATE_DTC_TABLE = """CREATE TABLE IF NOT EXISTS dtc ( DTC_ID text );""" +SQL_CREATE_SETTINGS_TABLE = """CREATE TABLE IF NOT EXISTS settings ( + key text PRIMARY KEY, + value text + );""" + +SQL_CREATE_DASHCAM_UPLOADS_TABLE = """CREATE TABLE IF NOT EXISTS dashcam_uploads ( + id integer PRIMARY KEY, + RELATIVE_PATH text NOT NULL, + SHA256 text NOT NULL, + UPLOADED_AT datetime, + UNIQUE(RELATIVE_PATH, SHA256) + );""" + def create_conn(db_file): """Open a SQLite connection to db_file, creating its parent folder if needed. @@ -82,11 +96,23 @@ def get_conn(): if _conn is not None: create_table(_conn, SQL_CREATE_TRACKING_TABLE) create_table(_conn, SQL_CREATE_DTC_TABLE) + create_table(_conn, SQL_CREATE_SETTINGS_TABLE) + create_table(_conn, SQL_CREATE_DASHCAM_UPLOADS_TABLE) + _migrate_tracking_table(_conn) else: print("Error! cannot create the database connection.") return _conn +def _migrate_tracking_table(conn): + """Add the UPLOADED column to a pre-existing tracking table that predates + the remote-upload feature (SQLite has no "ADD COLUMN IF NOT EXISTS").""" + existing_columns = {row[1] for row in conn.execute("PRAGMA table_info(tracking)").fetchall()} + if "UPLOADED" not in existing_columns: + conn.execute("ALTER TABLE tracking ADD COLUMN UPLOADED integer DEFAULT 0") + conn.commit() + + def insert_tracking(gps_x, gps_y, speed, odometer=None): """Insert a single trip-tracking snapshot row (GPS position + speed).""" conn = get_conn() @@ -103,6 +129,98 @@ def insert_tracking(gps_x, gps_y, speed, odometer=None): print("Failed to insert tracking row:", e) +def get_setting(key, default=None): + """Read a persisted UI/app setting (e.g. the dashcam-upload toggle).""" + conn = get_conn() + if conn is None: + return default + with _conn_lock: + cur = conn.execute("SELECT value FROM settings WHERE key = ?", (key,)) + row = cur.fetchone() + return row[0] if row is not None else default + + +def set_setting(key, value): + conn = get_conn() + if conn is None: + return + with _conn_lock: + conn.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (key, value), + ) + conn.commit() + + +def get_unsent_tracking_rows(limit=100): + """Return up to `limit` tracking rows not yet uploaded to the remote server, + oldest first, as a list of dicts ready to be sent as JSON.""" + conn = get_conn() + if conn is None: + return [] + with _conn_lock: + cur = conn.execute( + """SELECT id, VN, NAME, DATETIME, GPSX, GPSY, SPEED, ODOMETER + FROM tracking WHERE UPLOADED = 0 ORDER BY id ASC LIMIT ?""", + (limit,), + ) + rows = cur.fetchall() + return [ + { + "id": r[0], "vn": r[1], "name": r[2], "datetime": r[3], + "gpsx": r[4], "gpsy": r[5], "speed": r[6], "odometer": r[7], + } + for r in rows + ] + + +def count_unsent_tracking_rows(): + conn = get_conn() + if conn is None: + return 0 + with _conn_lock: + cur = conn.execute("SELECT COUNT(*) FROM tracking WHERE UPLOADED = 0") + return cur.fetchone()[0] + + +def mark_tracking_rows_uploaded(ids): + """Flip UPLOADED=1 for the given row ids after a successful server ack.""" + if not ids: + return + conn = get_conn() + if conn is None: + return + with _conn_lock: + conn.executemany("UPDATE tracking SET UPLOADED = 1 WHERE id = ?", [(i,) for i in ids]) + conn.commit() + + +def dashcam_file_marked_uploaded(relative_path, sha256): + """Whether this exact (path, content) was already confirmed uploaded.""" + conn = get_conn() + if conn is None: + return False + with _conn_lock: + cur = conn.execute( + "SELECT 1 FROM dashcam_uploads WHERE RELATIVE_PATH = ? AND SHA256 = ?", + (relative_path, sha256), + ) + return cur.fetchone() is not None + + +def mark_dashcam_file_uploaded(relative_path, sha256): + conn = get_conn() + if conn is None: + return + with _conn_lock: + conn.execute( + "INSERT OR IGNORE INTO dashcam_uploads (RELATIVE_PATH, SHA256, UPLOADED_AT) VALUES (?, ?, ?)", + (relative_path, sha256, datetime.now()), + ) + conn.commit() + + def _log_loop(): # Deferred import to avoid a circular import at module load time # (app_obd.py imports app_sql, and app_sql needs app_obd.speed here). diff --git a/modules/app_upload.py b/modules/app_upload.py new file mode 100644 index 0000000..bd38cac --- /dev/null +++ b/modules/app_upload.py @@ -0,0 +1,185 @@ +"""Uploads locally-queued trip/GPS tracking rows and dashcam files to the +remote server (see server/ in this repo) over HTTP, with retry/backoff. + +Tracking-log uploads always run when configured. Dashcam file uploads can be +paused independently via a persisted toggle (settings table in the same +SQLite db used for tracking) so a user can stop shipping large video files +over a slow/metered connection while trip logs keep flowing. + +Credentials/server URL are loaded from db/upload_creds.env (gitignored, +never commit real values - mirrors the db/creds.env pattern already used by +app_db.py for MongoDB). +""" +import hashlib +import os +import time +from threading import Thread + +import requests +from dotenv import load_dotenv + +from modules import app_sql, app_dash + +_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +load_dotenv(os.path.join(_BASE_DIR, "db", "upload_creds.env")) + +SERVER_URL = os.getenv("YARIS_SERVER_URL", "").rstrip("/") +DEVICE_NAME = os.getenv("YARIS_DEVICE_NAME", "") +API_KEY = os.getenv("YARIS_API_KEY", "") + +UPLOAD_INTERVAL = 30 # seconds between upload attempts when idle/healthy +TRACKING_BATCH_SIZE = 100 +BACKOFF_INITIAL = 5 # seconds, doubles on each consecutive failure +BACKOFF_MAX = 300 + +DASHCAM_UPLOAD_SETTING_KEY = "dashcam_upload_enabled" + +_upload_thread = None +_last_error = None +_last_success_at = None + + +def _headers(): + return {"X-Device-Name": DEVICE_NAME, "X-Api-Key": API_KEY} + + +def is_configured(): + return bool(SERVER_URL and DEVICE_NAME and API_KEY) + + +def is_dashcam_upload_enabled(): + """Defaults to enabled - the user opts out, not in.""" + return app_sql.get_setting(DASHCAM_UPLOAD_SETTING_KEY, "1") == "1" + + +def set_dashcam_upload_enabled(enabled): + app_sql.set_setting(DASHCAM_UPLOAD_SETTING_KEY, "1" if enabled else "0") + + +def get_status(): + """Snapshot used by the /api/upload/status endpoint for the web UI.""" + return { + "configured": is_configured(), + "dashcam_upload_enabled": is_dashcam_upload_enabled(), + "pending_tracking_rows": app_sql.count_unsent_tracking_rows(), + "last_error": _last_error, + "last_success_at": _last_success_at, + } + + +def _sha256_file(path): + hasher = hashlib.sha256() + with open(path, "rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + hasher.update(chunk) + return hasher.hexdigest() + + +def _upload_tracking_batch(): + """Send up to TRACKING_BATCH_SIZE not-yet-uploaded rows. Returns count sent.""" + rows = app_sql.get_unsent_tracking_rows(limit=TRACKING_BATCH_SIZE) + if not rows: + return 0 + + points = [ + { + "recorded_at": row["datetime"], + "gps_lat": row["gpsx"], + "gps_lon": row["gpsy"], + "speed": row["speed"], + "odometer": row["odometer"], + "vehicle_name": row["vn"], + "driver_name": row["name"], + } + for row in rows + ] + + resp = requests.post( + f"{SERVER_URL}/api/v1/tracking/batch", + json={"points": points}, + headers=_headers(), + timeout=15, + ) + resp.raise_for_status() + app_sql.mark_tracking_rows_uploaded([row["id"] for row in rows]) + return len(rows) + + +def _upload_dashcam_files(): + """Upload any not-yet-uploaded dashcam files, one at a time. Returns count sent. + + Skipped entirely (returns 0) while the dashcam-upload toggle is off - + tracking-log uploads are unaffected by this toggle. + """ + if not is_dashcam_upload_enabled(): + return 0 + + uploaded_count = 0 + for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION): + sha256 = _sha256_file(abs_path) + if app_sql.dashcam_file_marked_uploaded(rel_path, sha256): + continue + + with open(abs_path, "rb") as f: + resp = requests.post( + f"{SERVER_URL}/api/v1/dashcam/upload", + data={"relative_path": rel_path, "sha256": sha256}, + files={"file": f}, + headers=_headers(), + timeout=300, + ) + resp.raise_for_status() + app_sql.mark_dashcam_file_uploaded(rel_path, sha256) + uploaded_count += 1 + + # Re-check the toggle between files so flipping it off mid-run stops + # further dashcam uploads promptly without waiting for the next cycle. + if not is_dashcam_upload_enabled(): + break + + return uploaded_count + + +def _upload_loop(): + global _last_error, _last_success_at + import datetime as _dt + + backoff = BACKOFF_INITIAL + while True: + if not is_configured(): + time.sleep(UPLOAD_INTERVAL) + continue + try: + sent = _upload_tracking_batch() + uploaded = _upload_dashcam_files() + if sent or uploaded: + print(f"Upload: sent {sent} tracking rows, {uploaded} dashcam files") + _last_error = None + _last_success_at = _dt.datetime.now().isoformat() + backoff = BACKOFF_INITIAL + time.sleep(UPLOAD_INTERVAL) + except requests.RequestException as e: + _last_error = str(e) + print(f"Upload failed ({e}), retrying in {backoff}s") + time.sleep(backoff) + backoff = min(backoff * 2, BACKOFF_MAX) + + +def start_uploading(): + """Start the background upload thread, if the server is configured.""" + global _upload_thread + if not is_configured(): + print( + "Upload client not configured (set YARIS_SERVER_URL/YARIS_DEVICE_NAME/" + "YARIS_API_KEY in db/upload_creds.env) - uploads disabled." + ) + return + if _upload_thread is None or not _upload_thread.is_alive(): + _upload_thread = Thread(target=_upload_loop, daemon=True) + _upload_thread.start() + print("Upload-client thread started.") + else: + print("Upload-client thread already running.") diff --git a/requirements.txt b/requirements.txt index df83dea..df821e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ gevent pymongo python-dotenv pyserial -obd \ No newline at end of file +obd +requests \ No newline at end of file diff --git a/static/js/dash.js b/static/js/dash.js index 3e952dc..331eb3e 100644 --- a/static/js/dash.js +++ b/static/js/dash.js @@ -63,3 +63,54 @@ importBtn.addEventListener('click', importDashcam); // Auto-scan once when the page loads. scanForDashcam(); + +// --- Remote server upload status/toggle --- + +const uploadStatusEl = document.getElementById('upload-status'); +const dashcamToggle = document.getElementById('dashcam-upload-toggle'); + +function describeUploadStatus(status) { + if (!status.configured) { + return 'Remote server not configured on this device (see db/upload_creds.env).'; + } + let text = `Pending trip-log rows to upload: ${status.pending_tracking_rows}.`; + if (status.last_success_at) { + text += ` Last successful upload: ${status.last_success_at}.`; + } + if (status.last_error) { + text += ` Last error: ${status.last_error}.`; + } + return text; +} + +function refreshUploadStatus() { + fetch('/api/upload/status') + .then(response => response.json()) + .then(status => { + uploadStatusEl.textContent = describeUploadStatus(status); + dashcamToggle.checked = status.dashcam_upload_enabled; + dashcamToggle.disabled = !status.configured; + }) + .catch(err => { + uploadStatusEl.textContent = 'Error while checking upload status: ' + err; + }); +} + +dashcamToggle.addEventListener('change', () => { + fetch('/api/upload/dashcam_toggle', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ enabled: dashcamToggle.checked }), + }) + .then(response => response.json()) + .then(status => { + uploadStatusEl.textContent = describeUploadStatus(status); + dashcamToggle.checked = status.dashcam_upload_enabled; + }) + .catch(err => { + uploadStatusEl.textContent = 'Error while updating toggle: ' + err; + }); +}); + +refreshUploadStatus(); +setInterval(refreshUploadStatus, 15000); diff --git a/templates/dash.html b/templates/dash.html index 56859a0..322262d 100644 --- a/templates/dash.html +++ b/templates/dash.html @@ -68,7 +68,7 @@ -
+

Import results

@@ -76,6 +76,25 @@

No import has been run yet.

+
+
+

Remote server upload

+
+
+

Checking upload status...

+
+ + +
+

+ Trip/GPS log uploads always stay on. Turning this off only pauses + the (larger) dashcam video uploads - useful on a slow or metered + connection. +

+
+