Add Pi-side upload client for Phase 9 remote server

- app_sql.py: add UPLOADED column + migration on tracking table, add
  settings and dashcam_uploads tables, add helpers for unsent-row
  queries, marking rows uploaded, and dashcam upload dedupe tracking.
- app_upload.py: new background upload client. Periodically batches
  unsent tracking rows to POST /api/v1/tracking/batch and uploads new
  dashcam files (sha256-verified) to POST /api/v1/dashcam/upload, with
  exponential backoff on failure. Exposes get_status() and a
  dashcam-upload enable/disable toggle backed by the settings table.
- app.py: start the upload client thread at startup, add
  GET /api/upload/status and POST /api/upload/dashcam_toggle routes.
- dash.html/dash.js: add a "Remote server upload" card with upload
  status and a toggle switch to pause dashcam video uploads while
  trip/GPS log uploads keep running (per user request - useful on a
  slow/metered connection).
- db/upload_creds.env: new gitignore-exempted template (mirrors
  db/creds.env) for YARIS_SERVER_URL/YARIS_DEVICE_NAME/YARIS_API_KEY.
- requirements.txt: add requests.

Tested against the real deployed unraid server: tracking batch upload
+ dedupe, dashcam upload + dedupe, and toggle-off correctly pausing
dashcam uploads while tracking uploads continue. Also verified the
full Flask app imports/runs cleanly and the new API routes + dash.html
toggle render correctly in a headless browser.

Co-authored-by: Copilot <[email protected]>

.gitignore

@@ -19,6 +19,7 @@ static/vendor/tiles/
 .env
 *.env
 !db/creds.env
+!db/upload_creds.env
 
 # Local SQLite trip-tracking databases (per-device data, not source)
 db/sql/

app.py

@@ -1,5 +1,5 @@
-from flask import Flask, render_template, make_response, json, jsonify
-from modules import app_gps, app_events, app_car, app_account, app_gps,app_obd, app_sql, app_dash
+from flask import Flask, render_template, make_response, json, jsonify, request
+from modules import app_gps, app_events, app_car, app_account, app_gps,app_obd, app_sql, app_dash, app_upload
 import time
 from gevent.pywsgi import WSGIServer
 
@@ -12,6 +12,7 @@ obd = app_obd.OBD()
 car = app_car.Car()
 gps = app_gps.start_gps()
 app_sql.start_logging()
+app_upload.start_uploading()
 
 @app.route('/')
 def index():
@@ -37,6 +38,18 @@ def dash_import():
     summary = app_dash.import_dashcam()
     return jsonify(summary)
 
+@app.route('/api/upload/status')
+def upload_status():
+    return jsonify(app_upload.get_status())
+
+@app.route('/api/upload/dashcam_toggle', methods=['POST'])
+def upload_dashcam_toggle():
+    body = request.get_json(silent=True) or {}
+    if "enabled" not in body:
+        return jsonify(error="expected JSON body {'enabled': true|false}"), 400
+    app_upload.set_dashcam_upload_enabled(bool(body["enabled"]))
+    return jsonify(app_upload.get_status())
+
 # not sure if im gonna use this
 @app.route('/api/obd_speed/')
 def data():

db/upload_creds.env

@@ -0,0 +1,12 @@
+##
+## yaris-obd remote server upload client credentials
+##
+## Real values are set locally on the Pi only - never commit real credentials
+## here. Get YARIS_DEVICE_NAME/YARIS_API_KEY by running
+## `docker compose exec api python3 scripts/create_device.py <name>` on the
+## server (see server/README.md).
+##
+
+YARIS_SERVER_URL = ""
+YARIS_DEVICE_NAME = ""
+YARIS_API_KEY = ""

modules/app_sql.py

@@ -33,7 +33,8 @@ SQL_CREATE_TRACKING_TABLE = """CREATE TABLE IF NOT EXISTS tracking (
                                 GPSX float,
                                 GPSY float,
                                 SPEED float,
-                                ODOMETER float
+                                ODOMETER float,
+                                UPLOADED integer DEFAULT 0
                             );"""
 
 SQL_CREATE_DTC_TABLE = """CREATE TABLE IF NOT EXISTS dtc (
@@ -43,6 +44,19 @@ SQL_CREATE_DTC_TABLE = """CREATE TABLE IF NOT EXISTS dtc (
                                 DTC_ID text
                             );"""
 
+SQL_CREATE_SETTINGS_TABLE = """CREATE TABLE IF NOT EXISTS settings (
+                                key text PRIMARY KEY,
+                                value text
+                            );"""
+
+SQL_CREATE_DASHCAM_UPLOADS_TABLE = """CREATE TABLE IF NOT EXISTS dashcam_uploads (
+                                id integer PRIMARY KEY,
+                                RELATIVE_PATH text NOT NULL,
+                                SHA256 text NOT NULL,
+                                UPLOADED_AT datetime,
+                                UNIQUE(RELATIVE_PATH, SHA256)
+                            );"""
+
 
 def create_conn(db_file):
     """Open a SQLite connection to db_file, creating its parent folder if needed.
@@ -82,11 +96,23 @@ def get_conn():
         if _conn is not None:
             create_table(_conn, SQL_CREATE_TRACKING_TABLE)
             create_table(_conn, SQL_CREATE_DTC_TABLE)
+            create_table(_conn, SQL_CREATE_SETTINGS_TABLE)
+            create_table(_conn, SQL_CREATE_DASHCAM_UPLOADS_TABLE)
+            _migrate_tracking_table(_conn)
         else:
             print("Error! cannot create the database connection.")
     return _conn
 
 
+def _migrate_tracking_table(conn):
+    """Add the UPLOADED column to a pre-existing tracking table that predates
+    the remote-upload feature (SQLite has no "ADD COLUMN IF NOT EXISTS")."""
+    existing_columns = {row[1] for row in conn.execute("PRAGMA table_info(tracking)").fetchall()}
+    if "UPLOADED" not in existing_columns:
+        conn.execute("ALTER TABLE tracking ADD COLUMN UPLOADED integer DEFAULT 0")
+        conn.commit()
+
+
 def insert_tracking(gps_x, gps_y, speed, odometer=None):
     """Insert a single trip-tracking snapshot row (GPS position + speed)."""
     conn = get_conn()
@@ -103,6 +129,98 @@ def insert_tracking(gps_x, gps_y, speed, odometer=None):
             print("Failed to insert tracking row:", e)
 
 
+def get_setting(key, default=None):
+    """Read a persisted UI/app setting (e.g. the dashcam-upload toggle)."""
+    conn = get_conn()
+    if conn is None:
+        return default
+    with _conn_lock:
+        cur = conn.execute("SELECT value FROM settings WHERE key = ?", (key,))
+        row = cur.fetchone()
+    return row[0] if row is not None else default
+
+
+def set_setting(key, value):
+    conn = get_conn()
+    if conn is None:
+        return
+    with _conn_lock:
+        conn.execute(
+            "INSERT INTO settings (key, value) VALUES (?, ?) "
+            "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+            (key, value),
+        )
+        conn.commit()
+
+
+def get_unsent_tracking_rows(limit=100):
+    """Return up to `limit` tracking rows not yet uploaded to the remote server,
+    oldest first, as a list of dicts ready to be sent as JSON."""
+    conn = get_conn()
+    if conn is None:
+        return []
+    with _conn_lock:
+        cur = conn.execute(
+            """SELECT id, VN, NAME, DATETIME, GPSX, GPSY, SPEED, ODOMETER
+               FROM tracking WHERE UPLOADED = 0 ORDER BY id ASC LIMIT ?""",
+            (limit,),
+        )
+        rows = cur.fetchall()
+    return [
+        {
+            "id": r[0], "vn": r[1], "name": r[2], "datetime": r[3],
+            "gpsx": r[4], "gpsy": r[5], "speed": r[6], "odometer": r[7],
+        }
+        for r in rows
+    ]
+
+
+def count_unsent_tracking_rows():
+    conn = get_conn()
+    if conn is None:
+        return 0
+    with _conn_lock:
+        cur = conn.execute("SELECT COUNT(*) FROM tracking WHERE UPLOADED = 0")
+        return cur.fetchone()[0]
+
+
+def mark_tracking_rows_uploaded(ids):
+    """Flip UPLOADED=1 for the given row ids after a successful server ack."""
+    if not ids:
+        return
+    conn = get_conn()
+    if conn is None:
+        return
+    with _conn_lock:
+        conn.executemany("UPDATE tracking SET UPLOADED = 1 WHERE id = ?", [(i,) for i in ids])
+        conn.commit()
+
+
+def dashcam_file_marked_uploaded(relative_path, sha256):
+    """Whether this exact (path, content) was already confirmed uploaded."""
+    conn = get_conn()
+    if conn is None:
+        return False
+    with _conn_lock:
+        cur = conn.execute(
+            "SELECT 1 FROM dashcam_uploads WHERE RELATIVE_PATH = ? AND SHA256 = ?",
+            (relative_path, sha256),
+        )
+        return cur.fetchone() is not None
+
+
+def mark_dashcam_file_uploaded(relative_path, sha256):
+    conn = get_conn()
+    if conn is None:
+        return
+    with _conn_lock:
+        conn.execute(
+            "INSERT OR IGNORE INTO dashcam_uploads (RELATIVE_PATH, SHA256, UPLOADED_AT) VALUES (?, ?, ?)",
+            (relative_path, sha256, datetime.now()),
+        )
+        conn.commit()
+
+
 def _log_loop():
     # Deferred import to avoid a circular import at module load time
     # (app_obd.py imports app_sql, and app_sql needs app_obd.speed here).

modules/app_upload.py

@@ -0,0 +1,185 @@
+"""Uploads locally-queued trip/GPS tracking rows and dashcam files to the
+remote server (see server/ in this repo) over HTTP, with retry/backoff.
+
+Tracking-log uploads always run when configured. Dashcam file uploads can be
+paused independently via a persisted toggle (settings table in the same
+SQLite db used for tracking) so a user can stop shipping large video files
+over a slow/metered connection while trip logs keep flowing.
+
+Credentials/server URL are loaded from db/upload_creds.env (gitignored,
+never commit real values - mirrors the db/creds.env pattern already used by
+app_db.py for MongoDB).
+"""
+import hashlib
+import os
+import time
+from threading import Thread
+
+import requests
+from dotenv import load_dotenv
+
+from modules import app_sql, app_dash
+
+_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+load_dotenv(os.path.join(_BASE_DIR, "db", "upload_creds.env"))
+
+SERVER_URL = os.getenv("YARIS_SERVER_URL", "").rstrip("/")
+DEVICE_NAME = os.getenv("YARIS_DEVICE_NAME", "")
+API_KEY = os.getenv("YARIS_API_KEY", "")
+
+UPLOAD_INTERVAL = 30       # seconds between upload attempts when idle/healthy
+TRACKING_BATCH_SIZE = 100
+BACKOFF_INITIAL = 5        # seconds, doubles on each consecutive failure
+BACKOFF_MAX = 300
+
+DASHCAM_UPLOAD_SETTING_KEY = "dashcam_upload_enabled"
+
+_upload_thread = None
+_last_error = None
+_last_success_at = None
+
+
+def _headers():
+    return {"X-Device-Name": DEVICE_NAME, "X-Api-Key": API_KEY}
+
+
+def is_configured():
+    return bool(SERVER_URL and DEVICE_NAME and API_KEY)
+
+
+def is_dashcam_upload_enabled():
+    """Defaults to enabled - the user opts out, not in."""
+    return app_sql.get_setting(DASHCAM_UPLOAD_SETTING_KEY, "1") == "1"
+
+
+def set_dashcam_upload_enabled(enabled):
+    app_sql.set_setting(DASHCAM_UPLOAD_SETTING_KEY, "1" if enabled else "0")
+
+
+def get_status():
+    """Snapshot used by the /api/upload/status endpoint for the web UI."""
+    return {
+        "configured": is_configured(),
+        "dashcam_upload_enabled": is_dashcam_upload_enabled(),
+        "pending_tracking_rows": app_sql.count_unsent_tracking_rows(),
+        "last_error": _last_error,
+        "last_success_at": _last_success_at,
+    }
+
+
+def _sha256_file(path):
+    hasher = hashlib.sha256()
+    with open(path, "rb") as f:
+        while True:
+            chunk = f.read(1024 * 1024)
+            if not chunk:
+                break
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def _upload_tracking_batch():
+    """Send up to TRACKING_BATCH_SIZE not-yet-uploaded rows. Returns count sent."""
+    rows = app_sql.get_unsent_tracking_rows(limit=TRACKING_BATCH_SIZE)
+    if not rows:
+        return 0
+
+    points = [
+        {
+            "recorded_at": row["datetime"],
+            "gps_lat": row["gpsx"],
+            "gps_lon": row["gpsy"],
+            "speed": row["speed"],
+            "odometer": row["odometer"],
+            "vehicle_name": row["vn"],
+            "driver_name": row["name"],
+        }
+        for row in rows
+    ]
+
+    resp = requests.post(
+        f"{SERVER_URL}/api/v1/tracking/batch",
+        json={"points": points},
+        headers=_headers(),
+        timeout=15,
+    )
+    resp.raise_for_status()
+    app_sql.mark_tracking_rows_uploaded([row["id"] for row in rows])
+    return len(rows)
+
+
+def _upload_dashcam_files():
+    """Upload any not-yet-uploaded dashcam files, one at a time. Returns count sent.
+
+    Skipped entirely (returns 0) while the dashcam-upload toggle is off -
+    tracking-log uploads are unaffected by this toggle.
+    """
+    if not is_dashcam_upload_enabled():
+        return 0
+
+    uploaded_count = 0
+    for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
+        sha256 = _sha256_file(abs_path)
+        if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
+            continue
+
+        with open(abs_path, "rb") as f:
+            resp = requests.post(
+                f"{SERVER_URL}/api/v1/dashcam/upload",
+                data={"relative_path": rel_path, "sha256": sha256},
+                files={"file": f},
+                headers=_headers(),
+                timeout=300,
+            )
+        resp.raise_for_status()
+        app_sql.mark_dashcam_file_uploaded(rel_path, sha256)
+        uploaded_count += 1
+
+        # Re-check the toggle between files so flipping it off mid-run stops
+        # further dashcam uploads promptly without waiting for the next cycle.
+        if not is_dashcam_upload_enabled():
+            break
+
+    return uploaded_count
+
+
+def _upload_loop():
+    global _last_error, _last_success_at
+    import datetime as _dt
+
+    backoff = BACKOFF_INITIAL
+    while True:
+        if not is_configured():
+            time.sleep(UPLOAD_INTERVAL)
+            continue
+        try:
+            sent = _upload_tracking_batch()
+            uploaded = _upload_dashcam_files()
+            if sent or uploaded:
+                print(f"Upload: sent {sent} tracking rows, {uploaded} dashcam files")
+            _last_error = None
+            _last_success_at = _dt.datetime.now().isoformat()
+            backoff = BACKOFF_INITIAL
+            time.sleep(UPLOAD_INTERVAL)
+        except requests.RequestException as e:
+            _last_error = str(e)
+            print(f"Upload failed ({e}), retrying in {backoff}s")
+            time.sleep(backoff)
+            backoff = min(backoff * 2, BACKOFF_MAX)
+
+
+def start_uploading():
+    """Start the background upload thread, if the server is configured."""
+    global _upload_thread
+    if not is_configured():
+        print(
+            "Upload client not configured (set YARIS_SERVER_URL/YARIS_DEVICE_NAME/"
+            "YARIS_API_KEY in db/upload_creds.env) - uploads disabled."
+        )
+        return
+    if _upload_thread is None or not _upload_thread.is_alive():
+        _upload_thread = Thread(target=_upload_loop, daemon=True)
+        _upload_thread.start()
+        print("Upload-client thread started.")
+    else:
+        print("Upload-client thread already running.")

requirements.txt

@@ -5,3 +5,4 @@ pymongo
 python-dotenv
 pyserial
 obd
+requests

static/js/dash.js

@@ -63,3 +63,54 @@ importBtn.addEventListener('click', importDashcam);
 
 // Auto-scan once when the page loads.
 scanForDashcam();
+
+// --- Remote server upload status/toggle ---
+
+const uploadStatusEl = document.getElementById('upload-status');
+const dashcamToggle = document.getElementById('dashcam-upload-toggle');
+
+function describeUploadStatus(status) {
+    if (!status.configured) {
+        return 'Remote server not configured on this device (see db/upload_creds.env).';
+    }
+    let text = `Pending trip-log rows to upload: ${status.pending_tracking_rows}.`;
+    if (status.last_success_at) {
+        text += ` Last successful upload: ${status.last_success_at}.`;
+    }
+    if (status.last_error) {
+        text += ` Last error: ${status.last_error}.`;
+    }
+    return text;
+}
+
+function refreshUploadStatus() {
+    fetch('/api/upload/status')
+        .then(response => response.json())
+        .then(status => {
+            uploadStatusEl.textContent = describeUploadStatus(status);
+            dashcamToggle.checked = status.dashcam_upload_enabled;
+            dashcamToggle.disabled = !status.configured;
+        })
+        .catch(err => {
+            uploadStatusEl.textContent = 'Error while checking upload status: ' + err;
+        });
+}
+
+dashcamToggle.addEventListener('change', () => {
+    fetch('/api/upload/dashcam_toggle', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ enabled: dashcamToggle.checked }),
+    })
+        .then(response => response.json())
+        .then(status => {
+            uploadStatusEl.textContent = describeUploadStatus(status);
+            dashcamToggle.checked = status.dashcam_upload_enabled;
+        })
+        .catch(err => {
+            uploadStatusEl.textContent = 'Error while updating toggle: ' + err;
+        });
+});
+
+refreshUploadStatus();
+setInterval(refreshUploadStatus, 15000);

templates/dash.html

@@ -68,7 +68,7 @@
                             <button id="dash-import-btn" class="btn btn-primary" type="button" disabled>Import videos</button>
                         </div>
                     </div>
-                    <div class="card shadow">
+                    <div class="card shadow mb-4">
                         <div class="card-header py-3">
                             <p class="text-primary m-0 fw-bold">Import results</p>
                         </div>
@@ -76,6 +76,25 @@
                             <p id="dash-import-status">No import has been run yet.</p>
                         </div>
                     </div>
+                    <div class="card shadow">
+                        <div class="card-header py-3">
+                            <p class="text-primary m-0 fw-bold">Remote server upload</p>
+                        </div>
+                        <div class="card-body">
+                            <p id="upload-status">Checking upload status...</p>
+                            <div class="form-check form-switch">
+                                <input class="form-check-input" type="checkbox" role="switch" id="dashcam-upload-toggle" disabled>
+                                <label class="form-check-label" for="dashcam-upload-toggle">
+                                    Upload dashcam videos to remote server
+                                </label>
+                            </div>
+                            <p class="text-muted small mt-2 mb-0">
+                                Trip/GPS log uploads always stay on. Turning this off only pauses
+                                the (larger) dashcam video uploads - useful on a slow or metered
+                                connection.
+                            </p>
+                        </div>
+                    </div>
                 </div>
             </div>
             <footer class="bg-white sticky-footer">