Add Pi-side upload client for Phase 9 remote server

- app_sql.py: add UPLOADED column + migration on tracking table, add
  settings and dashcam_uploads tables, add helpers for unsent-row
  queries, marking rows uploaded, and dashcam upload dedupe tracking.
- app_upload.py: new background upload client. Periodically batches
  unsent tracking rows to POST /api/v1/tracking/batch and uploads new
  dashcam files (sha256-verified) to POST /api/v1/dashcam/upload, with
  exponential backoff on failure. Exposes get_status() and a
  dashcam-upload enable/disable toggle backed by the settings table.
- app.py: start the upload client thread at startup, add
  GET /api/upload/status and POST /api/upload/dashcam_toggle routes.
- dash.html/dash.js: add a "Remote server upload" card with upload
  status and a toggle switch to pause dashcam video uploads while
  trip/GPS log uploads keep running (per user request - useful on a
  slow/metered connection).
- db/upload_creds.env: new gitignore-exempted template (mirrors
  db/creds.env) for YARIS_SERVER_URL/YARIS_DEVICE_NAME/YARIS_API_KEY.
- requirements.txt: add requests.

Tested against the real deployed unraid server: tracking batch upload
+ dedupe, dashcam upload + dedupe, and toggle-off correctly pausing
dashcam uploads while tracking uploads continue. Also verified the
full Flask app imports/runs cleanly and the new API routes + dash.html
toggle render correctly in a headless browser.

Co-authored-by: Copilot <[email protected]>
This commit is contained in:
Mathieu Moran
2026-08-08 20:05:42 -04:00
co-authored by Copilot
parent 9bdf5bad2c
commit 42e679782e
8 changed files with 405 additions and 5 deletions
+185
View File
@@ -0,0 +1,185 @@
"""Uploads locally-queued trip/GPS tracking rows and dashcam files to the
remote server (see server/ in this repo) over HTTP, with retry/backoff.
Tracking-log uploads always run when configured. Dashcam file uploads can be
paused independently via a persisted toggle (settings table in the same
SQLite db used for tracking) so a user can stop shipping large video files
over a slow/metered connection while trip logs keep flowing.
Credentials/server URL are loaded from db/upload_creds.env (gitignored,
never commit real values - mirrors the db/creds.env pattern already used by
app_db.py for MongoDB).
"""
import hashlib
import os
import time
from threading import Thread
import requests
from dotenv import load_dotenv
from modules import app_sql, app_dash
_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
load_dotenv(os.path.join(_BASE_DIR, "db", "upload_creds.env"))
SERVER_URL = os.getenv("YARIS_SERVER_URL", "").rstrip("/")
DEVICE_NAME = os.getenv("YARIS_DEVICE_NAME", "")
API_KEY = os.getenv("YARIS_API_KEY", "")
UPLOAD_INTERVAL = 30 # seconds between upload attempts when idle/healthy
TRACKING_BATCH_SIZE = 100
BACKOFF_INITIAL = 5 # seconds, doubles on each consecutive failure
BACKOFF_MAX = 300
DASHCAM_UPLOAD_SETTING_KEY = "dashcam_upload_enabled"
_upload_thread = None
_last_error = None
_last_success_at = None
def _headers():
return {"X-Device-Name": DEVICE_NAME, "X-Api-Key": API_KEY}
def is_configured():
return bool(SERVER_URL and DEVICE_NAME and API_KEY)
def is_dashcam_upload_enabled():
"""Defaults to enabled - the user opts out, not in."""
return app_sql.get_setting(DASHCAM_UPLOAD_SETTING_KEY, "1") == "1"
def set_dashcam_upload_enabled(enabled):
app_sql.set_setting(DASHCAM_UPLOAD_SETTING_KEY, "1" if enabled else "0")
def get_status():
"""Snapshot used by the /api/upload/status endpoint for the web UI."""
return {
"configured": is_configured(),
"dashcam_upload_enabled": is_dashcam_upload_enabled(),
"pending_tracking_rows": app_sql.count_unsent_tracking_rows(),
"last_error": _last_error,
"last_success_at": _last_success_at,
}
def _sha256_file(path):
hasher = hashlib.sha256()
with open(path, "rb") as f:
while True:
chunk = f.read(1024 * 1024)
if not chunk:
break
hasher.update(chunk)
return hasher.hexdigest()
def _upload_tracking_batch():
"""Send up to TRACKING_BATCH_SIZE not-yet-uploaded rows. Returns count sent."""
rows = app_sql.get_unsent_tracking_rows(limit=TRACKING_BATCH_SIZE)
if not rows:
return 0
points = [
{
"recorded_at": row["datetime"],
"gps_lat": row["gpsx"],
"gps_lon": row["gpsy"],
"speed": row["speed"],
"odometer": row["odometer"],
"vehicle_name": row["vn"],
"driver_name": row["name"],
}
for row in rows
]
resp = requests.post(
f"{SERVER_URL}/api/v1/tracking/batch",
json={"points": points},
headers=_headers(),
timeout=15,
)
resp.raise_for_status()
app_sql.mark_tracking_rows_uploaded([row["id"] for row in rows])
return len(rows)
def _upload_dashcam_files():
"""Upload any not-yet-uploaded dashcam files, one at a time. Returns count sent.
Skipped entirely (returns 0) while the dashcam-upload toggle is off -
tracking-log uploads are unaffected by this toggle.
"""
if not is_dashcam_upload_enabled():
return 0
uploaded_count = 0
for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
sha256 = _sha256_file(abs_path)
if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
continue
with open(abs_path, "rb") as f:
resp = requests.post(
f"{SERVER_URL}/api/v1/dashcam/upload",
data={"relative_path": rel_path, "sha256": sha256},
files={"file": f},
headers=_headers(),
timeout=300,
)
resp.raise_for_status()
app_sql.mark_dashcam_file_uploaded(rel_path, sha256)
uploaded_count += 1
# Re-check the toggle between files so flipping it off mid-run stops
# further dashcam uploads promptly without waiting for the next cycle.
if not is_dashcam_upload_enabled():
break
return uploaded_count
def _upload_loop():
global _last_error, _last_success_at
import datetime as _dt
backoff = BACKOFF_INITIAL
while True:
if not is_configured():
time.sleep(UPLOAD_INTERVAL)
continue
try:
sent = _upload_tracking_batch()
uploaded = _upload_dashcam_files()
if sent or uploaded:
print(f"Upload: sent {sent} tracking rows, {uploaded} dashcam files")
_last_error = None
_last_success_at = _dt.datetime.now().isoformat()
backoff = BACKOFF_INITIAL
time.sleep(UPLOAD_INTERVAL)
except requests.RequestException as e:
_last_error = str(e)
print(f"Upload failed ({e}), retrying in {backoff}s")
time.sleep(backoff)
backoff = min(backoff * 2, BACKOFF_MAX)
def start_uploading():
"""Start the background upload thread, if the server is configured."""
global _upload_thread
if not is_configured():
print(
"Upload client not configured (set YARIS_SERVER_URL/YARIS_DEVICE_NAME/"
"YARIS_API_KEY in db/upload_creds.env) - uploads disabled."
)
return
if _upload_thread is None or not _upload_thread.is_alive():
_upload_thread = Thread(target=_upload_loop, daemon=True)
_upload_thread.start()
print("Upload-client thread started.")
else:
print("Upload-client thread already running.")