Add Pi-side upload client for Phase 9 remote server
- app_sql.py: add UPLOADED column + migration on tracking table, add settings and dashcam_uploads tables, add helpers for unsent-row queries, marking rows uploaded, and dashcam upload dedupe tracking. - app_upload.py: new background upload client. Periodically batches unsent tracking rows to POST /api/v1/tracking/batch and uploads new dashcam files (sha256-verified) to POST /api/v1/dashcam/upload, with exponential backoff on failure. Exposes get_status() and a dashcam-upload enable/disable toggle backed by the settings table. - app.py: start the upload client thread at startup, add GET /api/upload/status and POST /api/upload/dashcam_toggle routes. - dash.html/dash.js: add a "Remote server upload" card with upload status and a toggle switch to pause dashcam video uploads while trip/GPS log uploads keep running (per user request - useful on a slow/metered connection). - db/upload_creds.env: new gitignore-exempted template (mirrors db/creds.env) for YARIS_SERVER_URL/YARIS_DEVICE_NAME/YARIS_API_KEY. - requirements.txt: add requests. Tested against the real deployed unraid server: tracking batch upload + dedupe, dashcam upload + dedupe, and toggle-off correctly pausing dashcam uploads while tracking uploads continue. Also verified the full Flask app imports/runs cleanly and the new API routes + dash.html toggle render correctly in a headless browser. Co-authored-by: Copilot <[email protected]>
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
"""Uploads locally-queued trip/GPS tracking rows and dashcam files to the
|
||||
remote server (see server/ in this repo) over HTTP, with retry/backoff.
|
||||
|
||||
Tracking-log uploads always run when configured. Dashcam file uploads can be
|
||||
paused independently via a persisted toggle (settings table in the same
|
||||
SQLite db used for tracking) so a user can stop shipping large video files
|
||||
over a slow/metered connection while trip logs keep flowing.
|
||||
|
||||
Credentials/server URL are loaded from db/upload_creds.env (gitignored,
|
||||
never commit real values - mirrors the db/creds.env pattern already used by
|
||||
app_db.py for MongoDB).
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
from threading import Thread
|
||||
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from modules import app_sql, app_dash
|
||||
|
||||
_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
load_dotenv(os.path.join(_BASE_DIR, "db", "upload_creds.env"))
|
||||
|
||||
SERVER_URL = os.getenv("YARIS_SERVER_URL", "").rstrip("/")
|
||||
DEVICE_NAME = os.getenv("YARIS_DEVICE_NAME", "")
|
||||
API_KEY = os.getenv("YARIS_API_KEY", "")
|
||||
|
||||
UPLOAD_INTERVAL = 30 # seconds between upload attempts when idle/healthy
|
||||
TRACKING_BATCH_SIZE = 100
|
||||
BACKOFF_INITIAL = 5 # seconds, doubles on each consecutive failure
|
||||
BACKOFF_MAX = 300
|
||||
|
||||
DASHCAM_UPLOAD_SETTING_KEY = "dashcam_upload_enabled"
|
||||
|
||||
_upload_thread = None
|
||||
_last_error = None
|
||||
_last_success_at = None
|
||||
|
||||
|
||||
def _headers():
|
||||
return {"X-Device-Name": DEVICE_NAME, "X-Api-Key": API_KEY}
|
||||
|
||||
|
||||
def is_configured():
|
||||
return bool(SERVER_URL and DEVICE_NAME and API_KEY)
|
||||
|
||||
|
||||
def is_dashcam_upload_enabled():
|
||||
"""Defaults to enabled - the user opts out, not in."""
|
||||
return app_sql.get_setting(DASHCAM_UPLOAD_SETTING_KEY, "1") == "1"
|
||||
|
||||
|
||||
def set_dashcam_upload_enabled(enabled):
|
||||
app_sql.set_setting(DASHCAM_UPLOAD_SETTING_KEY, "1" if enabled else "0")
|
||||
|
||||
|
||||
def get_status():
|
||||
"""Snapshot used by the /api/upload/status endpoint for the web UI."""
|
||||
return {
|
||||
"configured": is_configured(),
|
||||
"dashcam_upload_enabled": is_dashcam_upload_enabled(),
|
||||
"pending_tracking_rows": app_sql.count_unsent_tracking_rows(),
|
||||
"last_error": _last_error,
|
||||
"last_success_at": _last_success_at,
|
||||
}
|
||||
|
||||
|
||||
def _sha256_file(path):
|
||||
hasher = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def _upload_tracking_batch():
|
||||
"""Send up to TRACKING_BATCH_SIZE not-yet-uploaded rows. Returns count sent."""
|
||||
rows = app_sql.get_unsent_tracking_rows(limit=TRACKING_BATCH_SIZE)
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
points = [
|
||||
{
|
||||
"recorded_at": row["datetime"],
|
||||
"gps_lat": row["gpsx"],
|
||||
"gps_lon": row["gpsy"],
|
||||
"speed": row["speed"],
|
||||
"odometer": row["odometer"],
|
||||
"vehicle_name": row["vn"],
|
||||
"driver_name": row["name"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
resp = requests.post(
|
||||
f"{SERVER_URL}/api/v1/tracking/batch",
|
||||
json={"points": points},
|
||||
headers=_headers(),
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
app_sql.mark_tracking_rows_uploaded([row["id"] for row in rows])
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _upload_dashcam_files():
|
||||
"""Upload any not-yet-uploaded dashcam files, one at a time. Returns count sent.
|
||||
|
||||
Skipped entirely (returns 0) while the dashcam-upload toggle is off -
|
||||
tracking-log uploads are unaffected by this toggle.
|
||||
"""
|
||||
if not is_dashcam_upload_enabled():
|
||||
return 0
|
||||
|
||||
uploaded_count = 0
|
||||
for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
|
||||
sha256 = _sha256_file(abs_path)
|
||||
if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
|
||||
continue
|
||||
|
||||
with open(abs_path, "rb") as f:
|
||||
resp = requests.post(
|
||||
f"{SERVER_URL}/api/v1/dashcam/upload",
|
||||
data={"relative_path": rel_path, "sha256": sha256},
|
||||
files={"file": f},
|
||||
headers=_headers(),
|
||||
timeout=300,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
app_sql.mark_dashcam_file_uploaded(rel_path, sha256)
|
||||
uploaded_count += 1
|
||||
|
||||
# Re-check the toggle between files so flipping it off mid-run stops
|
||||
# further dashcam uploads promptly without waiting for the next cycle.
|
||||
if not is_dashcam_upload_enabled():
|
||||
break
|
||||
|
||||
return uploaded_count
|
||||
|
||||
|
||||
def _upload_loop():
|
||||
global _last_error, _last_success_at
|
||||
import datetime as _dt
|
||||
|
||||
backoff = BACKOFF_INITIAL
|
||||
while True:
|
||||
if not is_configured():
|
||||
time.sleep(UPLOAD_INTERVAL)
|
||||
continue
|
||||
try:
|
||||
sent = _upload_tracking_batch()
|
||||
uploaded = _upload_dashcam_files()
|
||||
if sent or uploaded:
|
||||
print(f"Upload: sent {sent} tracking rows, {uploaded} dashcam files")
|
||||
_last_error = None
|
||||
_last_success_at = _dt.datetime.now().isoformat()
|
||||
backoff = BACKOFF_INITIAL
|
||||
time.sleep(UPLOAD_INTERVAL)
|
||||
except requests.RequestException as e:
|
||||
_last_error = str(e)
|
||||
print(f"Upload failed ({e}), retrying in {backoff}s")
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, BACKOFF_MAX)
|
||||
|
||||
|
||||
def start_uploading():
|
||||
"""Start the background upload thread, if the server is configured."""
|
||||
global _upload_thread
|
||||
if not is_configured():
|
||||
print(
|
||||
"Upload client not configured (set YARIS_SERVER_URL/YARIS_DEVICE_NAME/"
|
||||
"YARIS_API_KEY in db/upload_creds.env) - uploads disabled."
|
||||
)
|
||||
return
|
||||
if _upload_thread is None or not _upload_thread.is_alive():
|
||||
_upload_thread = Thread(target=_upload_loop, daemon=True)
|
||||
_upload_thread.start()
|
||||
print("Upload-client thread started.")
|
||||
else:
|
||||
print("Upload-client thread already running.")
|
||||
Reference in New Issue
Block a user