yaris-obd/modules/app_upload.py

"""Uploads locally-queued trip/GPS tracking rows and dashcam files to the
remote server (see server/ in this repo) over HTTP, with retry/backoff.

Tracking-log uploads always run when configured. Dashcam file uploads can be
paused independently via a persisted toggle (settings table in the same
SQLite db used for tracking) so a user can stop shipping large video files
over a slow/metered connection while trip logs keep flowing.

Credentials/server URL are loaded from db/upload_creds.env (gitignored,
never commit real values - mirrors the db/creds.env pattern already used by
app_db.py for MongoDB).
"""
import hashlib
import os
import time
from threading import Thread

import requests
from dotenv import load_dotenv

from modules import app_sql, app_dash

_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
load_dotenv(os.path.join(_BASE_DIR, "db", "upload_creds.env"))

SERVER_URL = os.getenv("YARIS_SERVER_URL", "").rstrip("/")
DEVICE_NAME = os.getenv("YARIS_DEVICE_NAME", "")
API_KEY = os.getenv("YARIS_API_KEY", "")

UPLOAD_INTERVAL = 30       # seconds between upload attempts once caught up
CATCHUP_INTERVAL = 1       # seconds between batches while draining a backlog
TRACKING_BATCH_SIZE = 500
BACKOFF_INITIAL = 5        # seconds, doubles on each consecutive failure
BACKOFF_MAX = 300

DASHCAM_UPLOAD_SETTING_KEY = "dashcam_upload_enabled"

_upload_thread = None
_last_error = None
_last_success_at = None


def _headers():
    return {"X-Device-Name": DEVICE_NAME, "X-Api-Key": API_KEY}


def is_configured():
    return bool(SERVER_URL and DEVICE_NAME and API_KEY)


def is_dashcam_upload_enabled():
    """Defaults to enabled - the user opts out, not in."""
    return app_sql.get_setting(DASHCAM_UPLOAD_SETTING_KEY, "1") == "1"


def set_dashcam_upload_enabled(enabled):
    app_sql.set_setting(DASHCAM_UPLOAD_SETTING_KEY, "1" if enabled else "0")


def get_status():
    """Snapshot used by the /api/upload/status endpoint for the web UI."""
    return {
        "configured": is_configured(),
        "dashcam_upload_enabled": is_dashcam_upload_enabled(),
        "pending_tracking_rows": app_sql.count_unsent_tracking_rows(),
        "last_error": _last_error,
        "last_success_at": _last_success_at,
    }


def _sha256_file(path):
    hasher = hashlib.sha256()
    with open(path, "rb") as f:
        while True:
            chunk = f.read(1024 * 1024)
            if not chunk:
                break
            hasher.update(chunk)
    return hasher.hexdigest()


def _upload_tracking_batch():
    """Send up to TRACKING_BATCH_SIZE not-yet-uploaded rows. Returns count sent."""
    rows = app_sql.get_unsent_tracking_rows(limit=TRACKING_BATCH_SIZE)
    if not rows:
        return 0

    points = [
        {
            "recorded_at": row["datetime"],
            "gps_lat": row["gpsx"],
            "gps_lon": row["gpsy"],
            "speed": row["speed"],
            "odometer": row["odometer"],
            "vehicle_name": row["vn"],
            "driver_name": row["name"],
        }
        for row in rows
    ]

    resp = requests.post(
        f"{SERVER_URL}/api/v1/tracking/batch",
        json={"points": points},
        headers=_headers(),
        timeout=15,
    )
    resp.raise_for_status()
    app_sql.mark_tracking_rows_uploaded([row["id"] for row in rows])
    return len(rows)


def _upload_dashcam_files():
    """Upload any not-yet-uploaded dashcam files, one at a time. Returns count sent.

    Skipped entirely (returns 0) while the dashcam-upload toggle is off -
    tracking-log uploads are unaffected by this toggle.
    """
    if not is_dashcam_upload_enabled():
        return 0

    uploaded_count = 0
    for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
        sha256 = _sha256_file(abs_path)
        if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
            continue

        with open(abs_path, "rb") as f:
            resp = requests.post(
                f"{SERVER_URL}/api/v1/dashcam/upload",
                data={"relative_path": rel_path, "sha256": sha256},
                files={"file": f},
                headers=_headers(),
                timeout=300,
            )
        resp.raise_for_status()
        app_sql.mark_dashcam_file_uploaded(rel_path, sha256)
        uploaded_count += 1

        # Re-check the toggle between files so flipping it off mid-run stops
        # further dashcam uploads promptly without waiting for the next cycle.
        if not is_dashcam_upload_enabled():
            break

    return uploaded_count


def _upload_loop():
    global _last_error, _last_success_at
    import datetime as _dt

    backoff = BACKOFF_INITIAL
    while True:
        if not is_configured():
            time.sleep(UPLOAD_INTERVAL)
            continue
        try:
            sent = _upload_tracking_batch()
            uploaded = _upload_dashcam_files()
            if sent or uploaded:
                print(f"Upload: sent {sent} tracking rows, {uploaded} dashcam files")
            _last_error = None
            _last_success_at = _dt.datetime.now().isoformat()
            backoff = BACKOFF_INITIAL

            # A full batch means there's likely more backlog queued up (the
            # tracking table logs a new row every 5s, so a stale/offline
            # device can build up thousands of rows quickly) - keep draining
            # back-to-back instead of idling for the full interval. Only
            # settle into the slow/idle cadence once a batch comes back
            # short, i.e. we're actually caught up.
            if sent >= TRACKING_BATCH_SIZE:
                time.sleep(CATCHUP_INTERVAL)
            else:
                time.sleep(UPLOAD_INTERVAL)
        except requests.RequestException as e:
            _last_error = str(e)
            print(f"Upload failed ({e}), retrying in {backoff}s")
            time.sleep(backoff)
            backoff = min(backoff * 2, BACKOFF_MAX)


def start_uploading():
    """Start the background upload thread, if the server is configured."""
    global _upload_thread
    if not is_configured():
        print(
            "Upload client not configured (set YARIS_SERVER_URL/YARIS_DEVICE_NAME/"
            "YARIS_API_KEY in db/upload_creds.env) - uploads disabled."
        )
        return
    if _upload_thread is None or not _upload_thread.is_alive():
        _upload_thread = Thread(target=_upload_loop, daemon=True)
        _upload_thread.start()
        print("Upload-client thread started.")
    else:
        print("Upload-client thread already running.")