yaris-obd/server/app.py

"""yaris-obd remote server: receives trip/GPS log batches and dashcam files
uploaded by the Pi, storing them in Postgres + on-disk file storage.
"""
import hashlib
import os
import shutil

from flask import Flask, request, jsonify, g

from modules import db
from modules.auth import require_device_auth

app = Flask(__name__)

# Where uploaded dashcam files get written on disk (mounted to the unraid
# array via docker-compose - see DASHCAM_STORAGE_DIR in .env).
STORAGE_DIR = os.environ.get("DASHCAM_STORAGE_DIR", "/data/dashcam")
MAX_DASHCAM_UPLOAD_MB = int(os.environ.get("MAX_DASHCAM_UPLOAD_MB", "2048"))
app.config["MAX_CONTENT_LENGTH"] = MAX_DASHCAM_UPLOAD_MB * 1024 * 1024

# Sanity cap on tracking points per request - the Pi client batches in
# groups of 500, this just guards against a runaway/misbehaving client.
MAX_TRACKING_BATCH_POINTS = 5000


@app.get("/health")
def health():
    """Unauthenticated liveness check - used by docker-compose healthcheck
    and to sanity-test connectivity from the Pi before attempting uploads."""
    return jsonify(status="ok"), 200


@app.post("/api/v1/tracking/batch")
@require_device_auth
def upload_tracking_batch():
    """Bulk-insert a batch of GPS/trip tracking points for the authenticated device.

    Expected JSON body: {"points": [{"recorded_at": "...", "gps_lat": ..., ...}, ...]}
    recorded_at must be an ISO-8601 timestamp string. Duplicate (device,
    recorded_at) pairs are silently ignored, so the Pi can safely re-send a
    batch after a network failure without creating duplicate rows.
    """
    body = request.get_json(silent=True) or {}
    points = body.get("points")
    if not isinstance(points, list) or not points:
        return jsonify(error="expected a non-empty 'points' list"), 400
    if len(points) > MAX_TRACKING_BATCH_POINTS:
        return jsonify(
            error=f"batch too large - max {MAX_TRACKING_BATCH_POINTS} points per request"
        ), 400

    for p in points:
        if "recorded_at" not in p:
            return jsonify(error="every point requires 'recorded_at'"), 400

    inserted = db.insert_tracking_points(g.device_id, points)
    return jsonify(status="ok", received=inserted), 201


@app.post("/api/v1/dashcam/upload")
@require_device_auth
def upload_dashcam_file():
    """Upload a single dashcam video file.

    Multipart form fields:
      file:          the video file itself
      relative_path: original sub-folder path on the dashcam SD card (form field)
      sha256:        hex digest of the file, computed by the Pi before sending

    Skips (returns 200, not re-saved) if a file with the same
    (device, relative_path, sha256) was already uploaded - safe to retry.
    """
    if "file" not in request.files:
        return jsonify(error="missing 'file' in multipart form"), 400

    relative_path = request.form.get("relative_path", "").strip()
    expected_sha256 = request.form.get("sha256", "").strip().lower()
    if not relative_path or not expected_sha256:
        return jsonify(error="missing 'relative_path' or 'sha256' form field"), 400

    device_id = g.device_id
    if db.dashcam_file_exists(device_id, relative_path, expected_sha256):
        return jsonify(status="already_uploaded"), 200

    uploaded = request.files["file"]
    device_dir = os.path.join(STORAGE_DIR, g.device_name)
    dest_path = os.path.normpath(os.path.join(device_dir, relative_path))
    # Guard against a malicious/buggy relative_path (e.g. "../../etc") escaping
    # the per-device storage directory.
    if os.path.commonpath([device_dir, dest_path]) != os.path.normpath(device_dir):
        return jsonify(error="invalid relative_path"), 400
    os.makedirs(os.path.dirname(dest_path), exist_ok=True)

    hasher = hashlib.sha256()
    tmp_path = dest_path + ".part"
    size_bytes = 0
    with open(tmp_path, "wb") as f:
        while True:
            chunk = uploaded.stream.read(1024 * 1024)
            if not chunk:
                break
            hasher.update(chunk)
            size_bytes += len(chunk)
            f.write(chunk)

    actual_sha256 = hasher.hexdigest()
    if actual_sha256 != expected_sha256:
        os.remove(tmp_path)
        return jsonify(error="sha256 mismatch, upload corrupted in transit"), 422

    os.replace(tmp_path, dest_path)
    db.insert_dashcam_file(device_id, relative_path, size_bytes, actual_sha256, dest_path)
    return jsonify(status="ok", size_bytes=size_bytes), 201


# --- Chunked dashcam upload ---------------------------------------------
# Real dashcam clips run 300+MB, well past Cloudflare Tunnel's ~100MB max
# request body size (confirmed with a real 250MB file: a direct LAN upload
# succeeds, the same request through the tunnel comes back 413). The Pi
# client splits files into small chunks (see CHUNK_SIZE in
# modules/app_upload.py) and PUTs them one at a time through this endpoint
# instead, staying comfortably under that limit. Chunks for a given upload
# are staged on disk under _CHUNK_TMP_DIR and reassembled once complete.

_CHUNK_TMP_DIRNAME = ".upload_tmp"


def _chunk_tmp_dir(device_name, sha256):
    return os.path.join(STORAGE_DIR, _CHUNK_TMP_DIRNAME, device_name, sha256)


def _received_chunk_indices(tmp_dir):
    if not os.path.isdir(tmp_dir):
        return set()
    indices = set()
    for name in os.listdir(tmp_dir):
        if name.endswith(".chunk"):
            try:
                indices.add(int(name[: -len(".chunk")]))
            except ValueError:
                continue
    return indices


@app.get("/api/v1/dashcam/upload/status")
@require_device_auth
def dashcam_upload_status():
    """Report upload progress for (relative_path, sha256) so the Pi client
    can resume a chunked upload without re-sending chunks the server
    already has (e.g. after a network drop or app restart mid-transfer)."""
    relative_path = request.args.get("relative_path", "").strip()
    sha256 = request.args.get("sha256", "").strip().lower()
    if not relative_path or not sha256:
        return jsonify(error="missing 'relative_path' or 'sha256' query param"), 400

    if db.dashcam_file_exists(g.device_id, relative_path, sha256):
        return jsonify(status="already_uploaded"), 200

    tmp_dir = _chunk_tmp_dir(g.device_name, sha256)
    received = sorted(_received_chunk_indices(tmp_dir))
    return jsonify(status="in_progress" if received else "not_started", received_chunk_indices=received), 200


@app.post("/api/v1/dashcam/upload/chunk")
@require_device_auth
def dashcam_upload_chunk():
    """Receive one chunk of a dashcam video file.

    Multipart form fields:
      file:          the chunk's bytes
      relative_path: original sub-folder path on the dashcam SD card
      sha256:        hex digest of the *whole* (reassembled) file
      chunk_index:   0-based index of this chunk
      total_chunks:  total number of chunks for this file

    Once every chunk 0..total_chunks-1 has arrived, the chunks are
    concatenated in order, the combined sha256 is verified, and the result
    is recorded exactly like the single-shot /upload endpoint.
    """
    if "file" not in request.files:
        return jsonify(error="missing 'file' in multipart form"), 400

    relative_path = request.form.get("relative_path", "").strip()
    expected_sha256 = request.form.get("sha256", "").strip().lower()
    try:
        chunk_index = int(request.form.get("chunk_index", ""))
        total_chunks = int(request.form.get("total_chunks", ""))
    except ValueError:
        return jsonify(error="'chunk_index'/'total_chunks' must be integers"), 400

    if not relative_path or not expected_sha256 or total_chunks < 1 or not (0 <= chunk_index < total_chunks):
        return jsonify(error="missing/invalid form fields"), 400

    device_id = g.device_id
    if db.dashcam_file_exists(device_id, relative_path, expected_sha256):
        return jsonify(status="already_uploaded"), 200

    device_dir = os.path.join(STORAGE_DIR, g.device_name)
    dest_path = os.path.normpath(os.path.join(device_dir, relative_path))
    if os.path.commonpath([device_dir, dest_path]) != os.path.normpath(device_dir):
        return jsonify(error="invalid relative_path"), 400

    tmp_dir = _chunk_tmp_dir(g.device_name, expected_sha256)
    # A stale tmp_dir left over from a previous attempt with a *different*
    # total_chunks (e.g. the client's chunk size changed between app
    # versions) would reassemble incorrectly - start clean if that happens.
    marker_path = os.path.join(tmp_dir, ".total_chunks")
    if os.path.isdir(tmp_dir) and os.path.exists(marker_path):
        with open(marker_path) as marker_file:
            if marker_file.read().strip() != str(total_chunks):
                shutil.rmtree(tmp_dir)

    os.makedirs(tmp_dir, exist_ok=True)
    with open(marker_path, "w") as marker_file:
        marker_file.write(str(total_chunks))

    chunk_path = os.path.join(tmp_dir, f"{chunk_index:06d}.chunk")
    uploaded = request.files["file"]
    with open(chunk_path, "wb") as f:
        while True:
            data = uploaded.stream.read(1024 * 1024)
            if not data:
                break
            f.write(data)

    received = _received_chunk_indices(tmp_dir)
    if len(received) < total_chunks:
        return jsonify(status="chunk_received", received_chunks=len(received), total_chunks=total_chunks), 202

    # All chunks in - reassemble, verify, and store.
    hasher = hashlib.sha256()
    tmp_path = dest_path + ".part"
    os.makedirs(os.path.dirname(dest_path), exist_ok=True)
    size_bytes = 0
    with open(tmp_path, "wb") as out:
        for idx in range(total_chunks):
            with open(os.path.join(tmp_dir, f"{idx:06d}.chunk"), "rb") as chunk_file:
                while True:
                    data = chunk_file.read(1024 * 1024)
                    if not data:
                        break
                    hasher.update(data)
                    size_bytes += len(data)
                    out.write(data)

    actual_sha256 = hasher.hexdigest()
    shutil.rmtree(tmp_dir, ignore_errors=True)
    if actual_sha256 != expected_sha256:
        os.remove(tmp_path)
        return jsonify(error="sha256 mismatch, upload corrupted in transit"), 422

    os.replace(tmp_path, dest_path)
    db.insert_dashcam_file(device_id, relative_path, size_bytes, actual_sha256, dest_path)
    return jsonify(status="complete", size_bytes=size_bytes), 201


def main():
    app.run(host="0.0.0.0", port=8098)


if __name__ == "__main__":
    main()