Mathieu MoranandCopilot
975526667e
The upload loop previously always slept the full 30s UPLOAD_INTERVAL between batches, even when a full batch was just sent (meaning more rows were likely still queued). With GPS/OBD rows logged every 5s locally, an offline/stale device could build up a large backlog that took a very long time to drain at 100 rows/30s. - Bump TRACKING_BATCH_SIZE 100 -> 500 (cuts round-trips per backlog). - Only sleep the full UPLOAD_INTERVAL once actually caught up (a short batch came back); while a full batch keeps coming back, drain back-to-back with a short CATCHUP_INTERVAL (1s) instead. - server/app.py: add a sane MAX_TRACKING_BATCH_POINTS (5000) cap on the tracking/batch endpoint as a defensive guard against a runaway client, independent of the Pi's own batch size. Verified against the real deployed server: a simulated 1250-row backlog now drains in 3 batches (~0.35s server round-trip time) instead of 13 batches spread across ~6.5 minutes of idle sleeping. Co-authored-by: Copilot <[email protected]>
121 lines
4.4 KiB
Python
121 lines
4.4 KiB
Python
"""yaris-obd remote server: receives trip/GPS log batches and dashcam files
|
|
uploaded by the Pi, storing them in Postgres + on-disk file storage.
|
|
"""
|
|
import hashlib
|
|
import os
|
|
|
|
from flask import Flask, request, jsonify, g
|
|
|
|
from modules import db
|
|
from modules.auth import require_device_auth
|
|
|
|
app = Flask(__name__)
|
|
|
|
# Where uploaded dashcam files get written on disk (mounted to the unraid
|
|
# array via docker-compose - see DASHCAM_STORAGE_DIR in .env).
|
|
STORAGE_DIR = os.environ.get("DASHCAM_STORAGE_DIR", "/data/dashcam")
|
|
MAX_DASHCAM_UPLOAD_MB = int(os.environ.get("MAX_DASHCAM_UPLOAD_MB", "2048"))
|
|
app.config["MAX_CONTENT_LENGTH"] = MAX_DASHCAM_UPLOAD_MB * 1024 * 1024
|
|
|
|
# Sanity cap on tracking points per request - the Pi client batches in
|
|
# groups of 500, this just guards against a runaway/misbehaving client.
|
|
MAX_TRACKING_BATCH_POINTS = 5000
|
|
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
"""Unauthenticated liveness check - used by docker-compose healthcheck
|
|
and to sanity-test connectivity from the Pi before attempting uploads."""
|
|
return jsonify(status="ok"), 200
|
|
|
|
|
|
@app.post("/api/v1/tracking/batch")
|
|
@require_device_auth
|
|
def upload_tracking_batch():
|
|
"""Bulk-insert a batch of GPS/trip tracking points for the authenticated device.
|
|
|
|
Expected JSON body: {"points": [{"recorded_at": "...", "gps_lat": ..., ...}, ...]}
|
|
recorded_at must be an ISO-8601 timestamp string. Duplicate (device,
|
|
recorded_at) pairs are silently ignored, so the Pi can safely re-send a
|
|
batch after a network failure without creating duplicate rows.
|
|
"""
|
|
body = request.get_json(silent=True) or {}
|
|
points = body.get("points")
|
|
if not isinstance(points, list) or not points:
|
|
return jsonify(error="expected a non-empty 'points' list"), 400
|
|
if len(points) > MAX_TRACKING_BATCH_POINTS:
|
|
return jsonify(
|
|
error=f"batch too large - max {MAX_TRACKING_BATCH_POINTS} points per request"
|
|
), 400
|
|
|
|
for p in points:
|
|
if "recorded_at" not in p:
|
|
return jsonify(error="every point requires 'recorded_at'"), 400
|
|
|
|
inserted = db.insert_tracking_points(g.device_id, points)
|
|
return jsonify(status="ok", received=inserted), 201
|
|
|
|
|
|
@app.post("/api/v1/dashcam/upload")
|
|
@require_device_auth
|
|
def upload_dashcam_file():
|
|
"""Upload a single dashcam video file.
|
|
|
|
Multipart form fields:
|
|
file: the video file itself
|
|
relative_path: original sub-folder path on the dashcam SD card (form field)
|
|
sha256: hex digest of the file, computed by the Pi before sending
|
|
|
|
Skips (returns 200, not re-saved) if a file with the same
|
|
(device, relative_path, sha256) was already uploaded - safe to retry.
|
|
"""
|
|
if "file" not in request.files:
|
|
return jsonify(error="missing 'file' in multipart form"), 400
|
|
|
|
relative_path = request.form.get("relative_path", "").strip()
|
|
expected_sha256 = request.form.get("sha256", "").strip().lower()
|
|
if not relative_path or not expected_sha256:
|
|
return jsonify(error="missing 'relative_path' or 'sha256' form field"), 400
|
|
|
|
device_id = g.device_id
|
|
if db.dashcam_file_exists(device_id, relative_path, expected_sha256):
|
|
return jsonify(status="already_uploaded"), 200
|
|
|
|
uploaded = request.files["file"]
|
|
device_dir = os.path.join(STORAGE_DIR, g.device_name)
|
|
dest_path = os.path.normpath(os.path.join(device_dir, relative_path))
|
|
# Guard against a malicious/buggy relative_path (e.g. "../../etc") escaping
|
|
# the per-device storage directory.
|
|
if os.path.commonpath([device_dir, dest_path]) != os.path.normpath(device_dir):
|
|
return jsonify(error="invalid relative_path"), 400
|
|
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
|
|
|
|
hasher = hashlib.sha256()
|
|
tmp_path = dest_path + ".part"
|
|
size_bytes = 0
|
|
with open(tmp_path, "wb") as f:
|
|
while True:
|
|
chunk = uploaded.stream.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
hasher.update(chunk)
|
|
size_bytes += len(chunk)
|
|
f.write(chunk)
|
|
|
|
actual_sha256 = hasher.hexdigest()
|
|
if actual_sha256 != expected_sha256:
|
|
os.remove(tmp_path)
|
|
return jsonify(error="sha256 mismatch, upload corrupted in transit"), 422
|
|
|
|
os.replace(tmp_path, dest_path)
|
|
db.insert_dashcam_file(device_id, relative_path, size_bytes, actual_sha256, dest_path)
|
|
return jsonify(status="ok", size_bytes=size_bytes), 201
|
|
|
|
|
|
def main():
|
|
app.run(host="0.0.0.0", port=8098)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|