Add phone-relay dashcam upload to bypass the Pi's slow WiFi

Root-caused the slow-upload complaint to the Pi's own congested 2.4GHz
WiFi radio (confirmed via iperf3 - same degraded throughput even on
pure local LAN traffic, ruling out Cloudflare/ISP). Since the Pi 3 B
has no 5GHz radio, add a way to bypass it for the internet-bound leg:
a phone opens /dash and manually relays pending dashcam files straight
to the remote server over its own network, reusing the exact same
chunked upload protocol the Pi's client already uses.

- server/app.py: add CORS headers (after_request) so the phone's
  browser can call the remote server cross-origin from a page served
  by the Pi.
- app.py: new endpoints - GET /api/dash/pending (list not-yet-uploaded
  files), GET /api/dash/file/<path> (Range-request-capable download,
  path-traversal safe), GET /api/dash/relay/config (hands the phone
  the device credentials), POST /api/dash/relay/heartbeat|stop (lease-
  based pause of the Pi's own background uploader, self-heals if the
  phone's tab closes uncleanly), POST /api/dash/relay/mark_uploaded.
- modules/app_upload.py: list_pending_dashcam_files(), relay lease
  helpers, _upload_dashcam_files() now pauses while a relay lease is
  active.
- modules/app_sql.py: new dashcam_hash_cache table so a file's sha256
  is only computed once (keyed by path+size+mtime) instead of being
  recomputed on every loop iteration/pending-list call - benefits the
  existing background uploader too.
- templates/dash.html / static/js/dash.js: new "Relay via phone" card
  - per-file and "relay all" buttons (manual trigger only, per user
  preference), downloads+uploads one 40MB slice at a time so the
  phone never holds a whole file in memory, retries chunks up to 3x.
- PLAN.md/readme.md/server/README.md updated (Phase 11).

Tested via a full local end-to-end simulation driving the real Pi and
server Flask apps side by side (Postgres mocked only at the db module
boundary - all HTTP/chunking/auth/CORS logic is real): verified Range
downloads (206), byte-for-byte reassembly via SHA256, path-traversal
rejection, and the relay lease correctly pausing/resuming the
background uploader. Not yet tested against a real phone/live network.
This commit is contained in:
Mathieu Moran
2026-08-09 00:15:27 -04:00
parent 8c33c669a5
commit 0d67a21ae1
9 changed files with 549 additions and 13 deletions
+41
View File
@@ -57,6 +57,18 @@ SQL_CREATE_DASHCAM_UPLOADS_TABLE = """CREATE TABLE IF NOT EXISTS dashcam_uploads
UNIQUE(RELATIVE_PATH, SHA256)
);"""
# Cache of already-computed sha256 hashes, keyed by (path, size, mtime) so a
# file's hash never needs recomputing once copied from the dashcam (its size
# and mtime won't change afterward). Avoids re-hashing the entire, possibly
# 20GB+, local backlog on every background-upload loop iteration and every
# "pending files" list request from the phone-relay feature.
SQL_CREATE_DASHCAM_HASH_CACHE_TABLE = """CREATE TABLE IF NOT EXISTS dashcam_hash_cache (
RELATIVE_PATH text PRIMARY KEY,
SIZE integer NOT NULL,
MTIME real NOT NULL,
SHA256 text NOT NULL
);"""
def create_conn(db_file):
"""Open a SQLite connection to db_file, creating its parent folder if needed.
@@ -98,6 +110,7 @@ def get_conn():
create_table(_conn, SQL_CREATE_DTC_TABLE)
create_table(_conn, SQL_CREATE_SETTINGS_TABLE)
create_table(_conn, SQL_CREATE_DASHCAM_UPLOADS_TABLE)
create_table(_conn, SQL_CREATE_DASHCAM_HASH_CACHE_TABLE)
_migrate_tracking_table(_conn)
else:
print("Error! cannot create the database connection.")
@@ -221,6 +234,34 @@ def mark_dashcam_file_uploaded(relative_path, sha256):
conn.commit()
def get_cached_dashcam_hash(relative_path, size, mtime):
"""Return a previously-computed sha256 for this exact (path, size, mtime),
or None if not cached / the file has changed since it was last hashed."""
conn = get_conn()
if conn is None:
return None
with _conn_lock:
cur = conn.execute(
"SELECT SHA256 FROM dashcam_hash_cache WHERE RELATIVE_PATH = ? AND SIZE = ? AND MTIME = ?",
(relative_path, size, mtime),
)
row = cur.fetchone()
return row[0] if row is not None else None
def set_cached_dashcam_hash(relative_path, size, mtime, sha256):
conn = get_conn()
if conn is None:
return
with _conn_lock:
conn.execute(
"INSERT INTO dashcam_hash_cache (RELATIVE_PATH, SIZE, MTIME, SHA256) VALUES (?, ?, ?, ?) "
"ON CONFLICT(RELATIVE_PATH) DO UPDATE SET SIZE = excluded.SIZE, MTIME = excluded.MTIME, SHA256 = excluded.SHA256",
(relative_path, size, mtime, sha256),
)
conn.commit()
def _log_loop():
# Deferred import to avoid a circular import at module load time
# (app_obd.py imports app_sql, and app_sql needs app_obd.speed here).
+68 -7
View File
@@ -67,10 +67,17 @@ DASHCAM_CHUNK_SIZE = 40 * 1024 * 1024
DASHCAM_UPLOAD_SETTING_KEY = "dashcam_upload_enabled"
# How long a phone-relay "lease" stays valid without a fresh heartbeat before
# the Pi's own background uploader assumes the phone gave up/closed its tab
# and resumes uploading dashcam files itself. The phone page is expected to
# heartbeat well under this (every ~15-20s) - see /api/dash/relay/heartbeat.
RELAY_LEASE_SECONDS = 45
_upload_thread = None
_last_error = None
_last_success_at = None
_last_network_path = None # "wifi" or "cellular" - last path a batch actually went out on
_relay_lease_until = None # monotonic time.time() the phone-relay lease expires, or None
def _headers():
@@ -100,9 +107,30 @@ def get_status():
"last_success_at": _last_success_at,
"last_network_path": _last_network_path,
"cellular_fallback_configured": app_cellular.is_configured(),
"relay_active": is_relay_active(),
}
# --- Phone-relay lease (pauses the Pi's own background dashcam uploader
# while a phone is manually relaying files, so they don't fight over the
# Pi's single, often WiFi-congested, radio) -------------------------------
def relay_heartbeat():
"""Called by the phone-relay page every ~15-20s while active."""
global _relay_lease_until
_relay_lease_until = time.time() + RELAY_LEASE_SECONDS
def relay_stop():
"""Called when the phone-relay page is done/closed cleanly."""
global _relay_lease_until
_relay_lease_until = None
def is_relay_active():
return _relay_lease_until is not None and time.time() < _relay_lease_until
def _sha256_file(path):
hasher = hashlib.sha256()
with open(path, "rb") as f:
@@ -114,6 +142,37 @@ def _sha256_file(path):
return hasher.hexdigest()
def _hash_dashcam_file(abs_path, rel_path):
"""sha256 of a dashcam file, cached by (path, size, mtime) so unchanged
files already copied from the dashcam never get rehashed - real backlogs
here run 20GB+, and both the background loop and the phone-relay pending
list would otherwise rehash everything, including already-uploaded files,
on every single call."""
stat = os.stat(abs_path)
cached = app_sql.get_cached_dashcam_hash(rel_path, stat.st_size, stat.st_mtime)
if cached is not None:
return cached
sha256 = _sha256_file(abs_path)
app_sql.set_cached_dashcam_hash(rel_path, stat.st_size, stat.st_mtime, sha256)
return sha256
def list_pending_dashcam_files():
"""Return [{relative_path, sha256, size}, ...] for dashcam files not yet
confirmed uploaded - used by the phone-relay page to pick what to fetch."""
pending = []
for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
sha256 = _hash_dashcam_file(abs_path, rel_path)
if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
continue
pending.append({
"relative_path": rel_path,
"sha256": sha256,
"size": os.path.getsize(abs_path),
})
return pending
def _upload_tracking_batch():
"""Send up to TRACKING_BATCH_SIZE not-yet-uploaded rows. Returns count sent.
@@ -231,15 +290,16 @@ def _upload_video_chunked(abs_path, rel_path, sha256):
def _upload_dashcam_files():
"""Upload any not-yet-uploaded dashcam files, one at a time. Returns count sent.
Skipped entirely (returns 0) while the dashcam-upload toggle is off -
tracking-log uploads are unaffected by this toggle.
Skipped entirely (returns 0) while the dashcam-upload toggle is off, or
while a phone-relay session is actively leased - tracking-log uploads are
unaffected by either of these.
"""
if not is_dashcam_upload_enabled():
if not is_dashcam_upload_enabled() or is_relay_active():
return 0
uploaded_count = 0
for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
sha256 = _sha256_file(abs_path)
sha256 = _hash_dashcam_file(abs_path, rel_path)
if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
continue
@@ -247,9 +307,10 @@ def _upload_dashcam_files():
app_sql.mark_dashcam_file_uploaded(rel_path, sha256)
uploaded_count += 1
# Re-check the toggle between files so flipping it off mid-run stops
# further dashcam uploads promptly without waiting for the next cycle.
if not is_dashcam_upload_enabled():
# Re-check between files so flipping the toggle off, or a phone
# relay starting, mid-run stops further uploads promptly without
# waiting for the next cycle.
if not is_dashcam_upload_enabled() or is_relay_active():
break
return uploaded_count