Add phone-relay dashcam upload to bypass the Pi's slow WiFi
Root-caused the slow-upload complaint to the Pi's own congested 2.4GHz WiFi radio (confirmed via iperf3 - same degraded throughput even on pure local LAN traffic, ruling out Cloudflare/ISP). Since the Pi 3 B has no 5GHz radio, add a way to bypass it for the internet-bound leg: a phone opens /dash and manually relays pending dashcam files straight to the remote server over its own network, reusing the exact same chunked upload protocol the Pi's client already uses. - server/app.py: add CORS headers (after_request) so the phone's browser can call the remote server cross-origin from a page served by the Pi. - app.py: new endpoints - GET /api/dash/pending (list not-yet-uploaded files), GET /api/dash/file/<path> (Range-request-capable download, path-traversal safe), GET /api/dash/relay/config (hands the phone the device credentials), POST /api/dash/relay/heartbeat|stop (lease- based pause of the Pi's own background uploader, self-heals if the phone's tab closes uncleanly), POST /api/dash/relay/mark_uploaded. - modules/app_upload.py: list_pending_dashcam_files(), relay lease helpers, _upload_dashcam_files() now pauses while a relay lease is active. - modules/app_sql.py: new dashcam_hash_cache table so a file's sha256 is only computed once (keyed by path+size+mtime) instead of being recomputed on every loop iteration/pending-list call - benefits the existing background uploader too. - templates/dash.html / static/js/dash.js: new "Relay via phone" card - per-file and "relay all" buttons (manual trigger only, per user preference), downloads+uploads one 40MB slice at a time so the phone never holds a whole file in memory, retries chunks up to 3x. - PLAN.md/readme.md/server/README.md updated (Phase 11). Tested via a full local end-to-end simulation driving the real Pi and server Flask apps side by side (Postgres mocked only at the db module boundary - all HTTP/chunking/auth/CORS logic is real): verified Range downloads (206), byte-for-byte reassembly via SHA256, path-traversal rejection, and the relay lease correctly pausing/resuming the background uploader. Not yet tested against a real phone/live network.
This commit is contained in:
@@ -57,6 +57,18 @@ SQL_CREATE_DASHCAM_UPLOADS_TABLE = """CREATE TABLE IF NOT EXISTS dashcam_uploads
|
||||
UNIQUE(RELATIVE_PATH, SHA256)
|
||||
);"""
|
||||
|
||||
# Cache of already-computed sha256 hashes, keyed by (path, size, mtime) so a
|
||||
# file's hash never needs recomputing once copied from the dashcam (its size
|
||||
# and mtime won't change afterward). Avoids re-hashing the entire, possibly
|
||||
# 20GB+, local backlog on every background-upload loop iteration and every
|
||||
# "pending files" list request from the phone-relay feature.
|
||||
SQL_CREATE_DASHCAM_HASH_CACHE_TABLE = """CREATE TABLE IF NOT EXISTS dashcam_hash_cache (
|
||||
RELATIVE_PATH text PRIMARY KEY,
|
||||
SIZE integer NOT NULL,
|
||||
MTIME real NOT NULL,
|
||||
SHA256 text NOT NULL
|
||||
);"""
|
||||
|
||||
|
||||
def create_conn(db_file):
|
||||
"""Open a SQLite connection to db_file, creating its parent folder if needed.
|
||||
@@ -98,6 +110,7 @@ def get_conn():
|
||||
create_table(_conn, SQL_CREATE_DTC_TABLE)
|
||||
create_table(_conn, SQL_CREATE_SETTINGS_TABLE)
|
||||
create_table(_conn, SQL_CREATE_DASHCAM_UPLOADS_TABLE)
|
||||
create_table(_conn, SQL_CREATE_DASHCAM_HASH_CACHE_TABLE)
|
||||
_migrate_tracking_table(_conn)
|
||||
else:
|
||||
print("Error! cannot create the database connection.")
|
||||
@@ -221,6 +234,34 @@ def mark_dashcam_file_uploaded(relative_path, sha256):
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_cached_dashcam_hash(relative_path, size, mtime):
|
||||
"""Return a previously-computed sha256 for this exact (path, size, mtime),
|
||||
or None if not cached / the file has changed since it was last hashed."""
|
||||
conn = get_conn()
|
||||
if conn is None:
|
||||
return None
|
||||
with _conn_lock:
|
||||
cur = conn.execute(
|
||||
"SELECT SHA256 FROM dashcam_hash_cache WHERE RELATIVE_PATH = ? AND SIZE = ? AND MTIME = ?",
|
||||
(relative_path, size, mtime),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return row[0] if row is not None else None
|
||||
|
||||
|
||||
def set_cached_dashcam_hash(relative_path, size, mtime, sha256):
|
||||
conn = get_conn()
|
||||
if conn is None:
|
||||
return
|
||||
with _conn_lock:
|
||||
conn.execute(
|
||||
"INSERT INTO dashcam_hash_cache (RELATIVE_PATH, SIZE, MTIME, SHA256) VALUES (?, ?, ?, ?) "
|
||||
"ON CONFLICT(RELATIVE_PATH) DO UPDATE SET SIZE = excluded.SIZE, MTIME = excluded.MTIME, SHA256 = excluded.SHA256",
|
||||
(relative_path, size, mtime, sha256),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _log_loop():
|
||||
# Deferred import to avoid a circular import at module load time
|
||||
# (app_obd.py imports app_sql, and app_sql needs app_obd.speed here).
|
||||
|
||||
+68
-7
@@ -67,10 +67,17 @@ DASHCAM_CHUNK_SIZE = 40 * 1024 * 1024
|
||||
|
||||
DASHCAM_UPLOAD_SETTING_KEY = "dashcam_upload_enabled"
|
||||
|
||||
# How long a phone-relay "lease" stays valid without a fresh heartbeat before
|
||||
# the Pi's own background uploader assumes the phone gave up/closed its tab
|
||||
# and resumes uploading dashcam files itself. The phone page is expected to
|
||||
# heartbeat well under this (every ~15-20s) - see /api/dash/relay/heartbeat.
|
||||
RELAY_LEASE_SECONDS = 45
|
||||
|
||||
_upload_thread = None
|
||||
_last_error = None
|
||||
_last_success_at = None
|
||||
_last_network_path = None # "wifi" or "cellular" - last path a batch actually went out on
|
||||
_relay_lease_until = None # monotonic time.time() the phone-relay lease expires, or None
|
||||
|
||||
|
||||
def _headers():
|
||||
@@ -100,9 +107,30 @@ def get_status():
|
||||
"last_success_at": _last_success_at,
|
||||
"last_network_path": _last_network_path,
|
||||
"cellular_fallback_configured": app_cellular.is_configured(),
|
||||
"relay_active": is_relay_active(),
|
||||
}
|
||||
|
||||
|
||||
# --- Phone-relay lease (pauses the Pi's own background dashcam uploader
|
||||
# while a phone is manually relaying files, so they don't fight over the
|
||||
# Pi's single, often WiFi-congested, radio) -------------------------------
|
||||
|
||||
def relay_heartbeat():
|
||||
"""Called by the phone-relay page every ~15-20s while active."""
|
||||
global _relay_lease_until
|
||||
_relay_lease_until = time.time() + RELAY_LEASE_SECONDS
|
||||
|
||||
|
||||
def relay_stop():
|
||||
"""Called when the phone-relay page is done/closed cleanly."""
|
||||
global _relay_lease_until
|
||||
_relay_lease_until = None
|
||||
|
||||
|
||||
def is_relay_active():
|
||||
return _relay_lease_until is not None and time.time() < _relay_lease_until
|
||||
|
||||
|
||||
def _sha256_file(path):
|
||||
hasher = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
@@ -114,6 +142,37 @@ def _sha256_file(path):
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def _hash_dashcam_file(abs_path, rel_path):
|
||||
"""sha256 of a dashcam file, cached by (path, size, mtime) so unchanged
|
||||
files already copied from the dashcam never get rehashed - real backlogs
|
||||
here run 20GB+, and both the background loop and the phone-relay pending
|
||||
list would otherwise rehash everything, including already-uploaded files,
|
||||
on every single call."""
|
||||
stat = os.stat(abs_path)
|
||||
cached = app_sql.get_cached_dashcam_hash(rel_path, stat.st_size, stat.st_mtime)
|
||||
if cached is not None:
|
||||
return cached
|
||||
sha256 = _sha256_file(abs_path)
|
||||
app_sql.set_cached_dashcam_hash(rel_path, stat.st_size, stat.st_mtime, sha256)
|
||||
return sha256
|
||||
|
||||
|
||||
def list_pending_dashcam_files():
|
||||
"""Return [{relative_path, sha256, size}, ...] for dashcam files not yet
|
||||
confirmed uploaded - used by the phone-relay page to pick what to fetch."""
|
||||
pending = []
|
||||
for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
|
||||
sha256 = _hash_dashcam_file(abs_path, rel_path)
|
||||
if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
|
||||
continue
|
||||
pending.append({
|
||||
"relative_path": rel_path,
|
||||
"sha256": sha256,
|
||||
"size": os.path.getsize(abs_path),
|
||||
})
|
||||
return pending
|
||||
|
||||
|
||||
def _upload_tracking_batch():
|
||||
"""Send up to TRACKING_BATCH_SIZE not-yet-uploaded rows. Returns count sent.
|
||||
|
||||
@@ -231,15 +290,16 @@ def _upload_video_chunked(abs_path, rel_path, sha256):
|
||||
def _upload_dashcam_files():
|
||||
"""Upload any not-yet-uploaded dashcam files, one at a time. Returns count sent.
|
||||
|
||||
Skipped entirely (returns 0) while the dashcam-upload toggle is off -
|
||||
tracking-log uploads are unaffected by this toggle.
|
||||
Skipped entirely (returns 0) while the dashcam-upload toggle is off, or
|
||||
while a phone-relay session is actively leased - tracking-log uploads are
|
||||
unaffected by either of these.
|
||||
"""
|
||||
if not is_dashcam_upload_enabled():
|
||||
if not is_dashcam_upload_enabled() or is_relay_active():
|
||||
return 0
|
||||
|
||||
uploaded_count = 0
|
||||
for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
|
||||
sha256 = _sha256_file(abs_path)
|
||||
sha256 = _hash_dashcam_file(abs_path, rel_path)
|
||||
if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
|
||||
continue
|
||||
|
||||
@@ -247,9 +307,10 @@ def _upload_dashcam_files():
|
||||
app_sql.mark_dashcam_file_uploaded(rel_path, sha256)
|
||||
uploaded_count += 1
|
||||
|
||||
# Re-check the toggle between files so flipping it off mid-run stops
|
||||
# further dashcam uploads promptly without waiting for the next cycle.
|
||||
if not is_dashcam_upload_enabled():
|
||||
# Re-check between files so flipping the toggle off, or a phone
|
||||
# relay starting, mid-run stops further uploads promptly without
|
||||
# waiting for the next cycle.
|
||||
if not is_dashcam_upload_enabled() or is_relay_active():
|
||||
break
|
||||
|
||||
return uploaded_count
|
||||
|
||||
Reference in New Issue
Block a user