Add local dashcam video library

Co-authored-by: Copilot <[email protected]>

modules/app_dash.py

@@ -33,6 +33,44 @@ DEFAULT_DESTINATION = os.path.join(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "dash", "videos"
 )
 _device_lock = threading.Lock()
+_local_video_state_lock = threading.Lock()
+_local_videos_in_use = {}
+_local_videos_deleting = set()
+
+
+def begin_local_video_use(relative_path):
+    with _local_video_state_lock:
+        if relative_path in _local_videos_deleting:
+            return False
+        _local_videos_in_use[relative_path] = (
+            _local_videos_in_use.get(relative_path, 0) + 1
+        )
+        return True
+
+
+def end_local_video_use(relative_path):
+    with _local_video_state_lock:
+        uses = _local_videos_in_use.get(relative_path, 0)
+        if uses <= 1:
+            _local_videos_in_use.pop(relative_path, None)
+        else:
+            _local_videos_in_use[relative_path] = uses - 1
+
+
+def begin_local_video_delete(relative_path):
+    with _local_video_state_lock:
+        if (
+            relative_path in _local_videos_in_use
+            or relative_path in _local_videos_deleting
+        ):
+            return False
+        _local_videos_deleting.add(relative_path)
+        return True
+
+
+def end_local_video_delete(relative_path):
+    with _local_video_state_lock:
+        _local_videos_deleting.discard(relative_path)
 
 
 def list_usb_storage_partitions():
@@ -275,7 +313,7 @@ def list_dashcam_files(mount_point=DEFAULT_MOUNT_POINT):
                 unmount_device(mount_point)
 
 
-def _safe_video_path(source_folder, relative_path):
+def safe_video_path(source_folder, relative_path):
     if not isinstance(relative_path, str) or not relative_path.strip():
         raise ValueError("A non-empty relative video path is required.")
     if os.path.isabs(relative_path):
@@ -337,7 +375,7 @@ def delete_dashcam_files(relative_path=None, delete_all=False, mount_point=DEFAU
                 targets = find_video_files(source)
             else:
                 try:
-                    target = _safe_video_path(source, relative_path)
+                    target = safe_video_path(source, relative_path)
                 except ValueError as e:
                     result = {"status": "invalid_path", "message": str(e)}
                 except FileNotFoundError:
@@ -353,7 +391,7 @@ def delete_dashcam_files(relative_path=None, delete_all=False, mount_point=DEFAU
                 errors = []
                 for _abs_path, rel_path in targets:
                     try:
-                        safe_path = _safe_video_path(source, rel_path)
+                        safe_path = safe_video_path(source, rel_path)
                         _remove_video_file(safe_path)
                         deleted.append(rel_path)
                     except (OSError, ValueError) as e:
@@ -413,7 +451,7 @@ def copy_files(source_folder, destination_folder, relative_paths=None):
                 continue
             seen.add(relative_path)
             try:
-                abs_path = _safe_video_path(source_folder, relative_path)
+                abs_path = safe_video_path(source_folder, relative_path)
             except ValueError as e:
                 return {
                     "found": 0,

modules/app_sql.py

@@ -222,6 +222,46 @@ def dashcam_file_marked_uploaded(relative_path, sha256):
         return cur.fetchone() is not None
 
 
+def dashcam_file_uploaded_for_stat(relative_path, size, mtime):
+    """Whether the current local file version is confirmed uploaded.
+
+    Uses the existing hash cache so list/retention scans never rehash large
+    video files merely to display their status.
+    """
+    conn = get_conn()
+    if conn is None:
+        return False
+    with _conn_lock:
+        cur = conn.execute(
+            """SELECT 1
+               FROM dashcam_hash_cache AS cache
+               JOIN dashcam_uploads AS uploads
+                 ON uploads.RELATIVE_PATH = cache.RELATIVE_PATH
+                AND uploads.SHA256 = cache.SHA256
+               WHERE cache.RELATIVE_PATH = ?
+                 AND cache.SIZE = ?
+                 AND cache.MTIME = ?""",
+            (relative_path, size, mtime),
+        )
+        return cur.fetchone() is not None
+
+
+def uploaded_dashcam_file_stats():
+    """Return exact uploaded file versions as (path, size, mtime) tuples."""
+    conn = get_conn()
+    if conn is None:
+        return set()
+    with _conn_lock:
+        cur = conn.execute(
+            """SELECT cache.RELATIVE_PATH, cache.SIZE, cache.MTIME
+               FROM dashcam_hash_cache AS cache
+               JOIN dashcam_uploads AS uploads
+                 ON uploads.RELATIVE_PATH = cache.RELATIVE_PATH
+                AND uploads.SHA256 = cache.SHA256"""
+        )
+        return set(cur.fetchall())
+
+
 def mark_dashcam_file_uploaded(relative_path, sha256):
     conn = get_conn()
     if conn is None:

modules/app_upload.py

@@ -43,7 +43,7 @@ import requests
 from dotenv import load_dotenv
 from requests_toolbelt import MultipartEncoder
 
-from modules import app_sql, app_dash, app_cellular
+from modules import app_sql, app_dash, app_cellular, app_video
 
 _BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 load_dotenv(os.path.join(_BASE_DIR, "db", "upload_creds.env"))
@@ -253,13 +253,20 @@ def _upload_dashcam_files():
 
     uploaded_count = 0
     for abs_path, rel_path in app_dash.find_video_files(app_dash.DEFAULT_DESTINATION):
-        sha256 = _hash_dashcam_file(abs_path, rel_path)
-        if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
+        if not app_dash.begin_local_video_use(rel_path):
             continue
+        try:
+            if not os.path.isfile(abs_path):
+                continue
+            sha256 = _hash_dashcam_file(abs_path, rel_path)
+            if app_sql.dashcam_file_marked_uploaded(rel_path, sha256):
+                continue
 
-        _upload_video_chunked(abs_path, rel_path, sha256)
-        app_sql.mark_dashcam_file_uploaded(rel_path, sha256)
-        uploaded_count += 1
+            _upload_video_chunked(abs_path, rel_path, sha256)
+            app_sql.mark_dashcam_file_uploaded(rel_path, sha256)
+            uploaded_count += 1
+        finally:
+            app_dash.end_local_video_use(rel_path)
 
         # Re-check between files so flipping the toggle off mid-run stops
         # further uploads promptly without waiting for the next cycle.
@@ -281,8 +288,15 @@ def _upload_loop():
         try:
             sent = _upload_tracking_batch()
             uploaded = _upload_dashcam_files()
+            cleanup = app_video.cleanup_uploaded_videos()
             if sent or uploaded:
                 print(f"Upload: sent {sent} tracking rows, {uploaded} dashcam files")
+            if cleanup["deleted"]:
+                print(
+                    f"Retention: deleted {len(cleanup['deleted'])} uploaded local videos"
+                )
+            if cleanup["errors"]:
+                print("Retention cleanup errors:", "; ".join(cleanup["errors"]))
             _last_error = None
             _last_success_at = _dt.datetime.now().isoformat()
             backoff = BACKOFF_INITIAL

modules/app_video.py

@@ -0,0 +1,210 @@
+"""Local imported-video browsing, playback, thumbnails, and retention."""
+import os
+import subprocess
+import time
+
+from gevent import sleep
+from gevent.lock import Semaphore
+from modules import app_dash, app_sql
+
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+THUMBNAIL_DIR = os.path.join(BASE_DIR, "dash", "thumbnails")
+RETENTION_SETTING = "local_video_retention_days"
+_thumbnail_lock = Semaphore()
+
+
+def _local_path(relative_path):
+    return app_dash.safe_video_path(app_dash.DEFAULT_DESTINATION, relative_path)
+
+
+def _thumbnail_path(relative_path):
+    source_root = os.path.realpath(app_dash.DEFAULT_DESTINATION)
+    source_path = app_dash.safe_video_path(source_root, relative_path)
+    thumbnail_root = os.path.realpath(THUMBNAIL_DIR)
+    thumbnail_path = os.path.realpath(
+        os.path.join(thumbnail_root, relative_path + ".jpg")
+    )
+    if os.path.commonpath([thumbnail_root, thumbnail_path]) != thumbnail_root:
+        raise ValueError("Thumbnail path escapes the cache directory.")
+    return source_path, thumbnail_path
+
+
+def list_local_videos():
+    files = []
+    errors = []
+    total_size = 0
+    uploaded_stats = app_sql.uploaded_dashcam_file_stats()
+    for abs_path, relative_path in app_dash.find_video_files(
+        app_dash.DEFAULT_DESTINATION
+    ):
+        try:
+            stat = os.stat(abs_path)
+        except OSError as e:
+            errors.append(f"{relative_path}: {e}")
+            continue
+        files.append(
+            {
+                "path": relative_path,
+                "size": stat.st_size,
+                "modified_at": stat.st_mtime,
+                "uploaded": (
+                    relative_path,
+                    stat.st_size,
+                    stat.st_mtime,
+                ) in uploaded_stats,
+            }
+        )
+        total_size += stat.st_size
+    files.sort(key=lambda item: item["path"].lower())
+    return {
+        "status": "ok",
+        "files": files,
+        "count": len(files),
+        "total_size": total_size,
+        "errors": errors,
+        "retention_days": retention_days(),
+    }
+
+
+def get_local_video_path(relative_path):
+    return _local_path(relative_path)
+
+
+def get_thumbnail_path(relative_path):
+    source_path, thumbnail_path = _thumbnail_path(relative_path)
+    source_mtime = os.path.getmtime(source_path)
+
+    with _thumbnail_lock:
+        if (
+            os.path.isfile(thumbnail_path)
+            and os.path.getmtime(thumbnail_path) >= source_mtime
+        ):
+            return thumbnail_path
+        if not app_dash.begin_local_video_use(relative_path):
+            raise RuntimeError(
+                "This video is currently being deleted. Try the thumbnail again shortly."
+            )
+
+        try:
+            os.makedirs(os.path.dirname(thumbnail_path), exist_ok=True)
+            temporary_path = thumbnail_path + ".tmp.jpg"
+            error = None
+            for seek_seconds in ("1", "0"):
+                command = [
+                    "ffmpeg",
+                    "-nostdin",
+                    "-loglevel",
+                    "error",
+                    "-ss",
+                    seek_seconds,
+                    "-i",
+                    source_path,
+                    "-frames:v",
+                    "1",
+                    "-vf",
+                    "scale=320:-2",
+                    "-q:v",
+                    "5",
+                    "-y",
+                    temporary_path,
+                ]
+                try:
+                    process = subprocess.Popen(
+                        command,
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.PIPE,
+                        text=True,
+                    )
+                except FileNotFoundError as e:
+                    raise RuntimeError("ffmpeg is not installed on this device.") from e
+                while process.poll() is None:
+                    sleep(0.05)
+                _stdout, error = process.communicate()
+                if (
+                    process.returncode == 0
+                    and os.path.isfile(temporary_path)
+                    and os.path.getsize(temporary_path) > 0
+                ):
+                    os.replace(temporary_path, thumbnail_path)
+                    return thumbnail_path
+                if os.path.exists(temporary_path):
+                    os.remove(temporary_path)
+
+            raise RuntimeError(
+                (error or "").strip()
+                or f"ffmpeg could not extract a frame from {relative_path}"
+            )
+        finally:
+            app_dash.end_local_video_use(relative_path)
+
+
+def delete_local_video(relative_path):
+    try:
+        video_path, thumbnail_path = _thumbnail_path(relative_path)
+    except ValueError as e:
+        return {"status": "invalid_path", "message": str(e)}
+    except FileNotFoundError:
+        return {
+            "status": "not_found",
+            "message": f"Local video not found: {relative_path}",
+        }
+    if not app_dash.begin_local_video_delete(relative_path):
+        return {
+            "status": "busy",
+            "message": "This video is currently uploading or being deleted. Try again shortly.",
+        }
+    try:
+        try:
+            os.remove(video_path)
+            if os.path.isfile(thumbnail_path):
+                os.remove(thumbnail_path)
+        except OSError as e:
+            return {"status": "delete_failed", "message": str(e)}
+    finally:
+        app_dash.end_local_video_delete(relative_path)
+    return {"status": "ok", "deleted": relative_path}
+
+
+def retention_days():
+    value = app_sql.get_setting(RETENTION_SETTING, "0")
+    try:
+        days = int(value)
+    except (TypeError, ValueError):
+        print(f"Invalid {RETENTION_SETTING} setting {value!r}; retention disabled.")
+        return 0
+    return max(days, 0)
+
+
+def cleanup_uploaded_videos(now=None):
+    """Delete uploaded local copies older than the configured number of days."""
+    days = retention_days()
+    if days <= 0:
+        return {"enabled": False, "deleted": [], "errors": []}
+
+    cutoff = (time.time() if now is None else now) - days * 86400
+    deleted = []
+    errors = []
+    uploaded_stats = app_sql.uploaded_dashcam_file_stats()
+    for abs_path, relative_path in app_dash.find_video_files(
+        app_dash.DEFAULT_DESTINATION
+    ):
+        try:
+            stat = os.stat(abs_path)
+            if stat.st_mtime >= cutoff:
+                continue
+            if (relative_path, stat.st_size, stat.st_mtime) not in uploaded_stats:
+                continue
+            if not app_dash.begin_local_video_delete(relative_path):
+                continue
+            try:
+                _source, thumbnail_path = _thumbnail_path(relative_path)
+                os.remove(abs_path)
+                if os.path.isfile(thumbnail_path):
+                    os.remove(thumbnail_path)
+                deleted.append(relative_path)
+            finally:
+                app_dash.end_local_video_delete(relative_path)
+        except (OSError, ValueError) as e:
+            errors.append(f"{relative_path}: {e}")
+    return {"enabled": True, "deleted": deleted, "errors": errors}