Mathieu MoranandCopilot
42e679782e
- app_sql.py: add UPLOADED column + migration on tracking table, add settings and dashcam_uploads tables, add helpers for unsent-row queries, marking rows uploaded, and dashcam upload dedupe tracking. - app_upload.py: new background upload client. Periodically batches unsent tracking rows to POST /api/v1/tracking/batch and uploads new dashcam files (sha256-verified) to POST /api/v1/dashcam/upload, with exponential backoff on failure. Exposes get_status() and a dashcam-upload enable/disable toggle backed by the settings table. - app.py: start the upload client thread at startup, add GET /api/upload/status and POST /api/upload/dashcam_toggle routes. - dash.html/dash.js: add a "Remote server upload" card with upload status and a toggle switch to pause dashcam video uploads while trip/GPS log uploads keep running (per user request - useful on a slow/metered connection). - db/upload_creds.env: new gitignore-exempted template (mirrors db/creds.env) for YARIS_SERVER_URL/YARIS_DEVICE_NAME/YARIS_API_KEY. - requirements.txt: add requests. Tested against the real deployed unraid server: tracking batch upload + dedupe, dashcam upload + dedupe, and toggle-off correctly pausing dashcam uploads while tracking uploads continue. Also verified the full Flask app imports/runs cleanly and the new API routes + dash.html toggle render correctly in a headless browser. Co-authored-by: Copilot <[email protected]>
255 lines
8.4 KiB
Python
255 lines
8.4 KiB
Python
import sqlite3
|
|
import os
|
|
import time
|
|
from datetime import datetime
|
|
from sqlite3 import Error
|
|
from threading import Thread, Lock
|
|
|
|
from modules import app_car, app_account, app_gps
|
|
|
|
# Resolve db/sql/ relative to the repo root regardless of the process's cwd
|
|
# (previously a relative "../db/sql/" path, which broke depending on how/from
|
|
# where app.py was launched).
|
|
_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
_SQL_DIR = os.path.join(_BASE_DIR, "db", "sql")
|
|
|
|
account = app_account.Account()
|
|
car = app_car.Car()
|
|
sCarModel = ''.join(letter for letter in car.model if letter.isalnum())
|
|
sql_file = account.name + "_" + sCarModel + ".db"
|
|
sql_path = os.path.join(_SQL_DIR, sql_file)
|
|
|
|
LOG_INTERVAL = 5 # seconds between trip-tracking snapshots
|
|
|
|
_conn = None
|
|
_conn_lock = Lock()
|
|
_log_thread = None
|
|
|
|
SQL_CREATE_TRACKING_TABLE = """CREATE TABLE IF NOT EXISTS tracking (
|
|
id integer PRIMARY KEY,
|
|
VN text NOT NULL,
|
|
NAME text NOT NULL,
|
|
DATETIME datetime,
|
|
GPSX float,
|
|
GPSY float,
|
|
SPEED float,
|
|
ODOMETER float,
|
|
UPLOADED integer DEFAULT 0
|
|
);"""
|
|
|
|
SQL_CREATE_DTC_TABLE = """CREATE TABLE IF NOT EXISTS dtc (
|
|
id integer PRIMARY KEY,
|
|
NAME text NOT NULL,
|
|
STATUS text,
|
|
DTC_ID text
|
|
);"""
|
|
|
|
SQL_CREATE_SETTINGS_TABLE = """CREATE TABLE IF NOT EXISTS settings (
|
|
key text PRIMARY KEY,
|
|
value text
|
|
);"""
|
|
|
|
SQL_CREATE_DASHCAM_UPLOADS_TABLE = """CREATE TABLE IF NOT EXISTS dashcam_uploads (
|
|
id integer PRIMARY KEY,
|
|
RELATIVE_PATH text NOT NULL,
|
|
SHA256 text NOT NULL,
|
|
UPLOADED_AT datetime,
|
|
UNIQUE(RELATIVE_PATH, SHA256)
|
|
);"""
|
|
|
|
|
|
def create_conn(db_file):
|
|
"""Open a SQLite connection to db_file, creating its parent folder if needed.
|
|
|
|
Returns the open connection (or None on failure) - previously this closed
|
|
the connection immediately in a `finally` block and returned nothing,
|
|
so every caller always got a dead `None` connection.
|
|
"""
|
|
os.makedirs(os.path.dirname(db_file), exist_ok=True)
|
|
conn = None
|
|
try:
|
|
conn = sqlite3.connect(db_file, check_same_thread=False)
|
|
print("SQLite library version:", sqlite3.sqlite_version)
|
|
except Error as e:
|
|
print(e)
|
|
return conn
|
|
|
|
|
|
def create_table(conn, create_table_sql):
|
|
""" create a table from the create_table_sql statement
|
|
:param conn: Connection object
|
|
:param create_table_sql: a CREATE TABLE statement
|
|
:return:
|
|
"""
|
|
try:
|
|
c = conn.cursor()
|
|
c.execute(create_table_sql)
|
|
except Error as e:
|
|
print(e)
|
|
|
|
|
|
def get_conn():
|
|
"""Return the shared SQLite connection, opening + initializing tables on first use."""
|
|
global _conn
|
|
if _conn is None:
|
|
_conn = create_conn(sql_path)
|
|
if _conn is not None:
|
|
create_table(_conn, SQL_CREATE_TRACKING_TABLE)
|
|
create_table(_conn, SQL_CREATE_DTC_TABLE)
|
|
create_table(_conn, SQL_CREATE_SETTINGS_TABLE)
|
|
create_table(_conn, SQL_CREATE_DASHCAM_UPLOADS_TABLE)
|
|
_migrate_tracking_table(_conn)
|
|
else:
|
|
print("Error! cannot create the database connection.")
|
|
return _conn
|
|
|
|
|
|
def _migrate_tracking_table(conn):
|
|
"""Add the UPLOADED column to a pre-existing tracking table that predates
|
|
the remote-upload feature (SQLite has no "ADD COLUMN IF NOT EXISTS")."""
|
|
existing_columns = {row[1] for row in conn.execute("PRAGMA table_info(tracking)").fetchall()}
|
|
if "UPLOADED" not in existing_columns:
|
|
conn.execute("ALTER TABLE tracking ADD COLUMN UPLOADED integer DEFAULT 0")
|
|
conn.commit()
|
|
|
|
|
|
def insert_tracking(gps_x, gps_y, speed, odometer=None):
|
|
"""Insert a single trip-tracking snapshot row (GPS position + speed)."""
|
|
conn = get_conn()
|
|
if conn is None:
|
|
return
|
|
|
|
sql = """INSERT INTO tracking (VN, NAME, DATETIME, GPSX, GPSY, SPEED, ODOMETER)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)"""
|
|
with _conn_lock:
|
|
try:
|
|
conn.execute(sql, (car.model, account.name, datetime.now(), gps_x, gps_y, speed, odometer))
|
|
conn.commit()
|
|
except Error as e:
|
|
print("Failed to insert tracking row:", e)
|
|
|
|
|
|
def get_setting(key, default=None):
|
|
"""Read a persisted UI/app setting (e.g. the dashcam-upload toggle)."""
|
|
conn = get_conn()
|
|
if conn is None:
|
|
return default
|
|
with _conn_lock:
|
|
cur = conn.execute("SELECT value FROM settings WHERE key = ?", (key,))
|
|
row = cur.fetchone()
|
|
return row[0] if row is not None else default
|
|
|
|
|
|
def set_setting(key, value):
|
|
conn = get_conn()
|
|
if conn is None:
|
|
return
|
|
with _conn_lock:
|
|
conn.execute(
|
|
"INSERT INTO settings (key, value) VALUES (?, ?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
|
(key, value),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def get_unsent_tracking_rows(limit=100):
|
|
"""Return up to `limit` tracking rows not yet uploaded to the remote server,
|
|
oldest first, as a list of dicts ready to be sent as JSON."""
|
|
conn = get_conn()
|
|
if conn is None:
|
|
return []
|
|
with _conn_lock:
|
|
cur = conn.execute(
|
|
"""SELECT id, VN, NAME, DATETIME, GPSX, GPSY, SPEED, ODOMETER
|
|
FROM tracking WHERE UPLOADED = 0 ORDER BY id ASC LIMIT ?""",
|
|
(limit,),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [
|
|
{
|
|
"id": r[0], "vn": r[1], "name": r[2], "datetime": r[3],
|
|
"gpsx": r[4], "gpsy": r[5], "speed": r[6], "odometer": r[7],
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def count_unsent_tracking_rows():
|
|
conn = get_conn()
|
|
if conn is None:
|
|
return 0
|
|
with _conn_lock:
|
|
cur = conn.execute("SELECT COUNT(*) FROM tracking WHERE UPLOADED = 0")
|
|
return cur.fetchone()[0]
|
|
|
|
|
|
def mark_tracking_rows_uploaded(ids):
|
|
"""Flip UPLOADED=1 for the given row ids after a successful server ack."""
|
|
if not ids:
|
|
return
|
|
conn = get_conn()
|
|
if conn is None:
|
|
return
|
|
with _conn_lock:
|
|
conn.executemany("UPDATE tracking SET UPLOADED = 1 WHERE id = ?", [(i,) for i in ids])
|
|
conn.commit()
|
|
|
|
|
|
def dashcam_file_marked_uploaded(relative_path, sha256):
|
|
"""Whether this exact (path, content) was already confirmed uploaded."""
|
|
conn = get_conn()
|
|
if conn is None:
|
|
return False
|
|
with _conn_lock:
|
|
cur = conn.execute(
|
|
"SELECT 1 FROM dashcam_uploads WHERE RELATIVE_PATH = ? AND SHA256 = ?",
|
|
(relative_path, sha256),
|
|
)
|
|
return cur.fetchone() is not None
|
|
|
|
|
|
def mark_dashcam_file_uploaded(relative_path, sha256):
|
|
conn = get_conn()
|
|
if conn is None:
|
|
return
|
|
with _conn_lock:
|
|
conn.execute(
|
|
"INSERT OR IGNORE INTO dashcam_uploads (RELATIVE_PATH, SHA256, UPLOADED_AT) VALUES (?, ?, ?)",
|
|
(relative_path, sha256, datetime.now()),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def _log_loop():
|
|
# Deferred import to avoid a circular import at module load time
|
|
# (app_obd.py imports app_sql, and app_sql needs app_obd.speed here).
|
|
from modules import app_obd
|
|
|
|
while True:
|
|
gps_x = app_gps.LocationX
|
|
gps_y = app_gps.LocationY
|
|
speed = app_obd.speed
|
|
|
|
if gps_x is not None and gps_y is not None:
|
|
insert_tracking(gps_x, gps_y, speed)
|
|
|
|
time.sleep(LOG_INTERVAL)
|
|
|
|
|
|
def start_logging():
|
|
"""Start the background thread that periodically snapshots GPS+speed into SQLite."""
|
|
global _log_thread
|
|
if _log_thread is None or not _log_thread.is_alive():
|
|
get_conn() # make sure the db file/tables exist before the first insert
|
|
_log_thread = Thread(target=_log_loop)
|
|
_log_thread.daemon = True
|
|
_log_thread.start()
|
|
print(f"SQL tracking-logger thread started, alive: {_log_thread.is_alive()}")
|
|
else:
|
|
print(f"SQL tracking-logger thread already running, alive: {_log_thread.is_alive()}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
get_conn()
|