From e2f23143182ee3d1737b594a89d32773c58e7344 Mon Sep 17 00:00:00 2001 From: Mathieu Moran Date: Tue, 11 Aug 2026 00:38:05 -0400 Subject: [PATCH] fix(phase26): persist DTC/MIL state + server speed_source (BUG-03, BUG-05) BUG-03: DTC/MIL codes were never persisted; lost on OBD disconnect or reboot. - Add new dtc table schema (code, description, MIL, first/last seen, IS_SET) and a migration that rebuilds the original unused NAME/STATUS/DTC_ID table. - Persist MIL + codes to SQLite on each successful OBD poll (_persist_dtc_state). - /api/dtc falls back to the last-persisted snapshot when OBD is offline/stale. - Fix get_overview_stats to count the real dtc table (was querying non-existent dtc_codes table). - Add cleared/since tracking via IS_SET + FIRST_SEEN_AT/LAST_SEEN_AT. BUG-05: server trip replay hardcoded speed_source:'gps'; mirror Pi's 'obd if speed>0 else gps' logic in server get_trip_route. Adds tests/test_dtc_persistence.py (7 tests). --- app.py | 12 +++- modules/app_obd.py | 13 ++++ modules/app_sql.py | 128 ++++++++++++++++++++++++++++++---- server/modules/db.py | 4 +- tests/test_dtc_persistence.py | 109 +++++++++++++++++++++++++++++ 5 files changed, 251 insertions(+), 15 deletions(-) create mode 100644 tests/test_dtc_persistence.py diff --git a/app.py b/app.py index 55a8e9c..792d9e5 100644 --- a/app.py +++ b/app.py @@ -335,9 +335,17 @@ def data(): @app.route('/api/dtc') def api_dtc(): + fresh_mil = app_obd.get_fresh_mil() + fresh_codes = app_obd.get_fresh_dtc_codes() + if fresh_mil is None and not fresh_codes: + # OBD offline/stale (disconnected or not yet polled): fall back to the + # last-persisted DTC snapshot so codes survive a disconnect/reboot. + persisted = app_sql.get_persisted_dtc_state() + fresh_mil = persisted["mil"] + fresh_codes = persisted["codes"] return jsonify({ - "mil": app_obd.get_fresh_mil(), - "codes": app_obd.get_fresh_dtc_codes(), + "mil": fresh_mil, + "codes": fresh_codes, }) @app.route('/api/stats') diff --git a/modules/app_obd.py b/modules/app_obd.py index 8b0efd7..3a88bc8 100644 --- a/modules/app_obd.py +++ b/modules/app_obd.py @@ -368,6 +368,19 @@ def _poll_diagnostics(): app_events.socketio.emit("obd_throttle", throttle_position) app_events.socketio.emit("obd_odometer", odometer) + # Persist the latest MIL + DTC snapshot so last-known state survives an + # OBD disconnect or a power cycle (deferred import avoids a circular import). + _persist_dtc_state() + + +def _persist_dtc_state(): + """Best-effort persist of the current MIL + DTC codes to SQLite.""" + try: + from modules import app_sql + app_sql.save_dtc_state(mil_on, dtc_codes) + except Exception as error: + print("Failed to persist DTC state to SQLite:", error) + def obd_sensors(): with obd_sensor_lock: diff --git a/modules/app_sql.py b/modules/app_sql.py index c5e6996..c6815c6 100644 --- a/modules/app_sql.py +++ b/modules/app_sql.py @@ -46,9 +46,12 @@ SQL_CREATE_TRACKING_TABLE = """CREATE TABLE IF NOT EXISTS tracking ( SQL_CREATE_DTC_TABLE = """CREATE TABLE IF NOT EXISTS dtc ( id integer PRIMARY KEY, - NAME text NOT NULL, - STATUS text, - DTC_ID text + code text NOT NULL UNIQUE, + description text, + MIL integer, + FIRST_SEEN_AT datetime, + LAST_SEEN_AT datetime, + IS_SET integer NOT NULL DEFAULT 1 );""" SQL_CREATE_SETTINGS_TABLE = """CREATE TABLE IF NOT EXISTS settings ( @@ -161,6 +164,7 @@ def get_conn(): _migrate_tracking_table(_conn) _migrate_trips_table(_conn) _migrate_trip_build_state_table(_conn) + _migrate_dtc_table(_conn) else: print("Error! cannot create the database connection.") return _conn @@ -203,6 +207,112 @@ def _migrate_trip_build_state_table(conn): conn.commit() +def _migrate_dtc_table(conn): + """Rebuild the dtc table if it predates the persisted-DTC schema. + + The original dtc table (NAME/STATUS/DTC_ID) was never written to by any + code path, so dropping and recreating it with the new schema is safe. + """ + existing_columns = { + row[1] for row in conn.execute("PRAGMA table_info(dtc)").fetchall() + } + if existing_columns and "code" not in existing_columns: + conn.execute("DROP TABLE IF EXISTS dtc") + create_table(conn, SQL_CREATE_DTC_TABLE) + conn.commit() + + +def save_dtc_state(mil_on, dtc_codes, recorded_at=None): + """Persist a DTC snapshot (MIL + currently-set codes) to SQLite. + + Last-known state can then survive an OBD disconnect or reboot. Codes that + were previously set but are no longer detected are marked cleared + (IS_SET = 0) and LAST_SEEN_AT is bumped, while FIRST_SEEN_AT preserves when + each code was first observed. + """ + conn = get_conn() + if conn is None: + return + now_str = (recorded_at or datetime.now()).isoformat() + current_codes = {c["code"] for c in dtc_codes if c.get("code")} + with _conn_lock: + try: + conn.execute( + "INSERT INTO settings (key, value) VALUES ('dtc_mil', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ("1" if mil_on else "0",), + ) + conn.execute( + "INSERT INTO settings (key, value) VALUES ('dtc_last_polled_at', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (now_str,), + ) + if current_codes: + placeholders = ",".join("?" for _ in current_codes) + conn.execute( + f"UPDATE dtc SET IS_SET = 0, LAST_SEEN_AT = ? " + f"WHERE code NOT IN ({placeholders})", + (now_str,) + tuple(current_codes), + ) + else: + conn.execute( + "UPDATE dtc SET IS_SET = 0, LAST_SEEN_AT = ? WHERE IS_SET = 1", + (now_str,), + ) + for code in sorted(current_codes): + description = next( + (c["description"] for c in dtc_codes if c.get("code") == code), + None, + ) + conn.execute( + """INSERT INTO dtc (code, description, MIL, FIRST_SEEN_AT, + LAST_SEEN_AT, IS_SET) + VALUES (?, ?, ?, ?, ?, 1) + ON CONFLICT(code) DO UPDATE SET + description = excluded.description, + MIL = excluded.MIL, + LAST_SEEN_AT = excluded.LAST_SEEN_AT, + IS_SET = 1""", + (code, description, 1 if mil_on else 0, now_str, now_str), + ) + conn.commit() + except sqlite3.Error as e: + print("Failed to persist DTC state:", e) + + +def get_persisted_dtc_state(): + """Return the last-persisted MIL + DTC codes for display when OBD is offline.""" + conn = get_conn() + if conn is None: + return {"mil": None, "codes": []} + with _conn_lock: + mil_row = conn.execute( + "SELECT value FROM settings WHERE key = 'dtc_mil'" + ).fetchone() + codes = conn.execute( + "SELECT code, description FROM dtc WHERE IS_SET = 1 ORDER BY code ASC" + ).fetchall() + mil = None + if mil_row and mil_row[0] not in (None, ""): + mil = mil_row[0] == "1" + return { + "mil": mil, + "codes": [{"code": r[0], "description": r[1]} for r in codes], + } + + +def count_active_dtcs(): + """Number of currently-set DTC codes persisted in SQLite (cleared excluded).""" + conn = get_conn() + if conn is None: + return 0 + with _conn_lock: + row = conn.execute( + "SELECT COUNT(*) FROM dtc WHERE IS_SET = 1" + ).fetchone() + return row[0] if row else 0 + + def insert_tracking(gps_x, gps_y, speed, odometer=None): """Insert a single trip-tracking snapshot row (GPS position + speed).""" conn = get_conn() @@ -334,9 +444,9 @@ def get_overview_stats(): odo_row = conn.execute(""" SELECT MAX(ODOMETER) FROM tracking WHERE ODOMETER IS NOT NULL AND ODOMETER > 0 """).fetchone() - dtc_row = conn.execute(""" - SELECT COUNT(*) FROM dtc_codes WHERE cleared_at IS NULL - """).fetchone() if _table_exists(conn, "dtc_codes") else (0,) + dtc_row = conn.execute( + "SELECT COUNT(*) FROM dtc WHERE IS_SET = 1" + ).fetchone() from modules import app_video as _av try: video_count = sum(1 for _ in _av.list_local_videos().get("files", [])) @@ -353,12 +463,6 @@ def get_overview_stats(): } -def _table_exists(conn, name): - return conn.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,) - ).fetchone() is not None - - def list_trips(): refresh_trips() conn = get_conn() diff --git a/server/modules/db.py b/server/modules/db.py index c82e949..e2a45c8 100644 --- a/server/modules/db.py +++ b/server/modules/db.py @@ -443,7 +443,9 @@ def get_trip_route(device_id, trip_id): "latitude": p[1], "longitude": p[2], "speed": p[3] or 0, - "speed_source": "gps", + # Mirror the Pi's route_points: OBD speed when we have it, + # otherwise the point is sourced from GPS. + "speed_source": "obd" if (p[3] or 0) > 0 else "gps", } for p in points ], diff --git a/tests/test_dtc_persistence.py b/tests/test_dtc_persistence.py new file mode 100644 index 0000000..801ec2c --- /dev/null +++ b/tests/test_dtc_persistence.py @@ -0,0 +1,109 @@ +import os +import tempfile +import unittest +from datetime import datetime + +from modules import app_sql + + +class DtcPersistenceTest(unittest.TestCase): + """Tests for persisted DTC/MIL state (BUG-03).""" + + def setUp(self): + handle, self.path = tempfile.mkstemp(suffix=".db") + os.close(handle) + self.original_path = app_sql.sql_path + app_sql._conn = None + app_sql.sql_path = self.path + self.conn = app_sql.get_conn() + + def tearDown(self): + if app_sql._conn is not None: + app_sql._conn.close() + app_sql._conn = None + app_sql.sql_path = self.original_path + os.unlink(self.path) + + def test_save_and_read_back_dtc_state(self): + app_sql.save_dtc_state(True, [ + {"code": "P0300", "description": "Random misfire"}, + {"code": "P0420", "description": "Catalyst efficiency"}, + ]) + state = app_sql.get_persisted_dtc_state() + self.assertIs(state["mil"], True) + self.assertEqual( + [c["code"] for c in state["codes"]], + ["P0300", "P0420"], + ) + self.assertEqual(state["codes"][0]["description"], "Random misfire") + self.assertEqual(app_sql.count_active_dtcs(), 2) + + def test_cleared_code_removed_from_active(self): + app_sql.save_dtc_state(True, [ + {"code": "P0300", "description": "Random misfire"}, + {"code": "P0420", "description": "Catalyst efficiency"}, + ]) + # A later poll reports only P0420 -> P0300 is cleared. + app_sql.save_dtc_state(True, [ + {"code": "P0420", "description": "Catalyst efficiency"}, + ]) + state = app_sql.get_persisted_dtc_state() + self.assertEqual([c["code"] for c in state["codes"]], ["P0420"]) + self.assertEqual(app_sql.count_active_dtcs(), 1) + # The cleared code keeps its history row (IS_SET=0). + rows = self.conn.execute( + "SELECT code, IS_SET FROM dtc ORDER BY code" + ).fetchall() + self.assertEqual(rows, [("P0300", 0), ("P0420", 1)]) + + def test_empty_poll_clears_all(self): + app_sql.save_dtc_state(True, [{"code": "P0300", "description": "Misfire"}]) + app_sql.save_dtc_state(False, []) + self.assertEqual(app_sql.count_active_dtcs(), 0) + self.assertIs(app_sql.get_persisted_dtc_state()["mil"], False) + + def test_first_seen_preserved_while_last_seen_bumps(self): + first = datetime(2026, 2, 1, 8, 0, 0) + later = datetime(2026, 2, 1, 8, 5, 0) + app_sql.save_dtc_state(True, [{"code": "P0300", "description": "M"}], first) + app_sql.save_dtc_state(True, [{"code": "P0300", "description": "M"}], later) + row = self.conn.execute( + "SELECT FIRST_SEEN_AT, LAST_SEEN_AT FROM dtc WHERE code = 'P0300'" + ).fetchone() + self.assertEqual(row[0], first.isoformat()) + self.assertEqual(row[1], later.isoformat()) + + def test_overview_stats_uses_dtc_table_not_broken_dtc_codes(self): + app_sql.save_dtc_state(True, [ + {"code": "P0300", "description": "Random misfire"}, + ]) + stats = app_sql.get_overview_stats() + self.assertEqual(stats["active_dtc"], 1) + + def test_old_schema_dtc_table_migrated(self): + # Simulate the original (never-populated) dtc schema. + app_sql._conn.execute("DROP TABLE dtc") + app_sql._conn.execute( + """CREATE TABLE dtc ( + id integer PRIMARY KEY, + NAME text NOT NULL, + STATUS text, + DTC_ID text + )""" + ) + app_sql._conn.commit() + app_sql._migrate_dtc_table(app_sql._conn) + columns = { + row[1] for row in app_sql._conn.execute("PRAGMA table_info(dtc)").fetchall() + } + self.assertIn("code", columns) + self.assertNotIn("DTC_ID", columns) + + def test_no_persisted_state_returns_empty(self): + state = app_sql.get_persisted_dtc_state() + self.assertIsNone(state["mil"]) + self.assertEqual(state["codes"], []) + + +if __name__ == "__main__": + unittest.main()