fix(phase26): persist DTC/MIL state + server speed_source (BUG-03, BUG-05)
BUG-03: DTC/MIL codes were never persisted; lost on OBD disconnect or reboot. - Add new dtc table schema (code, description, MIL, first/last seen, IS_SET) and a migration that rebuilds the original unused NAME/STATUS/DTC_ID table. - Persist MIL + codes to SQLite on each successful OBD poll (_persist_dtc_state). - /api/dtc falls back to the last-persisted snapshot when OBD is offline/stale. - Fix get_overview_stats to count the real dtc table (was querying non-existent dtc_codes table). - Add cleared/since tracking via IS_SET + FIRST_SEEN_AT/LAST_SEEN_AT. BUG-05: server trip replay hardcoded speed_source:'gps'; mirror Pi's 'obd if speed>0 else gps' logic in server get_trip_route. Adds tests/test_dtc_persistence.py (7 tests).
This commit is contained in:
Mathieu Moran
@@ -335,9 +335,17 @@ def data():
|
||||
|
||||
@app.route('/api/dtc')
|
||||
def api_dtc():
|
||||
fresh_mil = app_obd.get_fresh_mil()
|
||||
fresh_codes = app_obd.get_fresh_dtc_codes()
|
||||
if fresh_mil is None and not fresh_codes:
|
||||
# OBD offline/stale (disconnected or not yet polled): fall back to the
|
||||
# last-persisted DTC snapshot so codes survive a disconnect/reboot.
|
||||
persisted = app_sql.get_persisted_dtc_state()
|
||||
fresh_mil = persisted["mil"]
|
||||
fresh_codes = persisted["codes"]
|
||||
return jsonify({
|
||||
"mil": app_obd.get_fresh_mil(),
|
||||
"codes": app_obd.get_fresh_dtc_codes(),
|
||||
"mil": fresh_mil,
|
||||
"codes": fresh_codes,
|
||||
})
|
||||
|
||||
@app.route('/api/stats')
|
||||
|
||||
@@ -368,6 +368,19 @@ def _poll_diagnostics():
|
||||
app_events.socketio.emit("obd_throttle", throttle_position)
|
||||
app_events.socketio.emit("obd_odometer", odometer)
|
||||
|
||||
# Persist the latest MIL + DTC snapshot so last-known state survives an
|
||||
# OBD disconnect or a power cycle (deferred import avoids a circular import).
|
||||
_persist_dtc_state()
|
||||
|
||||
|
||||
def _persist_dtc_state():
|
||||
"""Best-effort persist of the current MIL + DTC codes to SQLite."""
|
||||
try:
|
||||
from modules import app_sql
|
||||
app_sql.save_dtc_state(mil_on, dtc_codes)
|
||||
except Exception as error:
|
||||
print("Failed to persist DTC state to SQLite:", error)
|
||||
|
||||
|
||||
def obd_sensors():
|
||||
with obd_sensor_lock:
|
||||
|
||||
+116
-12
@@ -46,9 +46,12 @@ SQL_CREATE_TRACKING_TABLE = """CREATE TABLE IF NOT EXISTS tracking (
|
||||
|
||||
SQL_CREATE_DTC_TABLE = """CREATE TABLE IF NOT EXISTS dtc (
|
||||
id integer PRIMARY KEY,
|
||||
NAME text NOT NULL,
|
||||
STATUS text,
|
||||
DTC_ID text
|
||||
code text NOT NULL UNIQUE,
|
||||
description text,
|
||||
MIL integer,
|
||||
FIRST_SEEN_AT datetime,
|
||||
LAST_SEEN_AT datetime,
|
||||
IS_SET integer NOT NULL DEFAULT 1
|
||||
);"""
|
||||
|
||||
SQL_CREATE_SETTINGS_TABLE = """CREATE TABLE IF NOT EXISTS settings (
|
||||
@@ -161,6 +164,7 @@ def get_conn():
|
||||
_migrate_tracking_table(_conn)
|
||||
_migrate_trips_table(_conn)
|
||||
_migrate_trip_build_state_table(_conn)
|
||||
_migrate_dtc_table(_conn)
|
||||
else:
|
||||
print("Error! cannot create the database connection.")
|
||||
return _conn
|
||||
@@ -203,6 +207,112 @@ def _migrate_trip_build_state_table(conn):
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _migrate_dtc_table(conn):
|
||||
"""Rebuild the dtc table if it predates the persisted-DTC schema.
|
||||
|
||||
The original dtc table (NAME/STATUS/DTC_ID) was never written to by any
|
||||
code path, so dropping and recreating it with the new schema is safe.
|
||||
"""
|
||||
existing_columns = {
|
||||
row[1] for row in conn.execute("PRAGMA table_info(dtc)").fetchall()
|
||||
}
|
||||
if existing_columns and "code" not in existing_columns:
|
||||
conn.execute("DROP TABLE IF EXISTS dtc")
|
||||
create_table(conn, SQL_CREATE_DTC_TABLE)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def save_dtc_state(mil_on, dtc_codes, recorded_at=None):
|
||||
"""Persist a DTC snapshot (MIL + currently-set codes) to SQLite.
|
||||
|
||||
Last-known state can then survive an OBD disconnect or reboot. Codes that
|
||||
were previously set but are no longer detected are marked cleared
|
||||
(IS_SET = 0) and LAST_SEEN_AT is bumped, while FIRST_SEEN_AT preserves when
|
||||
each code was first observed.
|
||||
"""
|
||||
conn = get_conn()
|
||||
if conn is None:
|
||||
return
|
||||
now_str = (recorded_at or datetime.now()).isoformat()
|
||||
current_codes = {c["code"] for c in dtc_codes if c.get("code")}
|
||||
with _conn_lock:
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO settings (key, value) VALUES ('dtc_mil', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
("1" if mil_on else "0",),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO settings (key, value) VALUES ('dtc_last_polled_at', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(now_str,),
|
||||
)
|
||||
if current_codes:
|
||||
placeholders = ",".join("?" for _ in current_codes)
|
||||
conn.execute(
|
||||
f"UPDATE dtc SET IS_SET = 0, LAST_SEEN_AT = ? "
|
||||
f"WHERE code NOT IN ({placeholders})",
|
||||
(now_str,) + tuple(current_codes),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"UPDATE dtc SET IS_SET = 0, LAST_SEEN_AT = ? WHERE IS_SET = 1",
|
||||
(now_str,),
|
||||
)
|
||||
for code in sorted(current_codes):
|
||||
description = next(
|
||||
(c["description"] for c in dtc_codes if c.get("code") == code),
|
||||
None,
|
||||
)
|
||||
conn.execute(
|
||||
"""INSERT INTO dtc (code, description, MIL, FIRST_SEEN_AT,
|
||||
LAST_SEEN_AT, IS_SET)
|
||||
VALUES (?, ?, ?, ?, ?, 1)
|
||||
ON CONFLICT(code) DO UPDATE SET
|
||||
description = excluded.description,
|
||||
MIL = excluded.MIL,
|
||||
LAST_SEEN_AT = excluded.LAST_SEEN_AT,
|
||||
IS_SET = 1""",
|
||||
(code, description, 1 if mil_on else 0, now_str, now_str),
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.Error as e:
|
||||
print("Failed to persist DTC state:", e)
|
||||
|
||||
|
||||
def get_persisted_dtc_state():
|
||||
"""Return the last-persisted MIL + DTC codes for display when OBD is offline."""
|
||||
conn = get_conn()
|
||||
if conn is None:
|
||||
return {"mil": None, "codes": []}
|
||||
with _conn_lock:
|
||||
mil_row = conn.execute(
|
||||
"SELECT value FROM settings WHERE key = 'dtc_mil'"
|
||||
).fetchone()
|
||||
codes = conn.execute(
|
||||
"SELECT code, description FROM dtc WHERE IS_SET = 1 ORDER BY code ASC"
|
||||
).fetchall()
|
||||
mil = None
|
||||
if mil_row and mil_row[0] not in (None, ""):
|
||||
mil = mil_row[0] == "1"
|
||||
return {
|
||||
"mil": mil,
|
||||
"codes": [{"code": r[0], "description": r[1]} for r in codes],
|
||||
}
|
||||
|
||||
|
||||
def count_active_dtcs():
|
||||
"""Number of currently-set DTC codes persisted in SQLite (cleared excluded)."""
|
||||
conn = get_conn()
|
||||
if conn is None:
|
||||
return 0
|
||||
with _conn_lock:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) FROM dtc WHERE IS_SET = 1"
|
||||
).fetchone()
|
||||
return row[0] if row else 0
|
||||
|
||||
|
||||
def insert_tracking(gps_x, gps_y, speed, odometer=None):
|
||||
"""Insert a single trip-tracking snapshot row (GPS position + speed)."""
|
||||
conn = get_conn()
|
||||
@@ -334,9 +444,9 @@ def get_overview_stats():
|
||||
odo_row = conn.execute("""
|
||||
SELECT MAX(ODOMETER) FROM tracking WHERE ODOMETER IS NOT NULL AND ODOMETER > 0
|
||||
""").fetchone()
|
||||
dtc_row = conn.execute("""
|
||||
SELECT COUNT(*) FROM dtc_codes WHERE cleared_at IS NULL
|
||||
""").fetchone() if _table_exists(conn, "dtc_codes") else (0,)
|
||||
dtc_row = conn.execute(
|
||||
"SELECT COUNT(*) FROM dtc WHERE IS_SET = 1"
|
||||
).fetchone()
|
||||
from modules import app_video as _av
|
||||
try:
|
||||
video_count = sum(1 for _ in _av.list_local_videos().get("files", []))
|
||||
@@ -353,12 +463,6 @@ def get_overview_stats():
|
||||
}
|
||||
|
||||
|
||||
def _table_exists(conn, name):
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)
|
||||
).fetchone() is not None
|
||||
|
||||
|
||||
def list_trips():
|
||||
refresh_trips()
|
||||
conn = get_conn()
|
||||
|
||||
@@ -443,7 +443,9 @@ def get_trip_route(device_id, trip_id):
|
||||
"latitude": p[1],
|
||||
"longitude": p[2],
|
||||
"speed": p[3] or 0,
|
||||
"speed_source": "gps",
|
||||
# Mirror the Pi's route_points: OBD speed when we have it,
|
||||
# otherwise the point is sourced from GPS.
|
||||
"speed_source": "obd" if (p[3] or 0) > 0 else "gps",
|
||||
}
|
||||
for p in points
|
||||
],
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
|
||||
from modules import app_sql
|
||||
|
||||
|
||||
class DtcPersistenceTest(unittest.TestCase):
|
||||
"""Tests for persisted DTC/MIL state (BUG-03)."""
|
||||
|
||||
def setUp(self):
|
||||
handle, self.path = tempfile.mkstemp(suffix=".db")
|
||||
os.close(handle)
|
||||
self.original_path = app_sql.sql_path
|
||||
app_sql._conn = None
|
||||
app_sql.sql_path = self.path
|
||||
self.conn = app_sql.get_conn()
|
||||
|
||||
def tearDown(self):
|
||||
if app_sql._conn is not None:
|
||||
app_sql._conn.close()
|
||||
app_sql._conn = None
|
||||
app_sql.sql_path = self.original_path
|
||||
os.unlink(self.path)
|
||||
|
||||
def test_save_and_read_back_dtc_state(self):
|
||||
app_sql.save_dtc_state(True, [
|
||||
{"code": "P0300", "description": "Random misfire"},
|
||||
{"code": "P0420", "description": "Catalyst efficiency"},
|
||||
])
|
||||
state = app_sql.get_persisted_dtc_state()
|
||||
self.assertIs(state["mil"], True)
|
||||
self.assertEqual(
|
||||
[c["code"] for c in state["codes"]],
|
||||
["P0300", "P0420"],
|
||||
)
|
||||
self.assertEqual(state["codes"][0]["description"], "Random misfire")
|
||||
self.assertEqual(app_sql.count_active_dtcs(), 2)
|
||||
|
||||
def test_cleared_code_removed_from_active(self):
|
||||
app_sql.save_dtc_state(True, [
|
||||
{"code": "P0300", "description": "Random misfire"},
|
||||
{"code": "P0420", "description": "Catalyst efficiency"},
|
||||
])
|
||||
# A later poll reports only P0420 -> P0300 is cleared.
|
||||
app_sql.save_dtc_state(True, [
|
||||
{"code": "P0420", "description": "Catalyst efficiency"},
|
||||
])
|
||||
state = app_sql.get_persisted_dtc_state()
|
||||
self.assertEqual([c["code"] for c in state["codes"]], ["P0420"])
|
||||
self.assertEqual(app_sql.count_active_dtcs(), 1)
|
||||
# The cleared code keeps its history row (IS_SET=0).
|
||||
rows = self.conn.execute(
|
||||
"SELECT code, IS_SET FROM dtc ORDER BY code"
|
||||
).fetchall()
|
||||
self.assertEqual(rows, [("P0300", 0), ("P0420", 1)])
|
||||
|
||||
def test_empty_poll_clears_all(self):
|
||||
app_sql.save_dtc_state(True, [{"code": "P0300", "description": "Misfire"}])
|
||||
app_sql.save_dtc_state(False, [])
|
||||
self.assertEqual(app_sql.count_active_dtcs(), 0)
|
||||
self.assertIs(app_sql.get_persisted_dtc_state()["mil"], False)
|
||||
|
||||
def test_first_seen_preserved_while_last_seen_bumps(self):
|
||||
first = datetime(2026, 2, 1, 8, 0, 0)
|
||||
later = datetime(2026, 2, 1, 8, 5, 0)
|
||||
app_sql.save_dtc_state(True, [{"code": "P0300", "description": "M"}], first)
|
||||
app_sql.save_dtc_state(True, [{"code": "P0300", "description": "M"}], later)
|
||||
row = self.conn.execute(
|
||||
"SELECT FIRST_SEEN_AT, LAST_SEEN_AT FROM dtc WHERE code = 'P0300'"
|
||||
).fetchone()
|
||||
self.assertEqual(row[0], first.isoformat())
|
||||
self.assertEqual(row[1], later.isoformat())
|
||||
|
||||
def test_overview_stats_uses_dtc_table_not_broken_dtc_codes(self):
|
||||
app_sql.save_dtc_state(True, [
|
||||
{"code": "P0300", "description": "Random misfire"},
|
||||
])
|
||||
stats = app_sql.get_overview_stats()
|
||||
self.assertEqual(stats["active_dtc"], 1)
|
||||
|
||||
def test_old_schema_dtc_table_migrated(self):
|
||||
# Simulate the original (never-populated) dtc schema.
|
||||
app_sql._conn.execute("DROP TABLE dtc")
|
||||
app_sql._conn.execute(
|
||||
"""CREATE TABLE dtc (
|
||||
id integer PRIMARY KEY,
|
||||
NAME text NOT NULL,
|
||||
STATUS text,
|
||||
DTC_ID text
|
||||
)"""
|
||||
)
|
||||
app_sql._conn.commit()
|
||||
app_sql._migrate_dtc_table(app_sql._conn)
|
||||
columns = {
|
||||
row[1] for row in app_sql._conn.execute("PRAGMA table_info(dtc)").fetchall()
|
||||
}
|
||||
self.assertIn("code", columns)
|
||||
self.assertNotIn("DTC_ID", columns)
|
||||
|
||||
def test_no_persisted_state_returns_empty(self):
|
||||
state = app_sql.get_persisted_dtc_state()
|
||||
self.assertIsNone(state["mil"])
|
||||
self.assertEqual(state["codes"], [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user