From d3bc313e7fef6699c3098fde6b352f71447a2425 Mon Sep 17 00:00:00 2001 From: Mathieu Moran Date: Sun, 9 Aug 2026 19:48:06 -0400 Subject: [PATCH] Add trip history and route replay Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- PLAN.md | 69 +++++++-- app.py | 15 ++ modules/app_obd.py | 10 +- modules/app_sql.py | 256 +++++++++++++++++++++++++++++++++- modules/app_trip.py | 243 ++++++++++++++++++++++++++++++++ readme.md | 15 +- static/js/history.js | 221 +++++++++++++++++++++++++++++ static/yaris.css | 5 + templates/base.html | 5 + templates/history.html | 74 ++++++++++ tests/test_app_trip.py | 152 ++++++++++++++++++++ tests/test_tracking_logger.py | 45 ++++++ tests/test_trip_sql.py | 73 ++++++++++ 13 files changed, 1158 insertions(+), 25 deletions(-) create mode 100644 modules/app_trip.py create mode 100644 static/js/history.js create mode 100644 templates/history.html create mode 100644 tests/test_app_trip.py create mode 100644 tests/test_tracking_logger.py create mode 100644 tests/test_trip_sql.py diff --git a/PLAN.md b/PLAN.md index 5a77785..b92c9be 100644 --- a/PLAN.md +++ b/PLAN.md @@ -925,20 +925,61 @@ Same file-browser treatment for videos already imported to the Pi's own disk returned to exactly 87 uploaded files. Background uploads were paused for the disposable test, restored afterward, and the service remained active. -### Phase 18: Trip history & replay โ€” ๐Ÿšง PLANNED -**New phase.** Right now the `tracking` table (Phase 7) is just a raw stream of GPS/speed points -with no concept of a discrete "trip" - there's no way to look back and see "trip #47, left home -8:03am, arrived work 8:31am, 22km." Add: -- Trip boundary detection: group existing tracking rows into discrete trips based on speed>0 - gaps (a simple, dependency-free heuristic - e.g. a gap of N minutes with no motion ends a trip), - computed either live as rows are logged or in a batch pass over existing data. Does not need to - wait on Phase 13's OBD hardening - can ship with GPS-only boundary detection first, refine with - ignition-on/off signals from OBD later if useful. -- A `trips` table (start/end time, start/end coordinates, distance, duration) in - `modules/app_sql.py`. -- A new history page/list (built on the Phase 15 shared list component) showing past trips, and - a map replay view (reusing the existing offline-tile Leaflet setup from Phase 3/4) that draws - the recorded route for a selected trip. +### Phase 18: Trip history & replay โ€” โœ… DONE +Added a derived, idempotently rebuilt `trips` index over the original non-destructive `tracking` +stream, plus `/api/trips`, bounded `/api/trip/` route responses, and a responsive `/history` +page using the Phase 15 sortable list and the existing offline Leaflet bundle. Trip rows and replay +details show distance in kilometres/miles and average/maximum speed in km/h and mph; speed metrics +prefer OBD wheel speed and fall back to plausible GPS-derived segment speed. The map replay includes +a timestamped scrubber and play/pause timeline that moves a vehicle-position marker through every +recorded point while showing its date, time, and recorded speed. + +**Motion-boundary decision after auditing the real Pi data:** the logger records every five seconds +for as long as GPS has a fix, including while parked, so timestamp gaps alone are not trip +boundaries. OBD wheel speed above zero is the primary motion signal. When OBD is absent or stuck at +zero, three sustained GPS displacements of at least eight metres at normal logger cadence act as +the fallback; isolated jumps, implausible jumps, malformed coordinates, stationary jitter, clock +rollback, and five-minute motion gaps are handled explicitly. Trip summaries contain start/end +time and coordinates, Haversine distance, duration, and point count. Route responses retain both +endpoints, filter implausible GPS outliers, and downsample only above 2,000 points. OBD samples are +used only while their last successful query is fresh, preventing a disconnected adapter's stale +last-known speed from creating a false trip. Rebuilds process an overlap after the persisted +high-water mark plus the latest trip rather than loading the unbounded full history every time. + +**Implemented and verified 2026-08-09:** +- Audited the live database before choosing the heuristic: 5,132 initial rows covering about 25.5 + hours, no null coordinates, all stored speeds zero, only 114 five-decimal coordinate positions, + and one old NUL-prefixed latitude. Future coordinates are normalized before insertion; invalid + snapshots are logged and rejected instead of poisoning trip calculations. +- Replaced unconditional five-second database writes with motion-aware persistence. Sensors are + still checked every five seconds, but rows are written only for the first fix, fresh OBD motion, + at least eight metres of accumulated GPS displacement, or a one-minute stationary heartbeat. + The Pi normally runs only while the car has power, so this is not presented as a full-day parked + storage problem; it reduces needless writes during bench testing and long idling while retaining + enough stopped points for traffic pauses in the replay timeline. +- Nine focused detector tests cover sustained GPS fallback, parked jitter/malformed coordinates, + inactivity splitting, OBD wheel-speed detection and slow-distance calculation, clock rollback, + route outlier filtering, and route downsampling. SQLite incremental-build integration was also + exercised against a disposable database, and five logger tests cover first-fix, parked, + wheel-speed, GPS-fallback, and heartbeat decisions. A copy of the real database + produced zero false trips across 5,179 parked rows. +- Deployed the uncommitted implementation to the Pi and built the schema against the real database + without modifying any original tracking rows. Inserted an eight-point disposable GPS route while + the service was stopped; the live API detected one 0.078km/35-second trip and returned all eight + ordered replay points. Restored the exact pre-test database afterward, confirmed zero synthetic + rows, `PRAGMA quick_check=ok`, zero derived trips, and an active service. +- Browser-tested `http://yaris.local/history` through the deployed mDNS/port-80 redirect at a + 390x844 mobile viewport. Empty state, populated sortable row, replay action, offline route + polyline following every route point, start/end markers, metric/imperial distance and speed + summary, and no-horizontal-overflow behavior passed. Offline tiles + now attach only after selecting a route, avoiding irrelevant missing-tile requests around the + generic empty-map center. +- With user approval, removed the accumulated stationary bench backlog after verification: + 5,454 local SQLite rows and 2,907 remotely synced Postgres rows, all with zero/null speed and no + detected trip. Preserved a dated pre-cleanup SQLite backup under `/home/yaris/backups/`, vacuumed + and integrity-checked SQLite, vacuum-analyzed Postgres, and left unrelated settings, upload + markers, DTCs, and video data untouched. The deployed motion-aware logger then proved two + consecutive stationary heartbeats were 60.02 seconds apart rather than five seconds. ### Phase 19: Pi health monitoring card โ€” ๐Ÿšง PLANNED **New phase, small and standalone - no dependencies.** CPU temperature, free disk space, uptime, diff --git a/app.py b/app.py index c848cff..e28ea11 100644 --- a/app.py +++ b/app.py @@ -28,6 +28,21 @@ def dtc(): def dash(): return render_template('dash.html', account = account, car = car) +@app.route('/history') +def history(): + return render_template('history.html', account=account, car=car) + +@app.route('/api/trips') +def trips(): + return jsonify({"trips": app_sql.list_trips()}) + +@app.route('/api/trip/') +def trip_route(trip_id): + trip = app_sql.get_trip_route(trip_id) + if trip is None: + return jsonify(error=f"Trip not found: {trip_id}"), 404 + return jsonify(trip) + @app.route('/api/dash/devices') def dash_devices(): devices = app_dash.list_usb_storage_partitions() diff --git a/modules/app_obd.py b/modules/app_obd.py index f9f7d19..b2d01af 100644 --- a/modules/app_obd.py +++ b/modules/app_obd.py @@ -14,6 +14,7 @@ SENSOR_POLL_INTERVAL = 5 status = None speed = 0 +speed_updated_at = None rpm = None coolant_temp = None fuel_level = None @@ -233,6 +234,12 @@ def get_speedtime(): return [time_now, speed] +def get_fresh_speed(max_age=5): + if speed_updated_at is None or time.monotonic() - speed_updated_at > max_age: + return None + return speed + + def obd_connection(): """Emit current state for a newly connected browser and ensure OBD is running.""" start_obd() @@ -245,7 +252,7 @@ def obd_connection(): def obd_speed(): - global speed + global speed, speed_updated_at with obd_speed_lock: while not stop_speed.is_set() and _is_car_connected(): response = _query(obd.commands.SPEED) @@ -256,6 +263,7 @@ def obd_speed(): print(app_gps.get_gps()) else: speed = round(response.value.to("kph").magnitude, 1) + speed_updated_at = time.monotonic() print("Speed:", speed, "kph") app_events.socketio.emit("obd_speed", get_speedtime()) diff --git a/modules/app_sql.py b/modules/app_sql.py index 9b8ead0..9851937 100644 --- a/modules/app_sql.py +++ b/modules/app_sql.py @@ -6,6 +6,7 @@ from sqlite3 import Error from threading import Thread, Lock from modules import app_car, app_account, app_gps +from modules import app_trip # Resolve db/sql/ relative to the repo root regardless of the process's cwd # (previously a relative "../db/sql/" path, which broke depending on how/from @@ -20,10 +21,16 @@ sql_file = account.name + "_" + sCarModel + ".db" sql_path = os.path.join(_SQL_DIR, sql_file) LOG_INTERVAL = 5 # seconds between trip-tracking snapshots +IDLE_LOG_INTERVAL = 60 +GPS_LOG_DISTANCE_METERS = 8 +TRIP_REFRESH_INTERVAL = 30 +TRIP_REBUILD_OVERLAP_ROWS = 120 +TRIP_BUILD_VERSION = 2 _conn = None _conn_lock = Lock() _log_thread = None +_last_trip_refresh = 0 SQL_CREATE_TRACKING_TABLE = """CREATE TABLE IF NOT EXISTS tracking ( id integer PRIMARY KEY, @@ -69,6 +76,29 @@ SQL_CREATE_DASHCAM_HASH_CACHE_TABLE = """CREATE TABLE IF NOT EXISTS dashcam_hash SHA256 text NOT NULL );""" +SQL_CREATE_TRIPS_TABLE = """CREATE TABLE IF NOT EXISTS trips ( + id integer PRIMARY KEY, + START_TRACKING_ID integer NOT NULL, + END_TRACKING_ID integer NOT NULL, + START_DATETIME datetime NOT NULL, + END_DATETIME datetime NOT NULL, + START_GPSX float NOT NULL, + START_GPSY float NOT NULL, + END_GPSX float NOT NULL, + END_GPSY float NOT NULL, + DISTANCE_KM float NOT NULL, + DURATION_SECONDS integer NOT NULL, + POINT_COUNT integer NOT NULL, + AVERAGE_SPEED_KMH float NOT NULL DEFAULT 0, + MAX_SPEED_KMH float NOT NULL DEFAULT 0 + );""" + +SQL_CREATE_TRIP_BUILD_STATE_TABLE = """CREATE TABLE IF NOT EXISTS trip_build_state ( + id integer PRIMARY KEY CHECK (id = 1), + LAST_TRACKING_ID integer NOT NULL, + BUILD_VERSION integer NOT NULL DEFAULT 1 + );""" + def create_conn(db_file): """Open a SQLite connection to db_file, creating its parent folder if needed. @@ -111,7 +141,11 @@ def get_conn(): create_table(_conn, SQL_CREATE_SETTINGS_TABLE) create_table(_conn, SQL_CREATE_DASHCAM_UPLOADS_TABLE) create_table(_conn, SQL_CREATE_DASHCAM_HASH_CACHE_TABLE) + create_table(_conn, SQL_CREATE_TRIPS_TABLE) + create_table(_conn, SQL_CREATE_TRIP_BUILD_STATE_TABLE) _migrate_tracking_table(_conn) + _migrate_trips_table(_conn) + _migrate_trip_build_state_table(_conn) else: print("Error! cannot create the database connection.") return _conn @@ -126,20 +160,212 @@ def _migrate_tracking_table(conn): conn.commit() +def _migrate_trips_table(conn): + existing_columns = { + row[1] for row in conn.execute("PRAGMA table_info(trips)").fetchall() + } + additions = { + "AVERAGE_SPEED_KMH": "float NOT NULL DEFAULT 0", + "MAX_SPEED_KMH": "float NOT NULL DEFAULT 0", + } + for name, definition in additions.items(): + if name not in existing_columns: + conn.execute(f"ALTER TABLE trips ADD COLUMN {name} {definition}") + conn.commit() + + +def _migrate_trip_build_state_table(conn): + existing_columns = { + row[1] for row in conn.execute("PRAGMA table_info(trip_build_state)").fetchall() + } + if "BUILD_VERSION" not in existing_columns: + conn.execute( + "ALTER TABLE trip_build_state " + "ADD COLUMN BUILD_VERSION integer NOT NULL DEFAULT 1" + ) + conn.commit() + + def insert_tracking(gps_x, gps_y, speed, odometer=None): """Insert a single trip-tracking snapshot row (GPS position + speed).""" conn = get_conn() if conn is None: - return + return False sql = """INSERT INTO tracking (VN, NAME, DATETIME, GPSX, GPSY, SPEED, ODOMETER) VALUES (?, ?, ?, ?, ?, ?, ?)""" with _conn_lock: try: - conn.execute(sql, (car.model, account.name, datetime.now(), gps_x, gps_y, speed, odometer)) + latitude = float(str(gps_x).replace("\x00", "").strip()) + longitude = float(str(gps_y).replace("\x00", "").strip()) + conn.execute( + sql, + (car.model, account.name, datetime.now(), latitude, longitude, speed, odometer), + ) conn.commit() - except Error as e: + return True + except (Error, TypeError, ValueError) as e: print("Failed to insert tracking row:", e) + return False + + +def should_log_tracking(position, speed, last_position, seconds_since_log): + if last_position is None: + return True + if speed is not None and speed > 1: + return True + distance_meters = app_trip.haversine_km(last_position, position) * 1000 + return ( + distance_meters >= GPS_LOG_DISTANCE_METERS + or seconds_since_log >= IDLE_LOG_INTERVAL + ) + + +def refresh_trips(force=False): + """Rebuild the derived trip index when new raw tracking rows are available.""" + global _last_trip_refresh + conn = get_conn() + if conn is None: + return 0 + with _conn_lock: + last_tracking_id = conn.execute( + "SELECT COALESCE(MAX(id), 0) FROM tracking" + ).fetchone()[0] + state = conn.execute( + "SELECT LAST_TRACKING_ID, BUILD_VERSION " + "FROM trip_build_state WHERE id = 1" + ).fetchone() + recently_refreshed = ( + _last_trip_refresh + and time.monotonic() - _last_trip_refresh < TRIP_REFRESH_INTERVAL + ) + if ( + not force + and state is not None + and state[1] == TRIP_BUILD_VERSION + and (state[0] == last_tracking_id or recently_refreshed) + ): + return conn.execute("SELECT COUNT(*) FROM trips").fetchone()[0] + + rebuild_start_id = 1 + if not force and state is not None and state[1] == TRIP_BUILD_VERSION: + rebuild_start_id = max(1, state[0] - TRIP_REBUILD_OVERLAP_ROWS) + intersecting_trip = conn.execute( + """SELECT MIN(START_TRACKING_ID) FROM trips + WHERE END_TRACKING_ID >= ?""", + (rebuild_start_id,), + ).fetchone() + if intersecting_trip[0] is not None: + rebuild_start_id = min(rebuild_start_id, intersecting_trip[0]) + rows = conn.execute( + """SELECT id, DATETIME, GPSX, GPSY, SPEED + FROM tracking WHERE id >= ? ORDER BY id ASC""", + (rebuild_start_id,), + ).fetchall() + + trips = app_trip.build_trip_summaries(rows) + with _conn_lock: + try: + conn.execute( + "DELETE FROM trips WHERE END_TRACKING_ID >= ?", + (rebuild_start_id,), + ) + conn.executemany( + """INSERT INTO trips ( + id, START_TRACKING_ID, END_TRACKING_ID, + START_DATETIME, END_DATETIME, + START_GPSX, START_GPSY, END_GPSX, END_GPSY, + DISTANCE_KM, DURATION_SECONDS, POINT_COUNT, + AVERAGE_SPEED_KMH, MAX_SPEED_KMH + ) VALUES ( + :id, :start_tracking_id, :end_tracking_id, + :start_datetime, :end_datetime, + :start_gpsx, :start_gpsy, :end_gpsx, :end_gpsy, + :distance_km, :duration_seconds, :point_count, + :average_speed_kmh, :max_speed_kmh + )""", + trips, + ) + conn.execute( + """INSERT INTO trip_build_state ( + id, LAST_TRACKING_ID, BUILD_VERSION + ) VALUES (1, ?, ?) + ON CONFLICT(id) DO UPDATE + SET LAST_TRACKING_ID = excluded.LAST_TRACKING_ID, + BUILD_VERSION = excluded.BUILD_VERSION""", + (last_tracking_id, TRIP_BUILD_VERSION), + ) + conn.commit() + _last_trip_refresh = time.monotonic() + except Error: + conn.rollback() + raise + with _conn_lock: + return conn.execute("SELECT COUNT(*) FROM trips").fetchone()[0] + + +def list_trips(): + refresh_trips() + conn = get_conn() + if conn is None: + return [] + with _conn_lock: + rows = conn.execute( + """SELECT id, START_DATETIME, END_DATETIME, + START_GPSX, START_GPSY, END_GPSX, END_GPSY, + DISTANCE_KM, DURATION_SECONDS, POINT_COUNT, + AVERAGE_SPEED_KMH, MAX_SPEED_KMH + FROM trips ORDER BY START_DATETIME DESC""" + ).fetchall() + return [{ + "id": row[0], + "start_datetime": row[1], + "end_datetime": row[2], + "start": {"latitude": row[3], "longitude": row[4]}, + "end": {"latitude": row[5], "longitude": row[6]}, + "distance_km": row[7], + "duration_seconds": row[8], + "point_count": row[9], + "average_speed_kmh": row[10], + "max_speed_kmh": row[11], + } for row in rows] + + +def get_trip_route(trip_id, max_points=2000): + refresh_trips() + conn = get_conn() + if conn is None: + return None + with _conn_lock: + trip = conn.execute( + """SELECT id, START_DATETIME, END_DATETIME, DISTANCE_KM, + DURATION_SECONDS, POINT_COUNT, + AVERAGE_SPEED_KMH, MAX_SPEED_KMH, + START_TRACKING_ID, END_TRACKING_ID + FROM trips WHERE id = ?""", + (trip_id,), + ).fetchone() + if trip is None: + return None + rows = conn.execute( + """SELECT id, DATETIME, GPSX, GPSY, SPEED + FROM tracking WHERE id BETWEEN ? AND ? + ORDER BY DATETIME ASC, id ASC""", + (trip[8], trip[9]), + ).fetchall() + return { + "trip": { + "id": trip[0], + "start_datetime": trip[1], + "end_datetime": trip[2], + "distance_km": trip[3], + "duration_seconds": trip[4], + "point_count": trip[5], + "average_speed_kmh": trip[6], + "max_speed_kmh": trip[7], + }, + "points": app_trip.route_points(rows, max_points=max_points), + } def get_setting(key, default=None): @@ -307,13 +533,33 @@ def _log_loop(): # (app_obd.py imports app_sql, and app_sql needs app_obd.speed here). from modules import app_obd + last_position = None + last_logged_at = None while True: gps_x = app_gps.LocationX gps_y = app_gps.LocationY - speed = app_obd.speed + speed = app_obd.get_fresh_speed() if gps_x is not None and gps_y is not None: - insert_tracking(gps_x, gps_y, speed) + try: + position = ( + float(str(gps_x).replace("\x00", "").strip()), + float(str(gps_y).replace("\x00", "").strip()), + ) + except (TypeError, ValueError) as error: + print("Skipping invalid GPS tracking snapshot:", error) + else: + now = time.monotonic() + seconds_since_log = ( + now - last_logged_at + if last_logged_at is not None + else IDLE_LOG_INTERVAL + ) + if should_log_tracking( + position, speed, last_position, seconds_since_log + ) and insert_tracking(position[0], position[1], speed): + last_position = position + last_logged_at = now time.sleep(LOG_INTERVAL) diff --git a/modules/app_trip.py b/modules/app_trip.py new file mode 100644 index 0000000..4b121d8 --- /dev/null +++ b/modules/app_trip.py @@ -0,0 +1,243 @@ +import math +from datetime import datetime + + +INACTIVITY_SECONDS = 5 * 60 +GPS_MOTION_METERS = 8 +MIN_GPS_MOTION_SAMPLES = 3 +MAX_GPS_SAMPLE_GAP_SECONDS = 30 +MAX_PLAUSIBLE_SPEED_KMH = 220 +EARTH_RADIUS_KM = 6371.0088 + + +def _number(value): + if value is None: + return None + try: + return float(str(value).replace("\x00", "").strip()) + except (TypeError, ValueError): + return None + + +def _timestamp(value): + try: + return datetime.fromisoformat(str(value)) + except (TypeError, ValueError): + return None + + +def haversine_km(left, right): + lat1, lon1 = map(math.radians, left) + lat2, lon2 = map(math.radians, right) + latitude_delta = lat2 - lat1 + longitude_delta = lon2 - lon1 + value = ( + math.sin(latitude_delta / 2) ** 2 + + math.cos(lat1) * math.cos(lat2) + * math.sin(longitude_delta / 2) ** 2 + ) + return EARTH_RADIUS_KM * 2 * math.atan2(math.sqrt(value), math.sqrt(1 - value)) + + +def normalize_tracking_rows(rows): + normalized = [] + for row in rows: + timestamp = _timestamp(row[1]) + latitude = _number(row[2]) + longitude = _number(row[3]) + speed = _number(row[4]) or 0 + if timestamp is None or latitude is None or longitude is None: + continue + if not (-90 <= latitude <= 90 and -180 <= longitude <= 180): + continue + normalized.append({ + "id": int(row[0]), + "datetime": timestamp, + "latitude": latitude, + "longitude": longitude, + "speed": max(0, speed), + }) + return normalized + + +def _segment(row, previous): + seconds = (row["datetime"] - previous["datetime"]).total_seconds() + if seconds <= 0: + return 0, 0 + distance_km = haversine_km( + (previous["latitude"], previous["longitude"]), + (row["latitude"], row["longitude"]), + ) + speed_kmh = distance_km / seconds * 3600 + if speed_kmh > MAX_PLAUSIBLE_SPEED_KMH: + return 0, speed_kmh + return distance_km, speed_kmh + + +def build_trip_summaries(raw_rows): + rows = normalize_tracking_rows(raw_rows) + if len(rows) < 2: + return [] + + obd_motion = [] + gps_motion = [] + for index in range(1, len(rows)): + seconds = ( + rows[index]["datetime"] - rows[index - 1]["datetime"] + ).total_seconds() + distance_km, _ = _segment(rows[index], rows[index - 1]) + has_obd_motion = rows[index]["speed"] > 1 + has_gps_motion = ( + 0 < seconds <= MAX_GPS_SAMPLE_GAP_SECONDS + and distance_km * 1000 >= GPS_MOTION_METERS + ) + if has_obd_motion: + obd_motion.append({ + "index": index, + "obd": True, + "distance_km": distance_km, + }) + if has_gps_motion: + gps_motion.append({ + "index": index, + "obd": False, + "distance_km": distance_km, + }) + + qualified_gps_motion = [] + gps_sequences = [] + for event in gps_motion: + if ( + not gps_sequences + or ( + rows[event["index"]]["datetime"] + - rows[gps_sequences[-1][-1]["index"]]["datetime"] + ).total_seconds() > MAX_GPS_SAMPLE_GAP_SECONDS + ): + gps_sequences.append([event]) + else: + gps_sequences[-1].append(event) + for sequence in gps_sequences: + if len(sequence) >= MIN_GPS_MOTION_SAMPLES: + qualified_gps_motion.extend(sequence) + + motion_by_index = {} + for event in obd_motion + qualified_gps_motion: + existing = motion_by_index.get(event["index"]) + if existing is None: + motion_by_index[event["index"]] = event.copy() + else: + existing["obd"] = existing["obd"] or event["obd"] + existing["distance_km"] = max( + existing["distance_km"], event["distance_km"] + ) + motion = [ + motion_by_index[index] for index in sorted(motion_by_index) + ] + + groups = [] + for event in motion: + event_gap = ( + ( + rows[event["index"]]["datetime"] + - rows[groups[-1][-1]["index"]]["datetime"] + ).total_seconds() + if groups + else None + ) + if ( + not groups + or event_gap <= 0 + or event_gap > INACTIVITY_SECONDS + ): + groups.append([event]) + else: + groups[-1].append(event) + + trips = [] + for events in groups: + if not any(event["obd"] for event in events) and len(events) < MIN_GPS_MOTION_SAMPLES: + continue + + start_index = max(0, events[0]["index"] - 1) + end_index = events[-1]["index"] + trip_rows = rows[start_index:end_index + 1] + distance_km = 0 + segment_speeds = [] + obd_speeds = [] + has_obd_motion = any(event["obd"] for event in events) + for previous, row in zip(trip_rows, trip_rows[1:]): + segment_km, segment_speed = _segment(row, previous) + minimum_distance_meters = 1 if has_obd_motion else GPS_MOTION_METERS + if segment_km * 1000 >= minimum_distance_meters: + distance_km += segment_km + segment_speeds.append(segment_speed) + if row["speed"] > 0: + obd_speeds.append(row["speed"]) + + start = trip_rows[0] + end = trip_rows[-1] + duration_seconds = max( + 0, int((end["datetime"] - start["datetime"]).total_seconds()) + ) + if not any(event["obd"] for event in events) and duration_seconds < 10: + continue + + trips.append({ + "id": start["id"], + "start_tracking_id": start["id"], + "end_tracking_id": end["id"], + "start_datetime": start["datetime"].isoformat(sep=" "), + "end_datetime": end["datetime"].isoformat(sep=" "), + "start_gpsx": start["latitude"], + "start_gpsy": start["longitude"], + "end_gpsx": end["latitude"], + "end_gpsy": end["longitude"], + "distance_km": round(distance_km, 3), + "duration_seconds": duration_seconds, + "point_count": len(trip_rows), + "average_speed_kmh": round( + distance_km / duration_seconds * 3600, 1 + ) if duration_seconds else 0, + "max_speed_kmh": round( + max(obd_speeds or segment_speeds or [0]), 1 + ), + }) + return trips + + +def route_points(raw_rows, max_points=2000): + rows = normalize_tracking_rows(raw_rows) + filtered_rows = [] + for row in rows: + if filtered_rows: + _, segment_speed = _segment(row, filtered_rows[-1]) + if segment_speed > MAX_PLAUSIBLE_SPEED_KMH: + continue + filtered_rows.append(row) + rows = filtered_rows + for index, row in enumerate(rows): + row["speed_source"] = "obd" if row["speed"] > 0 else "gps" + if row["speed"] <= 0 and index: + distance_km, segment_speed = _segment(row, rows[index - 1]) + row["speed"] = ( + segment_speed + if distance_km * 1000 >= GPS_MOTION_METERS + else 0 + ) + if len(rows) <= max_points: + selected = rows + else: + indexes = { + round(index * (len(rows) - 1) / (max_points - 1)) + for index in range(max_points) + } + selected = [row for index, row in enumerate(rows) if index in indexes] + return [{ + "id": row["id"], + "datetime": row["datetime"].isoformat(sep=" "), + "latitude": row["latitude"], + "longitude": row["longitude"], + "speed": round(row["speed"], 1), + "speed_source": row["speed_source"], + } for row in selected] diff --git a/readme.md b/readme.md index d626357..94f2579 100644 --- a/readme.md +++ b/readme.md @@ -55,7 +55,11 @@ i have a dream independently while trip logs keep syncing - ๐Ÿ“ก **Optional cellular fallback** through the SIM868 for small trip-log uploads when normal WiFi/Ethernet is unavailable (dashcam video stays off cellular) -- ๐Ÿ—„๏ธ **Local-first SQLite logging** with retry-safe remote sync and automatic backoff +- ๐Ÿ—„๏ธ **Motion-aware local-first SQLite logging** โ€” five-second motion checks without five-second + parked writes, plus retry-safe remote sync and automatic backoff +- ๐Ÿ›ฃ๏ธ **Trip history and route replay** โ€” detects motion from OBD wheel speed with sustained GPS + displacement as a fallback, summarizes distance and speed in metric/imperial units, and replays + each route as a line on the offline map with a timestamped play/pause timeline and moving marker - ๐Ÿ–ฅ๏ธ **Offline web UI** built with Flask, Jinja, Bootstrap 5, Socket.IO, Chart.js, and Leaflet, all served locally with a shared responsive page shell - ๐Ÿ”Œ **Headless Pi deployment** as a self-restarting `systemd` service with a 10-second BCM2835 @@ -92,9 +96,10 @@ For the remote server (receives trip logs + dashcam uploads), see [server/README ## Project status -This project was revived after a couple of years on the shelf. Phases 1-13 and 15-17 are complete: +This project was revived after a couple of years on the shelf. Phases 1-13 and 15-18 are complete: real GNSS, hardened OBD support, offline maps/UI, automatic WiFi hotspot switching, local logging, -dashcam import, resumable remote upload, deployment, and the shared UI foundation are implemented. +dashcam import, resumable remote upload, trip history/replay, deployment, and the shared UI +foundation are implemented. The pipeline has been exercised on the real Pi and dashcam, including a complete 28GB import and remote upload. Actual OBD PID availability still needs the first in-car test. @@ -104,6 +109,6 @@ partition, and clean shutdown still needs delayed-power or ignition-sense hardwa The browser-based phone relay was removed after real-world testing confirmed that mobile browsers cannot reliably use the Pi's WiFi and cellular data simultaneously. A native Android companion is -planned later instead. The next roadmap work is trip history, followed by Pi health, alerts, settings, and a final UI -polish pass. Full phase history, verification notes, and current scope live in +planned later instead. The next roadmap work is Pi health, followed by alerts, settings, and a +final UI polish pass. Full phase history, verification notes, and current scope live in [PLAN.md](./PLAN.md). diff --git a/static/js/history.js b/static/js/history.js new file mode 100644 index 0000000..6ed9a83 --- /dev/null +++ b/static/js/history.js @@ -0,0 +1,221 @@ +(function () { + "use strict"; + + const body = document.querySelector("#trip-list [data-data-list-body]"); + const loading = document.getElementById("trip-loading"); + const error = document.getElementById("trip-error"); + const summary = document.getElementById("trip-summary"); + const timeline = document.getElementById("trip-timeline"); + const scrubber = document.getElementById("trip-scrubber"); + const playButton = document.getElementById("trip-play"); + const replayTime = document.getElementById("trip-replay-time"); + const replaySpeed = document.getElementById("trip-replay-speed"); + const map = L.map("history-map", { minZoom: 8, maxZoom: 15 }).setView([48.505, -0.09], 12); + const tiles = L.tileLayer("/static/vendor/tiles/{z}/{x}/{y}.png", { + minZoom: 8, + maxZoom: 15, + attribution: "© MapTiler © OpenStreetMap contributors", + }); + let routeLayer = null; + let vehicleMarker = null; + let replayPoints = []; + let replayTimer = null; + let replayController = null; + + function formatDuration(seconds) { + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + if (hours) { + return `${hours}h ${minutes}m`; + } + return minutes ? `${minutes}m` : `${seconds}s`; + } + + function formatDate(value) { + return new Date(value.replace(" ", "T")).toLocaleString(); + } + + function formatDistance(kilometers) { + const miles = kilometers * 0.621371; + return `${kilometers.toFixed(kilometers < 1 ? 2 : 1)} km / ` + + `${miles.toFixed(miles < 1 ? 2 : 1)} mi`; + } + + function formatSpeed(kilometersPerHour) { + return `${kilometersPerHour.toFixed(1)} km/h / ` + + `${(kilometersPerHour * 0.621371).toFixed(1)} mph`; + } + + function showError(message) { + error.textContent = message; + error.hidden = false; + } + + function stopReplay() { + if (replayTimer) { + window.clearInterval(replayTimer); + replayTimer = null; + } + playButton.innerHTML = ''; + playButton.setAttribute("aria-label", "Play trip replay"); + } + + function showReplayPoint(index) { + const point = replayPoints[index]; + if (!point) { + return; + } + const coordinate = [point.latitude, point.longitude]; + vehicleMarker.setLatLng(coordinate); + scrubber.value = index; + replayTime.textContent = formatDate(point.datetime); + const source = point.speed_source === "obd" ? "OBD wheel" : "GPS"; + replaySpeed.textContent = `${source} speed: ${formatSpeed(point.speed)}`; + } + + function initializeTimeline(points) { + stopReplay(); + replayPoints = points; + scrubber.max = Math.max(0, points.length - 1); + scrubber.value = 0; + timeline.hidden = points.length === 0; + if (vehicleMarker) { + vehicleMarker.remove(); + } + if (!points.length) { + vehicleMarker = null; + return; + } + vehicleMarker = L.circleMarker( + [points[0].latitude, points[0].longitude], + { + radius: 8, + color: "#fff", + weight: 3, + fillColor: "#e74a3b", + fillOpacity: 1, + } + ).bindTooltip("Vehicle position").addTo(map); + showReplayPoint(0); + } + + async function replayTrip(tripId) { + error.hidden = true; + if (replayController) { + replayController.abort(); + } + replayController = new AbortController(); + const response = await fetch( + `/api/trip/${tripId}`, + { signal: replayController.signal } + ); + const result = await response.json(); + if (!response.ok) { + throw new Error(result.error || "Could not load trip route."); + } + + if (routeLayer) { + routeLayer.remove(); + } + if (!map.hasLayer(tiles)) { + tiles.addTo(map); + } + const coordinates = result.points.map(function (point) { + return [point.latitude, point.longitude]; + }); + routeLayer = L.featureGroup(); + L.polyline(coordinates, { color: "#4e73df", weight: 5 }).addTo(routeLayer); + if (coordinates.length) { + L.marker(coordinates[0]).bindPopup("Trip start").addTo(routeLayer); + L.marker(coordinates[coordinates.length - 1]).bindPopup("Trip end").addTo(routeLayer); + } + routeLayer.addTo(map); + if (coordinates.length > 1) { + map.fitBounds(routeLayer.getBounds(), { padding: [24, 24] }); + } else if (coordinates.length === 1) { + map.setView(coordinates[0], 15); + } + summary.textContent = `${formatDistance(result.trip.distance_km)} ยท ` + + `${formatDuration(result.trip.duration_seconds)} ยท ` + + `avg ${formatSpeed(result.trip.average_speed_kmh)} ยท ` + + `max ${formatSpeed(result.trip.max_speed_kmh)} ยท ` + + `${result.points.length} route points`; + initializeTimeline(result.points); + } + + function renderTrips(trips) { + body.replaceChildren(); + trips.forEach(function (trip) { + const row = document.createElement("tr"); + row.dataset.started = trip.start_datetime; + row.dataset.distance = trip.distance_km; + row.dataset.duration = trip.duration_seconds; + row.dataset.maxSpeed = trip.max_speed_kmh; + row.innerHTML = ` + ${formatDate(trip.start_datetime)} + ${formatDistance(trip.distance_km)} + ${formatDuration(trip.duration_seconds)} + ${formatSpeed(trip.max_speed_kmh)} + + + `; + body.appendChild(row); + }); + } + + body.addEventListener("click", function (event) { + const button = event.target.closest("[data-trip-id]"); + if (!button) { + return; + } + replayTrip(button.dataset.tripId).catch(function (replayError) { + if (replayError.name !== "AbortError") { + showError(replayError.message); + } + }); + }); + + scrubber.addEventListener("input", function () { + stopReplay(); + showReplayPoint(Number(scrubber.value)); + }); + + playButton.addEventListener("click", function () { + if (replayTimer) { + stopReplay(); + return; + } + if (Number(scrubber.value) >= replayPoints.length - 1) { + showReplayPoint(0); + } + playButton.innerHTML = ''; + playButton.setAttribute("aria-label", "Pause trip replay"); + replayTimer = window.setInterval(function () { + const nextIndex = Number(scrubber.value) + 1; + if (nextIndex >= replayPoints.length) { + stopReplay(); + return; + } + showReplayPoint(nextIndex); + }, 500); + }); + + fetch("/api/trips") + .then(function (response) { + if (!response.ok) { + throw new Error("Could not load trip history."); + } + return response.json(); + }) + .then(function (result) { + renderTrips(result.trips); + }) + .catch(function (loadError) { + showError(loadError.message); + }) + .finally(function () { + loading.hidden = true; + }); +}()); diff --git a/static/yaris.css b/static/yaris.css index 7777ec8..ace037a 100644 --- a/static/yaris.css +++ b/static/yaris.css @@ -1,5 +1,10 @@ #map { height: 20rem; } +#history-map { + height: 36rem; + min-height: 24rem; +} + .app-sidebar { color: var(--bs-accordion-btn-focus-border-color); background: #000; diff --git a/templates/base.html b/templates/base.html index 4363f50..6cd9d0b 100644 --- a/templates/base.html +++ b/templates/base.html @@ -31,6 +31,11 @@ Dashcam +
  • + + Trip history + +
  • DTC diff --git a/templates/history.html b/templates/history.html new file mode 100644 index 0000000..552dd2b --- /dev/null +++ b/templates/history.html @@ -0,0 +1,74 @@ +{% extends "base.html" %} +{% from "components.html" import data_table %} + +{% block title %}Trip history - Yaris OBD{% endblock %} + +{% block styles %} + +{% endblock %} + +{% block content %} +
    +
    +

    Trip history

    +

    Review recorded drives and replay their routes offline.

    +
    +
    + +
    +
    +
    +
    +
    Recorded trips
    +
    +
    +
    + + Loading trips... +
    + + {{ data_table( + "trip-list", + [ + {"key": "started", "label": "Started", "sortable": true}, + {"key": "distance", "label": "Distance", "sortable": true}, + {"key": "duration", "label": "Duration", "sortable": true}, + {"key": "maxSpeed", "label": "Max speed", "sortable": true}, + {"key": "action", "label": "", "sortable": false} + ], + "No trips detected", + "Trip history appears after sustained GPS or OBD movement is recorded." + ) }} +
    +
    +
    +
    +
    +
    +
    Route replay
    + Select a trip +
    +
    +
    + +
    +
    +
    +
    +{% endblock %} + +{% block scripts %} + + +{% endblock %} diff --git a/tests/test_app_trip.py b/tests/test_app_trip.py new file mode 100644 index 0000000..3ea2213 --- /dev/null +++ b/tests/test_app_trip.py @@ -0,0 +1,152 @@ +import unittest +from datetime import datetime, timedelta + +from modules import app_trip + + +class TripBuilderTest(unittest.TestCase): + def rows_for_route(self, coordinates, speeds=None, start_id=1): + started = datetime(2026, 1, 1, 8) + speeds = speeds or [0] * len(coordinates) + return [ + ( + start_id + index, + started + timedelta(seconds=index * 5), + latitude, + longitude, + speeds[index], + ) + for index, (latitude, longitude) in enumerate(coordinates) + ] + + def test_builds_gps_trip_after_sustained_motion(self): + rows = self.rows_for_route([ + (45.5000, -73.5000), + (45.5001, -73.5000), + (45.5002, -73.5000), + (45.5003, -73.5000), + ]) + + trips = app_trip.build_trip_summaries(rows) + + self.assertEqual(len(trips), 1) + self.assertEqual(trips[0]["start_tracking_id"], 1) + self.assertEqual(trips[0]["end_tracking_id"], 4) + self.assertGreater(trips[0]["distance_km"], 0.03) + self.assertGreater(trips[0]["average_speed_kmh"], 0) + self.assertGreater(trips[0]["max_speed_kmh"], 0) + + def test_ignores_stationary_jitter_and_one_bad_coordinate(self): + rows = self.rows_for_route([ + (45.500000, -73.500000), + (45.500010, -73.500000), + ("\x0045.500020", -73.500000), + ("not-a-coordinate", -73.500000), + (45.500010, -73.500000), + ]) + + self.assertEqual(app_trip.build_trip_summaries(rows), []) + + def test_inactivity_gap_splits_trips(self): + first = self.rows_for_route([ + (45.5000, -73.5000), + (45.5001, -73.5000), + (45.5002, -73.5000), + (45.5003, -73.5000), + ]) + second = self.rows_for_route([ + (45.5100, -73.5100), + (45.5101, -73.5100), + (45.5102, -73.5100), + (45.5103, -73.5100), + ], start_id=5) + second = [ + (row[0], row[1] + timedelta(minutes=10), row[2], row[3], row[4]) + for row in second + ] + + trips = app_trip.build_trip_summaries(first + second) + + self.assertEqual(len(trips), 2) + + def test_obd_speed_detects_slow_trip(self): + rows = self.rows_for_route( + [(45.5, -73.5), (45.5, -73.5)], + speeds=[0, 5], + ) + + trips = app_trip.build_trip_summaries(rows) + + self.assertEqual(len(trips), 1) + self.assertEqual(trips[0]["max_speed_kmh"], 5) + + def test_obd_trip_counts_sub_threshold_gps_distance(self): + rows = self.rows_for_route( + [(45.5 + index * 0.00006, -73.5) for index in range(10)], + speeds=[5] * 10, + ) + + trips = app_trip.build_trip_summaries(rows) + + self.assertGreater(trips[0]["distance_km"], 0.05) + self.assertGreater(trips[0]["average_speed_kmh"], 0) + + def test_combined_obd_and_gps_motion_creates_one_trip(self): + rows = self.rows_for_route( + [(45.5 + index * 0.0001, -73.5) for index in range(5)], + speeds=[20] * 5, + ) + + trips = app_trip.build_trip_summaries(rows) + + self.assertEqual(len(trips), 1) + self.assertEqual(trips[0]["start_tracking_id"], 1) + self.assertEqual(trips[0]["end_tracking_id"], 5) + + def test_clock_rollback_does_not_merge_motion_episodes(self): + first = self.rows_for_route( + [(45.5, -73.5), (45.5, -73.5)], + speeds=[0, 5], + ) + second = self.rows_for_route( + [(45.51, -73.51), (45.51, -73.51)], + speeds=[0, 5], + start_id=3, + ) + second = [ + (row[0], row[1] - timedelta(hours=1), row[2], row[3], row[4]) + for row in second + ] + + trips = app_trip.build_trip_summaries(first + second) + + self.assertEqual(len(trips), 2) + + def test_route_downsampling_keeps_endpoints(self): + rows = self.rows_for_route( + [(45.5 + index / 10000, -73.5) for index in range(20)] + ) + + points = app_trip.route_points(rows, max_points=5) + + self.assertEqual(len(points), 5) + self.assertEqual(points[0]["id"], 1) + self.assertEqual(points[-1]["id"], 20) + self.assertEqual(points[0]["speed"], 0) + self.assertGreater(points[-1]["speed"], 0) + self.assertEqual(points[-1]["speed_source"], "gps") + + def test_route_filters_implausible_gps_outlier(self): + rows = self.rows_for_route([ + (45.5000, -73.5000), + (0, 0), + (45.5001, -73.5000), + ]) + + points = app_trip.route_points(rows) + + self.assertEqual([point["id"] for point in points], [1, 3]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tracking_logger.py b/tests/test_tracking_logger.py new file mode 100644 index 0000000..08c833b --- /dev/null +++ b/tests/test_tracking_logger.py @@ -0,0 +1,45 @@ +import unittest + +from modules import app_sql + + +class TrackingLoggerTest(unittest.TestCase): + def test_logs_first_fix(self): + self.assertTrue( + app_sql.should_log_tracking((45.5, -73.5), None, None, 0) + ) + + def test_does_not_persist_parked_five_second_samples(self): + self.assertFalse( + app_sql.should_log_tracking( + (45.500001, -73.5), 0, (45.5, -73.5), 5 + ) + ) + + def test_logs_fresh_obd_motion(self): + self.assertTrue( + app_sql.should_log_tracking( + (45.5, -73.5), 2, (45.5, -73.5), 5 + ) + ) + + def test_logs_accumulated_gps_motion_without_obd(self): + self.assertTrue( + app_sql.should_log_tracking( + (45.5001, -73.5), None, (45.5, -73.5), 10 + ) + ) + + def test_logs_stationary_heartbeat_after_one_minute(self): + self.assertTrue( + app_sql.should_log_tracking( + (45.5, -73.5), + 0, + (45.5, -73.5), + app_sql.IDLE_LOG_INTERVAL, + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_trip_sql.py b/tests/test_trip_sql.py new file mode 100644 index 0000000..6fa64ec --- /dev/null +++ b/tests/test_trip_sql.py @@ -0,0 +1,73 @@ +import os +import tempfile +import unittest +from datetime import datetime, timedelta + +from modules import app_sql + + +class TripSqlTest(unittest.TestCase): + def setUp(self): + handle, self.path = tempfile.mkstemp(suffix=".db") + os.close(handle) + self.original_path = app_sql.sql_path + app_sql._conn = None + app_sql._last_trip_refresh = 0 + app_sql.sql_path = self.path + self.conn = app_sql.get_conn() + + def tearDown(self): + if app_sql._conn is not None: + app_sql._conn.close() + app_sql._conn = None + app_sql._last_trip_refresh = 0 + app_sql.sql_path = self.original_path + os.unlink(self.path) + + def insert_motion(self, start_id, count, started, latitude): + rows = [ + ( + start_id + index, + "Yaris", + "test", + started + timedelta(seconds=index * 5), + latitude + index * 0.0001, + -73.5, + 20, + None, + 0, + ) + for index in range(count) + ] + self.conn.executemany( + """INSERT INTO tracking ( + id, VN, NAME, DATETIME, GPSX, GPSY, SPEED, ODOMETER, UPLOADED + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + rows, + ) + self.conn.commit() + + def test_incremental_rebuild_preserves_trip_before_overlap_boundary(self): + started = datetime(2026, 1, 1, 8) + self.insert_motion(1, 160, started, 45.5) + self.insert_motion(161, 10, started + timedelta(minutes=30), 45.6) + app_sql.refresh_trips(force=True) + original = self.conn.execute( + """SELECT START_TRACKING_ID, END_TRACKING_ID, DISTANCE_KM + FROM trips ORDER BY START_TRACKING_ID""" + ).fetchall() + self.assertEqual(len(original), 2) + + self.insert_motion(171, 1, started + timedelta(minutes=31), 45.601) + app_sql._last_trip_refresh = 0 + app_sql.refresh_trips() + rebuilt = self.conn.execute( + """SELECT START_TRACKING_ID, END_TRACKING_ID, DISTANCE_KM + FROM trips ORDER BY START_TRACKING_ID""" + ).fetchall() + + self.assertEqual(rebuilt[0], original[0]) + + +if __name__ == "__main__": + unittest.main()