Add offline and ntfy alerts

Co-authored-by: Copilot <[email protected]>
This commit is contained in:
Mathieu Moran
2026-08-09 20:28:42 -04:00
co-authored by Copilot
parent 8b664667e5
commit 96332adab6
12 changed files with 1098 additions and 21 deletions
+489
View File
@@ -0,0 +1,489 @@
import json
import sqlite3
import threading
import time
from urllib.parse import urlparse
import requests
from modules import app_dash, app_gps, app_health, app_obd, app_sql, app_video
CHECK_INTERVAL = 60
RETRY_INTERVAL = 5 * 60
DASHCAM_CHECK_INTERVAL = 5 * 60
POWER_EVENT_GRACE_SECONDS = 5 * 60
SETTING_DEFAULTS = {
"alerts_enabled": "0",
"alert_ntfy_url": "",
"alert_ntfy_token": "",
"alert_temperature_c": "80",
"alert_disk_percent": "90",
"alert_tracking_backlog": "1000",
"alert_video_backlog": "10",
"alert_gps_loss_seconds": "60",
"alert_repeat_seconds": "3600",
}
ALERT_TITLES = {
"mil": "Check-engine light",
"disk": "Pi storage nearly full",
"upload_backlog": "Upload backlog growing",
"temperature": "Pi temperature high",
"power": "Pi power or throttling warning",
"gps_loss": "GPS fix lost while moving",
}
_thread = None
_last_run_at = None
_last_run_error = None
_last_conditions = {}
_moving_started_at = None
_last_dashcam_check_at = None
_last_dashcam_usage = {"status": "not_checked"}
def _number_setting(key, minimum):
raw = app_sql.get_setting(key, SETTING_DEFAULTS[key])
try:
value = float(raw)
except (TypeError, ValueError):
return float(SETTING_DEFAULTS[key]), f"{key} must be numeric"
if value < minimum:
return float(SETTING_DEFAULTS[key]), f"{key} must be at least {minimum}"
return value, None
def get_config():
errors = {}
values = {}
minimums = {
"alert_temperature_c": 1,
"alert_disk_percent": 1,
"alert_tracking_backlog": 1,
"alert_video_backlog": 1,
"alert_gps_loss_seconds": 1,
"alert_repeat_seconds": 60,
}
for key, minimum in minimums.items():
values[key], error = _number_setting(key, minimum)
if error:
errors[key] = error
url = app_sql.get_setting("alert_ntfy_url", "")
parsed = urlparse(url)
if url and (parsed.scheme not in {"http", "https"} or not parsed.netloc):
errors["alert_ntfy_url"] = "alert_ntfy_url must be an http(s) URL"
values.update({
"enabled": app_sql.get_setting("alerts_enabled", "0") == "1",
"ntfy_url": url,
"ntfy_token": app_sql.get_setting("alert_ntfy_token", ""),
"errors": errors,
})
return values
def send_ntfy(url, token, title, message, priority="default", tags=None):
headers = {
"Title": title,
"Priority": priority,
}
if tags:
headers["Tags"] = ",".join(tags)
if token:
headers["Authorization"] = f"Bearer {token}"
response = requests.post(
url,
data=message.encode("utf-8"),
headers=headers,
timeout=5,
)
response.raise_for_status()
def _transport_error(error):
if isinstance(error, requests.Timeout):
return "Notification request timed out"
if isinstance(error, requests.HTTPError):
status_code = (
error.response.status_code
if error.response is not None
else "unknown"
)
return f"Notification server returned HTTP {status_code}"
if isinstance(error, requests.ConnectionError):
return "Notification connection failed"
return "Notification request failed"
def gps_loss_while_moving(speed, fix_age, threshold, now=None):
global _moving_started_at
now = time.monotonic() if now is None else now
if speed is None or speed <= 1:
_moving_started_at = None
return False
if _moving_started_at is None:
_moving_started_at = now
last_fix_at = None if fix_age is None else now - fix_age
loss_started_at = (
_moving_started_at
if last_fix_at is None
else max(_moving_started_at, last_fix_at)
)
return now - loss_started_at >= threshold
def _dashcam_usage(now=None):
global _last_dashcam_check_at, _last_dashcam_usage
now = time.monotonic() if now is None else now
if (
_last_dashcam_check_at is None
or now - _last_dashcam_check_at >= DASHCAM_CHECK_INTERVAL
):
_last_dashcam_usage = app_dash.dashcam_storage_usage()
_last_dashcam_check_at = now
return _last_dashcam_usage
def _power_condition(throttling, now=None):
now = time.time() if now is None else now
try:
event_until = float(
app_sql.get_setting("alert_power_event_until", "0")
)
except (TypeError, ValueError) as error:
raise ValueError("alert_power_event_until is invalid") from error
if throttling is None:
return {
"active": now < event_until,
"current": [],
"occurred": [],
"newly_occurred": [],
"recent_event": now < event_until,
"raw": None,
}
current = set(throttling["current"] if throttling else [])
occurred = set(throttling["occurred"] if throttling else [])
raw_seen = app_sql.get_setting("alert_throttling_seen")
if raw_seen is None:
seen = occurred
app_sql.set_setting(
"alert_throttling_seen", json.dumps(sorted(occurred))
)
else:
try:
seen = set(json.loads(raw_seen))
except (json.JSONDecodeError, TypeError) as error:
raise ValueError("alert_throttling_seen is invalid") from error
newly_occurred = occurred - seen
if raw_seen is not None and occurred != seen:
app_sql.set_setting(
"alert_throttling_seen", json.dumps(sorted(occurred))
)
if newly_occurred:
event_until = now + POWER_EVENT_GRACE_SECONDS
app_sql.set_setting(
"alert_power_event_until",
str(event_until),
)
return {
"active": bool(current) or now < event_until,
"current": sorted(current),
"occurred": sorted(occurred),
"newly_occurred": sorted(newly_occurred),
"recent_event": now < event_until,
"raw": throttling["raw"] if throttling else None,
}
def collect_conditions(config):
health = app_health.collect_health()
videos = app_video.list_local_videos()
pending_videos = sum(1 for video in videos["files"] if not video["uploaded"])
pending_tracking = app_sql.count_unsent_tracking_rows()
speed = app_obd.get_fresh_speed()
fix_age = app_gps.get_fix_age()
temperature = health["temperature_c"]
disk = health["disk"]
throttling = health["throttling"]
dashcam_disk = _dashcam_usage()
power = _power_condition(throttling)
mil_on = app_obd.get_fresh_mil()
dtc_codes = [
item["code"] for item in app_obd.get_fresh_dtc_codes()
]
moving_without_fix = gps_loss_while_moving(
speed,
fix_age,
config["alert_gps_loss_seconds"],
)
return {
"mil": {
"active": mil_on is True,
"value": {
"mil_on": mil_on,
"dtc_codes": dtc_codes,
},
"message": (
"The vehicle MIL is on."
+ (f" Codes: {', '.join(dtc_codes)}" if dtc_codes else "")
),
"priority": "high",
"tags": ["warning", "car"],
},
"disk": {
"active": (
(
disk is not None
and disk["used_percent"] >= config["alert_disk_percent"]
)
or (
dashcam_disk["status"] == "ok"
and dashcam_disk["used_percent"]
>= config["alert_disk_percent"]
)
),
"value": {
"pi": disk,
"dashcam": dashcam_disk,
},
"message": (
(
f"Pi storage is {disk['used_percent']:.1f}% full. "
if disk
else "Pi disk usage is unavailable. "
)
+ (
f"Dashcam storage is {dashcam_disk['used_percent']:.1f}% full."
if dashcam_disk["status"] == "ok"
else f"Dashcam storage status: {dashcam_disk['status']}."
)
),
"priority": "high",
"tags": ["warning", "floppy_disk"],
},
"upload_backlog": {
"active": (
pending_tracking >= config["alert_tracking_backlog"]
or pending_videos >= config["alert_video_backlog"]
),
"value": {
"pending_tracking_rows": pending_tracking,
"pending_videos": pending_videos,
},
"message": (
f"{pending_tracking} tracking rows and {pending_videos} "
"dashcam videos are waiting to upload."
),
"priority": "default",
"tags": ["warning", "outbox_tray"],
},
"temperature": {
"active": (
temperature is not None
and temperature >= config["alert_temperature_c"]
),
"value": {"temperature_c": temperature},
"message": (
f"Pi CPU temperature is {temperature:.1f} °C."
if temperature is not None
else "Pi temperature is unavailable."
),
"priority": "high",
"tags": ["warning", "thermometer"],
},
"power": {
"active": power["active"],
"value": power,
"message": (
"Pi firmware reports: "
+ ", ".join(power["current"]).replace("_", " ")
+ "."
if power["current"]
else (
"Pi firmware recorded a recent power/throttling event: "
+ ", ".join(
power["newly_occurred"] or power["occurred"]
).replace("_", " ")
+ "."
if power["recent_event"]
else "Pi power and throttling state is normal."
)
),
"priority": "high",
"tags": ["warning", "electric_plug"],
},
"gps_loss": {
"active": moving_without_fix,
"value": {
"speed_kmh": speed,
"fix_age_seconds": fix_age,
},
"message": (
f"The vehicle is moving at {speed:.1f} km/h but the GPS fix "
+ (
"has never been acquired."
if fix_age is None
else f"is {fix_age:.0f} seconds old."
)
if speed is not None
else "GPS state unavailable."
),
"priority": "high",
"tags": ["warning", "round_pushpin"],
},
}
def _blank_state():
return {
"active": False,
"active_since": None,
"last_attempt_at": None,
"last_sent_at": None,
"recovery_pending": False,
"last_recovery_at": None,
"last_error": None,
"last_value": None,
}
def process_conditions(conditions, config, now=None, sender=send_ntfy):
now = time.time() if now is None else now
states = app_sql.get_alert_states()
configured = (
config["enabled"]
and bool(config["ntfy_url"])
and not config["errors"]
)
for key, condition in conditions.items():
previous = states.get(key, _blank_state())
state = previous.copy()
was_active = previous["active"]
state["active"] = condition["active"]
state["last_value"] = json.dumps(condition["value"], sort_keys=True)
if condition["active"]:
if not was_active:
state["active_since"] = now
state["recovery_pending"] = False
due = (
not was_active
or state["last_sent_at"] is None
or now - state["last_sent_at"] >= config["alert_repeat_seconds"]
)
retry_ready = (
not was_active
or state["last_attempt_at"] is None
or now - state["last_attempt_at"] >= (
CHECK_INTERVAL
if key == "power"
and condition["value"].get("recent_event")
else RETRY_INTERVAL
)
)
if configured and due and retry_ready:
state["last_attempt_at"] = now
try:
sender(
config["ntfy_url"],
config["ntfy_token"],
ALERT_TITLES[key],
condition["message"],
condition["priority"],
condition["tags"],
)
except requests.RequestException as error:
state["last_error"] = _transport_error(error)
else:
state["last_sent_at"] = now
state["last_error"] = None
else:
state["active_since"] = None
if was_active and state["last_sent_at"] is not None:
state["recovery_pending"] = True
retry_ready = (
was_active
or state["last_attempt_at"] is None
or now - state["last_attempt_at"] >= RETRY_INTERVAL
)
if configured and state["recovery_pending"] and retry_ready:
state["last_attempt_at"] = now
try:
sender(
config["ntfy_url"],
config["ntfy_token"],
f"Resolved: {ALERT_TITLES[key]}",
f"{ALERT_TITLES[key]} is no longer active.",
"default",
["white_check_mark"],
)
except requests.RequestException as error:
state["last_error"] = _transport_error(error)
else:
state["recovery_pending"] = False
state["last_recovery_at"] = now
state["last_error"] = None
app_sql.save_alert_state(key, state)
def check_alerts():
global _last_run_at, _last_run_error, _last_conditions
config = get_config()
conditions = collect_conditions(config)
process_conditions(conditions, config)
_last_conditions = conditions
_last_run_at = time.time()
_last_run_error = None
return conditions
def _loop():
global _last_run_error
while True:
try:
check_alerts()
except (
OSError,
ValueError,
RuntimeError,
sqlite3.Error,
requests.RequestException,
) as error:
_last_run_error = str(error)
print("Alert check failed:", error)
time.sleep(CHECK_INTERVAL)
def start_alerting():
global _thread
if _thread is None or not _thread.is_alive():
_thread = threading.Thread(target=_loop, name="alerts", daemon=True)
_thread.start()
print(f"Alert thread started, alive: {_thread.is_alive()}")
return _thread
def get_status():
config = get_config()
states = app_sql.get_alert_states()
thread_alive = _thread is not None and _thread.is_alive()
return {
"enabled": config["enabled"],
"configured": bool(config["ntfy_url"]) and not config["errors"],
"config_errors": config["errors"],
"thread_alive": thread_alive,
"last_run_at": _last_run_at,
"last_run_error": (
_last_run_error
or ("Alert thread is not running." if not thread_alive else None)
),
"conditions": _last_conditions,
"states": states,
}
+41
View File
@@ -274,6 +274,47 @@ def _mounted_source(partition, mount_point, read_only):
return mount_point, True, False
def dashcam_storage_usage(mount_point=DEFAULT_MOUNT_POINT):
with _device_lock:
partition = find_dashcam_partition()
if partition is None:
return {"status": "no_device"}
source, mounted_here, _remounted = _mounted_source(
partition, mount_point, read_only=True
)
if source is None:
return {
"status": "mount_failed",
"path": partition["path"],
}
try:
usage = shutil.disk_usage(source)
used_percent = round(
usage.used / (usage.used + usage.free) * 100, 1
)
result = {
"status": "ok",
"path": partition["path"],
"total_bytes": usage.total,
"used_bytes": usage.used,
"free_bytes": usage.free,
"used_percent": used_percent,
}
except OSError as error:
result = {
"status": "read_failed",
"path": partition["path"],
"error": str(error),
}
finally:
if mounted_here and not unmount_device(mount_point):
result = {
"status": "unmount_failed",
"path": partition["path"],
}
return result
def list_dashcam_files(mount_point=DEFAULT_MOUNT_POINT):
"""List video files on the detected dashcam without copying them."""
with _device_lock:
+7 -1
View File
@@ -27,6 +27,7 @@ PWRKEY_HOLD_SECONDS = 4 # matches Waveshare's reference script
LocationX = None
LocationY = None
last_fix_at = None
_serial_lock = Lock()
_serial = None
@@ -148,7 +149,7 @@ def _parse_cgnsinf(resp):
def _poll_loop():
global LocationX, LocationY
global LocationX, LocationY, last_fix_at
try:
with _serial_lock:
if _ensure_module_powered():
@@ -168,11 +169,16 @@ def _poll_loop():
if fix:
LocationX = fix["lat"]
LocationY = fix["lon"]
last_fix_at = time.monotonic()
socketio.emit("gps_location", [LocationX, LocationY])
time.sleep(POLL_INTERVAL)
def get_fix_age():
return None if last_fix_at is None else time.monotonic() - last_fix_at
def start_gps():
"""Start the background thread that continuously polls the SIM868 for a GNSS fix."""
global _gps_thread
+17 -1
View File
@@ -22,6 +22,8 @@ throttle_position = None
mil_on = None
dtc_codes = []
odometer = None
mil_updated_at = None
dtc_updated_at = None
class OBD:
@@ -251,6 +253,18 @@ def obd_connection():
_emit_disconnected()
def get_fresh_mil(max_age=SENSOR_POLL_INTERVAL * 3):
if mil_updated_at is None or time.monotonic() - mil_updated_at > max_age:
return None
return mil_on
def get_fresh_dtc_codes(max_age=SENSOR_POLL_INTERVAL * 3):
if dtc_updated_at is None or time.monotonic() - dtc_updated_at > max_age:
return []
return list(dtc_codes)
def obd_speed():
global speed, speed_updated_at
with obd_speed_lock:
@@ -298,7 +312,7 @@ def _quantity_value(command, unit=None):
def _poll_diagnostics():
global coolant_temp, fuel_level, throttle_position
global mil_on, dtc_codes, odometer
global mil_on, dtc_codes, odometer, mil_updated_at, dtc_updated_at
coolant_temp = _quantity_value(obd.commands.COOLANT_TEMP, "degC")
fuel_level = _quantity_value(obd.commands.FUEL_LEVEL, "percent")
@@ -307,6 +321,7 @@ def _poll_diagnostics():
status_response = _query(obd.commands.STATUS)
if status_response is not None and not status_response.is_null():
mil_on = bool(status_response.value.MIL)
mil_updated_at = time.monotonic()
app_events.socketio.emit(
"obd_mil",
{"on": mil_on, "dtc_count": status_response.value.DTC_count},
@@ -318,6 +333,7 @@ def _poll_diagnostics():
{"code": code, "description": description}
for code, description in dtc_response.value
]
dtc_updated_at = time.monotonic()
app_events.socketio.emit("obd_dtc", dtc_codes)
odometer_response = _query(ODOMETER_COMMAND, force=True)
+73
View File
@@ -99,6 +99,18 @@ SQL_CREATE_TRIP_BUILD_STATE_TABLE = """CREATE TABLE IF NOT EXISTS trip_build_sta
BUILD_VERSION integer NOT NULL DEFAULT 1
);"""
SQL_CREATE_ALERT_STATE_TABLE = """CREATE TABLE IF NOT EXISTS alert_state (
ALERT_KEY text PRIMARY KEY,
ACTIVE integer NOT NULL DEFAULT 0,
ACTIVE_SINCE real,
LAST_ATTEMPT_AT real,
LAST_SENT_AT real,
RECOVERY_PENDING integer NOT NULL DEFAULT 0,
LAST_RECOVERY_AT real,
LAST_ERROR text,
LAST_VALUE text
);"""
def create_conn(db_file):
"""Open a SQLite connection to db_file, creating its parent folder if needed.
@@ -143,6 +155,7 @@ def get_conn():
create_table(_conn, SQL_CREATE_DASHCAM_HASH_CACHE_TABLE)
create_table(_conn, SQL_CREATE_TRIPS_TABLE)
create_table(_conn, SQL_CREATE_TRIP_BUILD_STATE_TABLE)
create_table(_conn, SQL_CREATE_ALERT_STATE_TABLE)
_migrate_tracking_table(_conn)
_migrate_trips_table(_conn)
_migrate_trip_build_state_table(_conn)
@@ -392,6 +405,66 @@ def set_setting(key, value):
conn.commit()
def get_alert_states():
conn = get_conn()
if conn is None:
return {}
with _conn_lock:
rows = conn.execute(
"""SELECT ALERT_KEY, ACTIVE, ACTIVE_SINCE, LAST_ATTEMPT_AT,
LAST_SENT_AT, RECOVERY_PENDING, LAST_RECOVERY_AT,
LAST_ERROR, LAST_VALUE
FROM alert_state"""
).fetchall()
return {
row[0]: {
"active": bool(row[1]),
"active_since": row[2],
"last_attempt_at": row[3],
"last_sent_at": row[4],
"recovery_pending": bool(row[5]),
"last_recovery_at": row[6],
"last_error": row[7],
"last_value": row[8],
}
for row in rows
}
def save_alert_state(alert_key, state):
conn = get_conn()
if conn is None:
raise RuntimeError("SQLite connection unavailable")
with _conn_lock:
conn.execute(
"""INSERT INTO alert_state (
ALERT_KEY, ACTIVE, ACTIVE_SINCE, LAST_ATTEMPT_AT,
LAST_SENT_AT, RECOVERY_PENDING, LAST_RECOVERY_AT,
LAST_ERROR, LAST_VALUE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(ALERT_KEY) DO UPDATE SET
ACTIVE = excluded.ACTIVE,
ACTIVE_SINCE = excluded.ACTIVE_SINCE,
LAST_ATTEMPT_AT = excluded.LAST_ATTEMPT_AT,
LAST_SENT_AT = excluded.LAST_SENT_AT,
RECOVERY_PENDING = excluded.RECOVERY_PENDING,
LAST_RECOVERY_AT = excluded.LAST_RECOVERY_AT,
LAST_ERROR = excluded.LAST_ERROR,
LAST_VALUE = excluded.LAST_VALUE""",
(
alert_key,
1 if state["active"] else 0,
state["active_since"],
state["last_attempt_at"],
state["last_sent_at"],
1 if state["recovery_pending"] else 0,
state["last_recovery_at"],
state["last_error"],
state["last_value"],
),
)
conn.commit()
def get_unsent_tracking_rows(limit=100):
"""Return up to `limit` tracking rows not yet uploaded to the remote server,
oldest first, as a list of dicts ready to be sent as JSON."""