feat: add scripts/diagnose.sh — system diagnostic & recovery tool
Read-only health check (no args) or auto-fix mode (--fix):
- Service: active state, crash loop detection, recent log
- Hardware: OBD USB adapter (/dev/ttyUSBx), SIM868 UART (/dev/serial0),
dashcam USB mount, permissions
- Network: hotspot IP, web UI reachability, internet, upload server
- Storage: disk usage, SQLite integrity check, row counts, upload backlog,
log file size
- OBD status: parsed from log (last connect/disconnect/speed/errors)
- Python env: venv present, key packages importable
- Pi health: throttling flags decoded, CPU temp, free memory
- Manual recovery command cheatsheet printed at the end
Exit code = number of critical issues found (0 = healthy).
Co-authored-by: Copilot <[email protected]>
This commit is contained in:
Mathieu Moran
Executable
+384
@@ -0,0 +1,384 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# yaris-obd diagnostic & recovery script
|
||||
# Run ON THE PI (over SSH or directly):
|
||||
# bash ~/app/yaris-obd/scripts/diagnose.sh [--fix]
|
||||
#
|
||||
# Without --fix : read-only health check, prints a report, exits 0 if healthy.
|
||||
# With --fix : attempts to auto-fix every issue it finds. Safe to run while
|
||||
# the service is running — restarts are done with systemctl.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
LOG_FILE="$REPO_DIR/yaris-obd.log"
|
||||
SERVICE="yaris-obd.service"
|
||||
DB_FILE="$REPO_DIR/$(ls "$REPO_DIR"/*.db 2>/dev/null | head -1 | xargs basename 2>/dev/null || true)"
|
||||
FIX=false
|
||||
ISSUES=0
|
||||
WARNINGS=0
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
red() { printf '\033[0;31m%s\033[0m\n' "$*"; }
|
||||
yellow() { printf '\033[0;33m%s\033[0m\n' "$*"; }
|
||||
green() { printf '\033[0;32m%s\033[0m\n' "$*"; }
|
||||
bold() { printf '\033[1m%s\033[0m\n' "$*"; }
|
||||
info() { printf ' %s\n' "$*"; }
|
||||
|
||||
pass() { green " ✓ $*"; }
|
||||
warn() { yellow " ⚠ $*"; WARNINGS=$((WARNINGS+1)); }
|
||||
fail() { red " ✗ $*"; ISSUES=$((ISSUES+1)); }
|
||||
|
||||
section() { echo; bold "── $* ──"; }
|
||||
|
||||
fix_action() {
|
||||
# fix_action "description" "command..."
|
||||
local desc="$1"; shift
|
||||
if $FIX; then
|
||||
printf ' → Fixing: %s\n' "$desc"
|
||||
if eval "$@"; then
|
||||
green " Done."
|
||||
else
|
||||
red " Failed — manual intervention needed."
|
||||
fi
|
||||
else
|
||||
yellow " ↳ Run with --fix to attempt: $desc"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── argument parsing ──────────────────────────────────────────────────────────
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--fix|-f) FIX=true ;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [--fix]"
|
||||
echo " --fix Attempt to auto-fix found issues (restarts service as needed)"
|
||||
exit 0 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# =============================================================================
|
||||
echo
|
||||
bold "=========================================="
|
||||
bold " Yaris OBD — Diagnostic Report"
|
||||
bold " $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
bold "=========================================="
|
||||
|
||||
# ── 1. Service ────────────────────────────────────────────────────────────────
|
||||
section "1. Service"
|
||||
|
||||
if systemctl is-active --quiet "$SERVICE"; then
|
||||
pass "Service is active (running)"
|
||||
UPTIME=$(systemctl show "$SERVICE" --property=ActiveEnterTimestamp --value 2>/dev/null || echo "unknown")
|
||||
info "Running since: $UPTIME"
|
||||
else
|
||||
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null || true)
|
||||
fail "Service is NOT running (state: $STATE)"
|
||||
LAST_ERR=$(journalctl -u "$SERVICE" -n 5 --no-pager -q 2>/dev/null || tail -5 "$LOG_FILE" 2>/dev/null || echo "no logs")
|
||||
info "Last log lines:"
|
||||
echo "$LAST_ERR" | sed 's/^/ /'
|
||||
fix_action "restart $SERVICE" "sudo systemctl restart '$SERVICE'"
|
||||
fi
|
||||
|
||||
# Check for recent crash/restart loop
|
||||
RESTART_COUNT=$(systemctl show "$SERVICE" --property=NRestarts --value 2>/dev/null || echo "0")
|
||||
if [ "${RESTART_COUNT:-0}" -gt 3 ]; then
|
||||
warn "Service has restarted $RESTART_COUNT times — possible crash loop"
|
||||
info "Recent log (last 20 lines):"
|
||||
(journalctl -u "$SERVICE" -n 20 --no-pager -q 2>/dev/null || tail -20 "$LOG_FILE" 2>/dev/null) | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# ── 2. USB / Serial devices ───────────────────────────────────────────────────
|
||||
section "2. Hardware Devices"
|
||||
|
||||
# OBD adapter
|
||||
if ls /dev/ttyUSB* &>/dev/null; then
|
||||
OBD_PORT=$(ls /dev/ttyUSB* | head -1)
|
||||
pass "OBD USB adapter detected at $OBD_PORT"
|
||||
# Check permissions
|
||||
if [ -r "$OBD_PORT" ] && [ -w "$OBD_PORT" ]; then
|
||||
pass " $OBD_PORT is readable/writable by current user"
|
||||
else
|
||||
fail " $OBD_PORT is NOT accessible — user '$(whoami)' not in 'dialout' group?"
|
||||
fix_action "add $(whoami) to dialout group" "sudo usermod -aG dialout '$(whoami)'"
|
||||
fi
|
||||
else
|
||||
warn "No /dev/ttyUSB* found — OBD adapter not detected"
|
||||
info " Is the ELM327 USB adapter plugged in?"
|
||||
info " Check: lsusb | grep -i 'prolific\|pl2303\|elm'"
|
||||
lsusb 2>/dev/null | grep -iE 'prolific|pl2303|elm|ftdi|ch340' | sed 's/^/ /' || true
|
||||
fi
|
||||
|
||||
# GPS / SIM868 UART
|
||||
if [ -e /dev/serial0 ]; then
|
||||
pass "SIM868 UART present at /dev/serial0"
|
||||
else
|
||||
fail "/dev/serial0 not found — UART may not be enabled"
|
||||
info " Check /boot/firmware/config.txt for enable_uart=1 and dtoverlay=disable-bt"
|
||||
fi
|
||||
|
||||
# Dashcam (USB mass storage)
|
||||
DASHCAM_MOUNTS=$(lsblk -rno NAME,MOUNTPOINT 2>/dev/null | awk '$2 ~ /dashcam|media|mnt/' || true)
|
||||
if [ -n "$DASHCAM_MOUNTS" ]; then
|
||||
pass "USB storage device mounted:"
|
||||
echo "$DASHCAM_MOUNTS" | sed 's/^/ /'
|
||||
else
|
||||
info "No dashcam USB device currently mounted (normal when not in import mode)"
|
||||
fi
|
||||
|
||||
# ── 3. Network ────────────────────────────────────────────────────────────────
|
||||
section "3. Network & Hotspot"
|
||||
|
||||
# Hotspot (wlan0 or wlan1)
|
||||
HOTSPOT_IFACE=$(ip -o link show | awk '/wlan/{print $2}' | tr -d ':' | head -1)
|
||||
if [ -n "$HOTSPOT_IFACE" ]; then
|
||||
HOTSPOT_IP=$(ip -4 addr show "$HOTSPOT_IFACE" 2>/dev/null | awk '/inet /{print $2}' | head -1)
|
||||
if [ -n "$HOTSPOT_IP" ]; then
|
||||
pass "WiFi interface $HOTSPOT_IFACE has IP $HOTSPOT_IP"
|
||||
else
|
||||
warn "WiFi interface $HOTSPOT_IFACE has no IP address"
|
||||
fi
|
||||
else
|
||||
warn "No WiFi interface detected"
|
||||
fi
|
||||
|
||||
# Is the web UI reachable locally?
|
||||
if curl -sf --max-time 3 http://localhost:5000/ &>/dev/null; then
|
||||
pass "Web UI reachable at http://localhost:5000/"
|
||||
elif curl -sf --max-time 3 http://localhost/ &>/dev/null; then
|
||||
pass "Web UI reachable at http://localhost/"
|
||||
else
|
||||
fail "Web UI not responding on localhost — service may be down or port changed"
|
||||
fi
|
||||
|
||||
# Internet/upload connectivity
|
||||
if curl -sf --max-time 5 https://cloudflare.com/ &>/dev/null; then
|
||||
pass "Internet reachable"
|
||||
# Check server upload endpoint if configured
|
||||
UPLOAD_CREDS="$REPO_DIR/db/upload_creds.env"
|
||||
if [ -f "$UPLOAD_CREDS" ]; then
|
||||
SERVER_URL=$(grep -E '^SERVER_URL' "$UPLOAD_CREDS" 2>/dev/null | cut -d= -f2- | tr -d '"' | tr -d "'" | xargs || true)
|
||||
if [ -n "$SERVER_URL" ]; then
|
||||
if curl -sf --max-time 5 "${SERVER_URL}/health" &>/dev/null || \
|
||||
curl -sf --max-time 5 "${SERVER_URL}/" &>/dev/null; then
|
||||
pass "Upload server reachable at $SERVER_URL"
|
||||
else
|
||||
warn "Upload server unreachable at $SERVER_URL (may be expected off-car)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
warn "No internet connectivity (expected when not on hotspot/WiFi)"
|
||||
fi
|
||||
|
||||
# ── 4. Storage & Database ─────────────────────────────────────────────────────
|
||||
section "4. Storage & Database"
|
||||
|
||||
# Disk space
|
||||
DISK_USE=$(df -h "$REPO_DIR" | awk 'NR==2{print $5}' | tr -d '%')
|
||||
DISK_AVAIL=$(df -h "$REPO_DIR" | awk 'NR==2{print $4}')
|
||||
if [ "${DISK_USE:-0}" -lt 85 ]; then
|
||||
pass "Disk usage: ${DISK_USE}% used, $DISK_AVAIL available"
|
||||
elif [ "${DISK_USE:-0}" -lt 95 ]; then
|
||||
warn "Disk usage high: ${DISK_USE}% used, $DISK_AVAIL available"
|
||||
else
|
||||
fail "Disk nearly full: ${DISK_USE}% used — service may fail to write"
|
||||
info " Largest items in repo dir:"
|
||||
du -sh "$REPO_DIR"/*/ 2>/dev/null | sort -rh | head -5 | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# SQLite DB
|
||||
DB_PATH=$(find "$REPO_DIR" -maxdepth 1 -name "*.db" 2>/dev/null | head -1)
|
||||
if [ -n "$DB_PATH" ]; then
|
||||
DB_SIZE=$(du -sh "$DB_PATH" | cut -f1)
|
||||
pass "Database found: $(basename "$DB_PATH") ($DB_SIZE)"
|
||||
# Integrity check
|
||||
INTEGRITY=$(sqlite3 "$DB_PATH" "PRAGMA integrity_check;" 2>/dev/null || echo "error")
|
||||
if [ "$INTEGRITY" = "ok" ]; then
|
||||
pass "Database integrity: OK"
|
||||
# Row counts
|
||||
TRACKING_ROWS=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM tracking;" 2>/dev/null || echo "?")
|
||||
UNSENT_ROWS=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM tracking WHERE sent=0;" 2>/dev/null || echo "?")
|
||||
TRIP_ROWS=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM trips;" 2>/dev/null || echo "?")
|
||||
info " tracking rows: $TRACKING_ROWS (unsent: $UNSENT_ROWS)"
|
||||
info " trip summaries: $TRIP_ROWS"
|
||||
if [ "${UNSENT_ROWS:-0}" -gt 5000 ]; then
|
||||
warn "Large upload backlog: $UNSENT_ROWS unsent tracking rows"
|
||||
fi
|
||||
else
|
||||
fail "Database integrity check FAILED: $INTEGRITY"
|
||||
info " !! Do NOT delete the database without backing it up first !!"
|
||||
info " Backup: cp '$DB_PATH' '${DB_PATH}.bak.$(date +%Y%m%d_%H%M%S)'"
|
||||
fix_action "backup and attempt WAL checkpoint" \
|
||||
"cp '$DB_PATH' '${DB_PATH}.bak.$(date +%Y%m%d_%H%M%S)' && sqlite3 '$DB_PATH' 'PRAGMA wal_checkpoint(FULL);'"
|
||||
fi
|
||||
else
|
||||
warn "No .db file found in $REPO_DIR — DB not yet created (normal on first run)"
|
||||
fi
|
||||
|
||||
# Log file size
|
||||
if [ -f "$LOG_FILE" ]; then
|
||||
LOG_SIZE=$(du -sh "$LOG_FILE" | cut -f1)
|
||||
LOG_BYTES=$(stat -c%s "$LOG_FILE" 2>/dev/null || echo 0)
|
||||
if [ "${LOG_BYTES:-0}" -gt 104857600 ]; then # 100MB
|
||||
warn "Log file is large: $LOG_SIZE at $LOG_FILE"
|
||||
fix_action "rotate log (keep last 10k lines)" \
|
||||
"tail -n 10000 '$LOG_FILE' > '$LOG_FILE.tmp' && mv '$LOG_FILE.tmp' '$LOG_FILE'"
|
||||
else
|
||||
pass "Log file: $LOG_SIZE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 5. OBD Status (from log) ──────────────────────────────────────────────────
|
||||
section "5. OBD Status (from recent log)"
|
||||
|
||||
if [ -f "$LOG_FILE" ]; then
|
||||
# Look for recent OBD connection state
|
||||
LAST_CONNECTED=$(grep -a "CAR_CONNECTED\|Car Connected\|Connected to vehicle" "$LOG_FILE" 2>/dev/null | tail -1 || true)
|
||||
LAST_ELM=$(grep -a "ELM327\|elm327\|handshake" "$LOG_FILE" 2>/dev/null | tail -1 || true)
|
||||
LAST_DISCONNECT=$(grep -a "disconnected\|Cannot load commands\|ECU did not respond" "$LOG_FILE" 2>/dev/null | tail -1 || true)
|
||||
LAST_SPEED=$(grep -a "^Speed:" "$LOG_FILE" 2>/dev/null | tail -1 || true)
|
||||
|
||||
if [ -n "$LAST_CONNECTED" ]; then
|
||||
pass "Last successful OBD connection found in log:"
|
||||
info " $LAST_CONNECTED"
|
||||
else
|
||||
info "No successful CAR_CONNECTED log entry found"
|
||||
fi
|
||||
if [ -n "$LAST_DISCONNECT" ]; then
|
||||
info "Last disconnect/error: $LAST_DISCONNECT"
|
||||
fi
|
||||
if [ -n "$LAST_SPEED" ]; then
|
||||
pass "Last logged speed: $LAST_SPEED"
|
||||
fi
|
||||
if [ -n "$LAST_ELM" ]; then
|
||||
info "Last ELM327 event: $LAST_ELM"
|
||||
fi
|
||||
|
||||
# Errors in last 100 lines
|
||||
RECENT_ERRORS=$(tail -100 "$LOG_FILE" 2>/dev/null | grep -iE "error|traceback|exception|crash|failed|fatal" | grep -v "^::" || true)
|
||||
if [ -n "$RECENT_ERRORS" ]; then
|
||||
warn "Recent errors in log:"
|
||||
echo "$RECENT_ERRORS" | tail -10 | sed 's/^/ /'
|
||||
else
|
||||
pass "No errors in recent 100 log lines"
|
||||
fi
|
||||
else
|
||||
info "Log file not found at $LOG_FILE"
|
||||
fi
|
||||
|
||||
# ── 6. Python environment ─────────────────────────────────────────────────────
|
||||
section "6. Python Environment"
|
||||
|
||||
VENV="$REPO_DIR/venv"
|
||||
if [ -d "$VENV" ]; then
|
||||
PYTHON="$VENV/bin/python"
|
||||
PY_VER=$("$PYTHON" --version 2>&1)
|
||||
pass "venv found: $PY_VER"
|
||||
|
||||
# Check key packages
|
||||
for pkg in flask flask_socketio obd; do
|
||||
if "$PYTHON" -c "import $pkg" 2>/dev/null; then
|
||||
VER=$("$PYTHON" -c "import importlib.metadata; print(importlib.metadata.version('$pkg'))" 2>/dev/null || echo "installed")
|
||||
pass " $pkg ($VER)"
|
||||
else
|
||||
fail " $pkg NOT importable — reinstall requirements?"
|
||||
fix_action "reinstall requirements.txt" \
|
||||
"'$VENV/bin/pip' install -r '$REPO_DIR/requirements.txt'"
|
||||
fi
|
||||
done
|
||||
else
|
||||
fail "venv not found at $VENV"
|
||||
fix_action "create venv and install requirements" \
|
||||
"python3 -m venv '$VENV' && '$VENV/bin/pip' install -r '$REPO_DIR/requirements.txt'"
|
||||
fi
|
||||
|
||||
# ── 7. Pi health ──────────────────────────────────────────────────────────────
|
||||
section "7. Pi Hardware Health"
|
||||
|
||||
# Throttling
|
||||
THROTTLED=$(vcgencmd get_throttled 2>/dev/null | cut -d= -f2 || echo "unavailable")
|
||||
if [ "$THROTTLED" = "0x0" ]; then
|
||||
pass "No throttling or undervoltage (throttled=0x0)"
|
||||
elif [ "$THROTTLED" = "unavailable" ]; then
|
||||
info "vcgencmd not available (not a Pi?)"
|
||||
else
|
||||
# Decode the flags
|
||||
THROT_HEX="${THROTTLED#0x}"
|
||||
THROT_DEC=$((16#${THROT_HEX}))
|
||||
FLAGS=""
|
||||
(( THROT_DEC & 0x1 )) && FLAGS="$FLAGS • Currently under-voltage detected"
|
||||
(( THROT_DEC & 0x2 )) && FLAGS="$FLAGS • Currently arm frequency capped"
|
||||
(( THROT_DEC & 0x4 )) && FLAGS="$FLAGS • Currently throttled"
|
||||
(( THROT_DEC & 0x10000 )) && FLAGS="$FLAGS • Under-voltage HAS occurred since boot"
|
||||
(( THROT_DEC & 0x20000 )) && FLAGS="$FLAGS • Frequency cap HAS occurred since boot"
|
||||
(( THROT_DEC & 0x40000 )) && FLAGS="$FLAGS • Throttling HAS occurred since boot"
|
||||
|
||||
# Current bits (bits 0-3) are critical; historical bits (16-19) are warnings
|
||||
if (( THROT_DEC & 0xF )); then
|
||||
fail "Pi is CURRENTLY throttled/undervoltaged ($THROTTLED)"
|
||||
else
|
||||
warn "Throttling/undervoltage has occurred since boot ($THROTTLED)"
|
||||
fi
|
||||
[ -n "$FLAGS" ] && echo "$FLAGS" | sed 's/^/ /'
|
||||
info " Use a 3A+ USB-C power supply or a powered USB hub for OBD/dashcam"
|
||||
fi
|
||||
|
||||
# Temperature
|
||||
TEMP=$(vcgencmd measure_temp 2>/dev/null | cut -d= -f2 | cut -d\' -f1 || echo "unavailable")
|
||||
if [ "$TEMP" != "unavailable" ]; then
|
||||
TEMP_INT=${TEMP%.*}
|
||||
if [ "${TEMP_INT:-0}" -lt 70 ]; then
|
||||
pass "CPU temperature: ${TEMP}°C"
|
||||
elif [ "${TEMP_INT:-0}" -lt 80 ]; then
|
||||
warn "CPU temperature elevated: ${TEMP}°C"
|
||||
else
|
||||
fail "CPU temperature critical: ${TEMP}°C — check ventilation"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Memory
|
||||
FREE_MB=$(free -m | awk '/^Mem:/{print $7}')
|
||||
if [ "${FREE_MB:-0}" -gt 100 ]; then
|
||||
pass "Free memory: ${FREE_MB}MB"
|
||||
elif [ "${FREE_MB:-0}" -gt 50 ]; then
|
||||
warn "Low free memory: ${FREE_MB}MB"
|
||||
else
|
||||
fail "Very low free memory: ${FREE_MB}MB"
|
||||
fi
|
||||
|
||||
# ── 8. Quick fixes menu ───────────────────────────────────────────────────────
|
||||
section "8. Manual Recovery Commands"
|
||||
|
||||
info "Restart service: sudo systemctl restart $SERVICE"
|
||||
info "View live logs: tail -f $LOG_FILE"
|
||||
info "Journalctl logs: sudo journalctl -u $SERVICE -f"
|
||||
info "Force OBD_PROTOCOL=6: sudo systemctl edit $SERVICE"
|
||||
info " (add: Environment=OBD_PROTOCOL=6 then restart)"
|
||||
info "Skip voltage check: sudo systemctl edit $SERVICE"
|
||||
info " (add: Environment=OBD_SKIP_VOLTAGE_CHECK=1 then restart)"
|
||||
info "DB backup: cp $DB_PATH ${DB_PATH}.bak.\$(date +%Y%m%d_%H%M%S)"
|
||||
info "Recompute trips: curl http://localhost:5000/api/trips/refresh"
|
||||
info "Check USB devices: lsusb && ls /dev/ttyUSB*"
|
||||
info "Check serial port: ls -la /dev/serial0"
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────
|
||||
echo
|
||||
bold "=========================================="
|
||||
if [ "$ISSUES" -eq 0 ] && [ "$WARNINGS" -eq 0 ]; then
|
||||
green " ✓ All checks passed — system looks healthy"
|
||||
elif [ "$ISSUES" -eq 0 ]; then
|
||||
yellow " ⚠ $WARNINGS warning(s) found, $ISSUES critical issue(s)"
|
||||
if ! $FIX; then
|
||||
yellow " Run with --fix to attempt auto-fixes"
|
||||
fi
|
||||
else
|
||||
red " ✗ $ISSUES critical issue(s), $WARNINGS warning(s) found"
|
||||
if ! $FIX; then
|
||||
yellow " Run with --fix to attempt auto-fixes"
|
||||
fi
|
||||
fi
|
||||
bold "=========================================="
|
||||
echo
|
||||
|
||||
exit "$ISSUES"
|
||||
Reference in New Issue
Block a user