diff --git a/scripts/diagnose.sh b/scripts/diagnose.sh new file mode 100755 index 0000000..b227db4 --- /dev/null +++ b/scripts/diagnose.sh @@ -0,0 +1,384 @@ +#!/usr/bin/env bash +# ============================================================================= +# yaris-obd diagnostic & recovery script +# Run ON THE PI (over SSH or directly): +# bash ~/app/yaris-obd/scripts/diagnose.sh [--fix] +# +# Without --fix : read-only health check, prints a report, exits 0 if healthy. +# With --fix : attempts to auto-fix every issue it finds. Safe to run while +# the service is running — restarts are done with systemctl. +# ============================================================================= +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_FILE="$REPO_DIR/yaris-obd.log" +SERVICE="yaris-obd.service" +DB_FILE="$REPO_DIR/$(ls "$REPO_DIR"/*.db 2>/dev/null | head -1 | xargs basename 2>/dev/null || true)" +FIX=false +ISSUES=0 +WARNINGS=0 + +# ── helpers ────────────────────────────────────────────────────────────────── +red() { printf '\033[0;31m%s\033[0m\n' "$*"; } +yellow() { printf '\033[0;33m%s\033[0m\n' "$*"; } +green() { printf '\033[0;32m%s\033[0m\n' "$*"; } +bold() { printf '\033[1m%s\033[0m\n' "$*"; } +info() { printf ' %s\n' "$*"; } + +pass() { green " ✓ $*"; } +warn() { yellow " ⚠ $*"; WARNINGS=$((WARNINGS+1)); } +fail() { red " ✗ $*"; ISSUES=$((ISSUES+1)); } + +section() { echo; bold "── $* ──"; } + +fix_action() { + # fix_action "description" "command..." + local desc="$1"; shift + if $FIX; then + printf ' → Fixing: %s\n' "$desc" + if eval "$@"; then + green " Done." + else + red " Failed — manual intervention needed." + fi + else + yellow " ↳ Run with --fix to attempt: $desc" + fi +} + +# ── argument parsing ────────────────────────────────────────────────────────── +for arg in "$@"; do + case "$arg" in + --fix|-f) FIX=true ;; + --help|-h) + echo "Usage: $0 [--fix]" + echo " --fix Attempt to auto-fix found issues (restarts service as needed)" + exit 0 ;; + esac +done + +# ============================================================================= +echo +bold "==========================================" +bold " Yaris OBD — Diagnostic Report" +bold " $(date '+%Y-%m-%d %H:%M:%S')" +bold "==========================================" + +# ── 1. Service ──────────────────────────────────────────────────────────────── +section "1. Service" + +if systemctl is-active --quiet "$SERVICE"; then + pass "Service is active (running)" + UPTIME=$(systemctl show "$SERVICE" --property=ActiveEnterTimestamp --value 2>/dev/null || echo "unknown") + info "Running since: $UPTIME" +else + STATE=$(systemctl is-active "$SERVICE" 2>/dev/null || true) + fail "Service is NOT running (state: $STATE)" + LAST_ERR=$(journalctl -u "$SERVICE" -n 5 --no-pager -q 2>/dev/null || tail -5 "$LOG_FILE" 2>/dev/null || echo "no logs") + info "Last log lines:" + echo "$LAST_ERR" | sed 's/^/ /' + fix_action "restart $SERVICE" "sudo systemctl restart '$SERVICE'" +fi + +# Check for recent crash/restart loop +RESTART_COUNT=$(systemctl show "$SERVICE" --property=NRestarts --value 2>/dev/null || echo "0") +if [ "${RESTART_COUNT:-0}" -gt 3 ]; then + warn "Service has restarted $RESTART_COUNT times — possible crash loop" + info "Recent log (last 20 lines):" + (journalctl -u "$SERVICE" -n 20 --no-pager -q 2>/dev/null || tail -20 "$LOG_FILE" 2>/dev/null) | sed 's/^/ /' +fi + +# ── 2. USB / Serial devices ─────────────────────────────────────────────────── +section "2. Hardware Devices" + +# OBD adapter +if ls /dev/ttyUSB* &>/dev/null; then + OBD_PORT=$(ls /dev/ttyUSB* | head -1) + pass "OBD USB adapter detected at $OBD_PORT" + # Check permissions + if [ -r "$OBD_PORT" ] && [ -w "$OBD_PORT" ]; then + pass " $OBD_PORT is readable/writable by current user" + else + fail " $OBD_PORT is NOT accessible — user '$(whoami)' not in 'dialout' group?" + fix_action "add $(whoami) to dialout group" "sudo usermod -aG dialout '$(whoami)'" + fi +else + warn "No /dev/ttyUSB* found — OBD adapter not detected" + info " Is the ELM327 USB adapter plugged in?" + info " Check: lsusb | grep -i 'prolific\|pl2303\|elm'" + lsusb 2>/dev/null | grep -iE 'prolific|pl2303|elm|ftdi|ch340' | sed 's/^/ /' || true +fi + +# GPS / SIM868 UART +if [ -e /dev/serial0 ]; then + pass "SIM868 UART present at /dev/serial0" +else + fail "/dev/serial0 not found — UART may not be enabled" + info " Check /boot/firmware/config.txt for enable_uart=1 and dtoverlay=disable-bt" +fi + +# Dashcam (USB mass storage) +DASHCAM_MOUNTS=$(lsblk -rno NAME,MOUNTPOINT 2>/dev/null | awk '$2 ~ /dashcam|media|mnt/' || true) +if [ -n "$DASHCAM_MOUNTS" ]; then + pass "USB storage device mounted:" + echo "$DASHCAM_MOUNTS" | sed 's/^/ /' +else + info "No dashcam USB device currently mounted (normal when not in import mode)" +fi + +# ── 3. Network ──────────────────────────────────────────────────────────────── +section "3. Network & Hotspot" + +# Hotspot (wlan0 or wlan1) +HOTSPOT_IFACE=$(ip -o link show | awk '/wlan/{print $2}' | tr -d ':' | head -1) +if [ -n "$HOTSPOT_IFACE" ]; then + HOTSPOT_IP=$(ip -4 addr show "$HOTSPOT_IFACE" 2>/dev/null | awk '/inet /{print $2}' | head -1) + if [ -n "$HOTSPOT_IP" ]; then + pass "WiFi interface $HOTSPOT_IFACE has IP $HOTSPOT_IP" + else + warn "WiFi interface $HOTSPOT_IFACE has no IP address" + fi +else + warn "No WiFi interface detected" +fi + +# Is the web UI reachable locally? +if curl -sf --max-time 3 http://localhost:5000/ &>/dev/null; then + pass "Web UI reachable at http://localhost:5000/" +elif curl -sf --max-time 3 http://localhost/ &>/dev/null; then + pass "Web UI reachable at http://localhost/" +else + fail "Web UI not responding on localhost — service may be down or port changed" +fi + +# Internet/upload connectivity +if curl -sf --max-time 5 https://cloudflare.com/ &>/dev/null; then + pass "Internet reachable" + # Check server upload endpoint if configured + UPLOAD_CREDS="$REPO_DIR/db/upload_creds.env" + if [ -f "$UPLOAD_CREDS" ]; then + SERVER_URL=$(grep -E '^SERVER_URL' "$UPLOAD_CREDS" 2>/dev/null | cut -d= -f2- | tr -d '"' | tr -d "'" | xargs || true) + if [ -n "$SERVER_URL" ]; then + if curl -sf --max-time 5 "${SERVER_URL}/health" &>/dev/null || \ + curl -sf --max-time 5 "${SERVER_URL}/" &>/dev/null; then + pass "Upload server reachable at $SERVER_URL" + else + warn "Upload server unreachable at $SERVER_URL (may be expected off-car)" + fi + fi + fi +else + warn "No internet connectivity (expected when not on hotspot/WiFi)" +fi + +# ── 4. Storage & Database ───────────────────────────────────────────────────── +section "4. Storage & Database" + +# Disk space +DISK_USE=$(df -h "$REPO_DIR" | awk 'NR==2{print $5}' | tr -d '%') +DISK_AVAIL=$(df -h "$REPO_DIR" | awk 'NR==2{print $4}') +if [ "${DISK_USE:-0}" -lt 85 ]; then + pass "Disk usage: ${DISK_USE}% used, $DISK_AVAIL available" +elif [ "${DISK_USE:-0}" -lt 95 ]; then + warn "Disk usage high: ${DISK_USE}% used, $DISK_AVAIL available" +else + fail "Disk nearly full: ${DISK_USE}% used — service may fail to write" + info " Largest items in repo dir:" + du -sh "$REPO_DIR"/*/ 2>/dev/null | sort -rh | head -5 | sed 's/^/ /' +fi + +# SQLite DB +DB_PATH=$(find "$REPO_DIR" -maxdepth 1 -name "*.db" 2>/dev/null | head -1) +if [ -n "$DB_PATH" ]; then + DB_SIZE=$(du -sh "$DB_PATH" | cut -f1) + pass "Database found: $(basename "$DB_PATH") ($DB_SIZE)" + # Integrity check + INTEGRITY=$(sqlite3 "$DB_PATH" "PRAGMA integrity_check;" 2>/dev/null || echo "error") + if [ "$INTEGRITY" = "ok" ]; then + pass "Database integrity: OK" + # Row counts + TRACKING_ROWS=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM tracking;" 2>/dev/null || echo "?") + UNSENT_ROWS=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM tracking WHERE sent=0;" 2>/dev/null || echo "?") + TRIP_ROWS=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM trips;" 2>/dev/null || echo "?") + info " tracking rows: $TRACKING_ROWS (unsent: $UNSENT_ROWS)" + info " trip summaries: $TRIP_ROWS" + if [ "${UNSENT_ROWS:-0}" -gt 5000 ]; then + warn "Large upload backlog: $UNSENT_ROWS unsent tracking rows" + fi + else + fail "Database integrity check FAILED: $INTEGRITY" + info " !! Do NOT delete the database without backing it up first !!" + info " Backup: cp '$DB_PATH' '${DB_PATH}.bak.$(date +%Y%m%d_%H%M%S)'" + fix_action "backup and attempt WAL checkpoint" \ + "cp '$DB_PATH' '${DB_PATH}.bak.$(date +%Y%m%d_%H%M%S)' && sqlite3 '$DB_PATH' 'PRAGMA wal_checkpoint(FULL);'" + fi +else + warn "No .db file found in $REPO_DIR — DB not yet created (normal on first run)" +fi + +# Log file size +if [ -f "$LOG_FILE" ]; then + LOG_SIZE=$(du -sh "$LOG_FILE" | cut -f1) + LOG_BYTES=$(stat -c%s "$LOG_FILE" 2>/dev/null || echo 0) + if [ "${LOG_BYTES:-0}" -gt 104857600 ]; then # 100MB + warn "Log file is large: $LOG_SIZE at $LOG_FILE" + fix_action "rotate log (keep last 10k lines)" \ + "tail -n 10000 '$LOG_FILE' > '$LOG_FILE.tmp' && mv '$LOG_FILE.tmp' '$LOG_FILE'" + else + pass "Log file: $LOG_SIZE" + fi +fi + +# ── 5. OBD Status (from log) ────────────────────────────────────────────────── +section "5. OBD Status (from recent log)" + +if [ -f "$LOG_FILE" ]; then + # Look for recent OBD connection state + LAST_CONNECTED=$(grep -a "CAR_CONNECTED\|Car Connected\|Connected to vehicle" "$LOG_FILE" 2>/dev/null | tail -1 || true) + LAST_ELM=$(grep -a "ELM327\|elm327\|handshake" "$LOG_FILE" 2>/dev/null | tail -1 || true) + LAST_DISCONNECT=$(grep -a "disconnected\|Cannot load commands\|ECU did not respond" "$LOG_FILE" 2>/dev/null | tail -1 || true) + LAST_SPEED=$(grep -a "^Speed:" "$LOG_FILE" 2>/dev/null | tail -1 || true) + + if [ -n "$LAST_CONNECTED" ]; then + pass "Last successful OBD connection found in log:" + info " $LAST_CONNECTED" + else + info "No successful CAR_CONNECTED log entry found" + fi + if [ -n "$LAST_DISCONNECT" ]; then + info "Last disconnect/error: $LAST_DISCONNECT" + fi + if [ -n "$LAST_SPEED" ]; then + pass "Last logged speed: $LAST_SPEED" + fi + if [ -n "$LAST_ELM" ]; then + info "Last ELM327 event: $LAST_ELM" + fi + + # Errors in last 100 lines + RECENT_ERRORS=$(tail -100 "$LOG_FILE" 2>/dev/null | grep -iE "error|traceback|exception|crash|failed|fatal" | grep -v "^::" || true) + if [ -n "$RECENT_ERRORS" ]; then + warn "Recent errors in log:" + echo "$RECENT_ERRORS" | tail -10 | sed 's/^/ /' + else + pass "No errors in recent 100 log lines" + fi +else + info "Log file not found at $LOG_FILE" +fi + +# ── 6. Python environment ───────────────────────────────────────────────────── +section "6. Python Environment" + +VENV="$REPO_DIR/venv" +if [ -d "$VENV" ]; then + PYTHON="$VENV/bin/python" + PY_VER=$("$PYTHON" --version 2>&1) + pass "venv found: $PY_VER" + + # Check key packages + for pkg in flask flask_socketio obd; do + if "$PYTHON" -c "import $pkg" 2>/dev/null; then + VER=$("$PYTHON" -c "import importlib.metadata; print(importlib.metadata.version('$pkg'))" 2>/dev/null || echo "installed") + pass " $pkg ($VER)" + else + fail " $pkg NOT importable — reinstall requirements?" + fix_action "reinstall requirements.txt" \ + "'$VENV/bin/pip' install -r '$REPO_DIR/requirements.txt'" + fi + done +else + fail "venv not found at $VENV" + fix_action "create venv and install requirements" \ + "python3 -m venv '$VENV' && '$VENV/bin/pip' install -r '$REPO_DIR/requirements.txt'" +fi + +# ── 7. Pi health ────────────────────────────────────────────────────────────── +section "7. Pi Hardware Health" + +# Throttling +THROTTLED=$(vcgencmd get_throttled 2>/dev/null | cut -d= -f2 || echo "unavailable") +if [ "$THROTTLED" = "0x0" ]; then + pass "No throttling or undervoltage (throttled=0x0)" +elif [ "$THROTTLED" = "unavailable" ]; then + info "vcgencmd not available (not a Pi?)" +else + # Decode the flags + THROT_HEX="${THROTTLED#0x}" + THROT_DEC=$((16#${THROT_HEX})) + FLAGS="" + (( THROT_DEC & 0x1 )) && FLAGS="$FLAGS • Currently under-voltage detected" + (( THROT_DEC & 0x2 )) && FLAGS="$FLAGS • Currently arm frequency capped" + (( THROT_DEC & 0x4 )) && FLAGS="$FLAGS • Currently throttled" + (( THROT_DEC & 0x10000 )) && FLAGS="$FLAGS • Under-voltage HAS occurred since boot" + (( THROT_DEC & 0x20000 )) && FLAGS="$FLAGS • Frequency cap HAS occurred since boot" + (( THROT_DEC & 0x40000 )) && FLAGS="$FLAGS • Throttling HAS occurred since boot" + + # Current bits (bits 0-3) are critical; historical bits (16-19) are warnings + if (( THROT_DEC & 0xF )); then + fail "Pi is CURRENTLY throttled/undervoltaged ($THROTTLED)" + else + warn "Throttling/undervoltage has occurred since boot ($THROTTLED)" + fi + [ -n "$FLAGS" ] && echo "$FLAGS" | sed 's/^/ /' + info " Use a 3A+ USB-C power supply or a powered USB hub for OBD/dashcam" +fi + +# Temperature +TEMP=$(vcgencmd measure_temp 2>/dev/null | cut -d= -f2 | cut -d\' -f1 || echo "unavailable") +if [ "$TEMP" != "unavailable" ]; then + TEMP_INT=${TEMP%.*} + if [ "${TEMP_INT:-0}" -lt 70 ]; then + pass "CPU temperature: ${TEMP}°C" + elif [ "${TEMP_INT:-0}" -lt 80 ]; then + warn "CPU temperature elevated: ${TEMP}°C" + else + fail "CPU temperature critical: ${TEMP}°C — check ventilation" + fi +fi + +# Memory +FREE_MB=$(free -m | awk '/^Mem:/{print $7}') +if [ "${FREE_MB:-0}" -gt 100 ]; then + pass "Free memory: ${FREE_MB}MB" +elif [ "${FREE_MB:-0}" -gt 50 ]; then + warn "Low free memory: ${FREE_MB}MB" +else + fail "Very low free memory: ${FREE_MB}MB" +fi + +# ── 8. Quick fixes menu ─────────────────────────────────────────────────────── +section "8. Manual Recovery Commands" + +info "Restart service: sudo systemctl restart $SERVICE" +info "View live logs: tail -f $LOG_FILE" +info "Journalctl logs: sudo journalctl -u $SERVICE -f" +info "Force OBD_PROTOCOL=6: sudo systemctl edit $SERVICE" +info " (add: Environment=OBD_PROTOCOL=6 then restart)" +info "Skip voltage check: sudo systemctl edit $SERVICE" +info " (add: Environment=OBD_SKIP_VOLTAGE_CHECK=1 then restart)" +info "DB backup: cp $DB_PATH ${DB_PATH}.bak.\$(date +%Y%m%d_%H%M%S)" +info "Recompute trips: curl http://localhost:5000/api/trips/refresh" +info "Check USB devices: lsusb && ls /dev/ttyUSB*" +info "Check serial port: ls -la /dev/serial0" + +# ── Summary ─────────────────────────────────────────────────────────────────── +echo +bold "==========================================" +if [ "$ISSUES" -eq 0 ] && [ "$WARNINGS" -eq 0 ]; then + green " ✓ All checks passed — system looks healthy" +elif [ "$ISSUES" -eq 0 ]; then + yellow " ⚠ $WARNINGS warning(s) found, $ISSUES critical issue(s)" + if ! $FIX; then + yellow " Run with --fix to attempt auto-fixes" + fi +else + red " ✗ $ISSUES critical issue(s), $WARNINGS warning(s) found" + if ! $FIX; then + yellow " Run with --fix to attempt auto-fixes" + fi +fi +bold "==========================================" +echo + +exit "$ISSUES"