Root-caused the slow-upload complaint to the Pi's own congested 2.4GHz WiFi radio (confirmed via iperf3 - same degraded throughput even on pure local LAN traffic, ruling out Cloudflare/ISP). Since the Pi 3 B has no 5GHz radio, add a way to bypass it for the internet-bound leg: a phone opens /dash and manually relays pending dashcam files straight to the remote server over its own network, reusing the exact same chunked upload protocol the Pi's client already uses. - server/app.py: add CORS headers (after_request) so the phone's browser can call the remote server cross-origin from a page served by the Pi. - app.py: new endpoints - GET /api/dash/pending (list not-yet-uploaded files), GET /api/dash/file/<path> (Range-request-capable download, path-traversal safe), GET /api/dash/relay/config (hands the phone the device credentials), POST /api/dash/relay/heartbeat|stop (lease- based pause of the Pi's own background uploader, self-heals if the phone's tab closes uncleanly), POST /api/dash/relay/mark_uploaded. - modules/app_upload.py: list_pending_dashcam_files(), relay lease helpers, _upload_dashcam_files() now pauses while a relay lease is active. - modules/app_sql.py: new dashcam_hash_cache table so a file's sha256 is only computed once (keyed by path+size+mtime) instead of being recomputed on every loop iteration/pending-list call - benefits the existing background uploader too. - templates/dash.html / static/js/dash.js: new "Relay via phone" card - per-file and "relay all" buttons (manual trigger only, per user preference), downloads+uploads one 40MB slice at a time so the phone never holds a whole file in memory, retries chunks up to 3x. - PLAN.md/readme.md/server/README.md updated (Phase 11). Tested via a full local end-to-end simulation driving the real Pi and server Flask apps side by side (Postgres mocked only at the db module boundary - all HTTP/chunking/auth/CORS logic is real): verified Range downloads (206), byte-for-byte reassembly via SHA256, path-traversal rejection, and the relay lease correctly pausing/resuming the background uploader. Not yet tested against a real phone/live network.
383 lines
14 KiB
JavaScript
383 lines
14 KiB
JavaScript
// Dashcam page: scan for a USB dashcam, then trigger importing its videos.
|
|
// Uses plain fetch() (not socket.io / yaris.js) since this is a one-shot
|
|
// request/response flow, not a live data stream, and yaris.js assumes DOM
|
|
// elements (#map, #speedGraph) that don't exist on this page.
|
|
|
|
const deviceStatus = document.getElementById('dash-device-status');
|
|
const importStatus = document.getElementById('dash-import-status');
|
|
const scanBtn = document.getElementById('dash-scan-btn');
|
|
const importBtn = document.getElementById('dash-import-btn');
|
|
|
|
function describeDevice(device) {
|
|
const label = device.label ? ` "${device.label}"` : '';
|
|
return `${device.path}${label} (${device.size}, ${device.fstype || 'unknown fs'})`;
|
|
}
|
|
|
|
function scanForDashcam() {
|
|
deviceStatus.textContent = 'Scanning...';
|
|
importBtn.disabled = true;
|
|
|
|
fetch('/api/dash/devices')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
const devices = data.devices || [];
|
|
if (devices.length === 0) {
|
|
deviceStatus.textContent = 'No USB dashcam/storage device detected. Plug it in and scan again.';
|
|
return;
|
|
}
|
|
deviceStatus.textContent = 'Detected: ' + devices.map(describeDevice).join(', ');
|
|
importBtn.disabled = false;
|
|
})
|
|
.catch(err => {
|
|
deviceStatus.textContent = 'Error while scanning: ' + err;
|
|
});
|
|
}
|
|
|
|
function importDashcam() {
|
|
importStatus.textContent = 'Importing... this can take a while for large videos.';
|
|
importBtn.disabled = true;
|
|
|
|
fetch('/api/dash/import', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(summary => {
|
|
if (summary.status !== 'ok') {
|
|
importStatus.textContent = summary.message || ('Import failed: ' + summary.status);
|
|
return;
|
|
}
|
|
let text = `Found ${summary.found} video(s), copied ${summary.copied}, skipped ${summary.skipped} (already imported).`;
|
|
if (summary.errors && summary.errors.length > 0) {
|
|
text += ` Errors: ${summary.errors.join('; ')}`;
|
|
}
|
|
importStatus.textContent = text;
|
|
})
|
|
.catch(err => {
|
|
importStatus.textContent = 'Error while importing: ' + err;
|
|
})
|
|
.finally(() => {
|
|
importBtn.disabled = false;
|
|
});
|
|
}
|
|
|
|
scanBtn.addEventListener('click', scanForDashcam);
|
|
importBtn.addEventListener('click', importDashcam);
|
|
|
|
// Auto-scan once when the page loads.
|
|
scanForDashcam();
|
|
|
|
// --- Remote server upload status/toggle ---
|
|
|
|
const uploadStatusEl = document.getElementById('upload-status');
|
|
const dashcamToggle = document.getElementById('dashcam-upload-toggle');
|
|
|
|
function describeUploadStatus(status) {
|
|
if (!status.configured) {
|
|
return 'Remote server not configured on this device (see db/upload_creds.env).';
|
|
}
|
|
let text = `Pending trip-log rows to upload: ${status.pending_tracking_rows}.`;
|
|
if (status.last_success_at) {
|
|
text += ` Last successful upload: ${status.last_success_at}`;
|
|
if (status.last_network_path) {
|
|
text += ` (via ${status.last_network_path === 'cellular' ? 'cellular fallback' : 'WiFi/Ethernet'})`;
|
|
}
|
|
text += '.';
|
|
}
|
|
if (status.cellular_fallback_configured) {
|
|
text += ' Cellular (SIM868) fallback is enabled for trip logs if WiFi/Ethernet is unreachable.';
|
|
}
|
|
if (status.last_error) {
|
|
text += ` Last error: ${status.last_error}.`;
|
|
}
|
|
return text;
|
|
}
|
|
|
|
function refreshUploadStatus() {
|
|
fetch('/api/upload/status')
|
|
.then(response => response.json())
|
|
.then(status => {
|
|
uploadStatusEl.textContent = describeUploadStatus(status);
|
|
dashcamToggle.checked = status.dashcam_upload_enabled;
|
|
dashcamToggle.disabled = !status.configured;
|
|
})
|
|
.catch(err => {
|
|
uploadStatusEl.textContent = 'Error while checking upload status: ' + err;
|
|
});
|
|
}
|
|
|
|
dashcamToggle.addEventListener('change', () => {
|
|
fetch('/api/upload/dashcam_toggle', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ enabled: dashcamToggle.checked }),
|
|
})
|
|
.then(response => response.json())
|
|
.then(status => {
|
|
uploadStatusEl.textContent = describeUploadStatus(status);
|
|
dashcamToggle.checked = status.dashcam_upload_enabled;
|
|
})
|
|
.catch(err => {
|
|
uploadStatusEl.textContent = 'Error while updating toggle: ' + err;
|
|
});
|
|
});
|
|
|
|
refreshUploadStatus();
|
|
setInterval(refreshUploadStatus, 15000);
|
|
|
|
// --- Relay via phone ---
|
|
// Lets a phone (open on this same page, over the Pi's hotspot/LAN) download
|
|
// pending dashcam files from the Pi and re-upload them directly to the
|
|
// remote server using its own network, reusing the exact same chunked
|
|
// upload wire protocol the Pi's own background uploader uses
|
|
// (/api/v1/dashcam/upload/status + /chunk) - just orchestrated by this
|
|
// browser instead of the Pi's Python client. Downloads are pulled in the
|
|
// same chunk size and immediately re-uploaded, one at a time, so the phone
|
|
// never has to hold a whole (300+MB) file in memory at once.
|
|
|
|
const relayStatusEl = document.getElementById('relay-status');
|
|
const relayRefreshBtn = document.getElementById('relay-refresh-btn');
|
|
const relayAllBtn = document.getElementById('relay-all-btn');
|
|
const relayStopBtn = document.getElementById('relay-stop-btn');
|
|
const relayFileList = document.getElementById('relay-file-list');
|
|
|
|
let relayConfig = null;
|
|
let pendingFiles = [];
|
|
const relayRowEls = new Map(); // relative_path -> { statusSpan, btn }
|
|
let heartbeatTimer = null;
|
|
let activeRelayCount = 0;
|
|
let relayStopRequested = false;
|
|
|
|
function humanSize(bytes) {
|
|
if (bytes >= 1024 * 1024 * 1024) return (bytes / (1024 * 1024 * 1024)).toFixed(2) + ' GB';
|
|
if (bytes >= 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
|
|
if (bytes >= 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
|
return bytes + ' B';
|
|
}
|
|
|
|
function getRelayConfig() {
|
|
if (relayConfig) return Promise.resolve(relayConfig);
|
|
return fetch('/api/dash/relay/config')
|
|
.then(response => response.json())
|
|
.then(config => {
|
|
relayConfig = config;
|
|
return config;
|
|
});
|
|
}
|
|
|
|
function startHeartbeat() {
|
|
activeRelayCount++;
|
|
relayStopBtn.disabled = false;
|
|
if (heartbeatTimer) return;
|
|
fetch('/api/dash/relay/heartbeat', { method: 'POST' }).catch(() => {});
|
|
heartbeatTimer = setInterval(() => {
|
|
fetch('/api/dash/relay/heartbeat', { method: 'POST' }).catch(() => {});
|
|
}, 15000);
|
|
}
|
|
|
|
function stopHeartbeatIfIdle() {
|
|
activeRelayCount = Math.max(0, activeRelayCount - 1);
|
|
if (activeRelayCount > 0) return;
|
|
if (heartbeatTimer) {
|
|
clearInterval(heartbeatTimer);
|
|
heartbeatTimer = null;
|
|
}
|
|
relayStopBtn.disabled = true;
|
|
fetch('/api/dash/relay/stop', { method: 'POST' }).catch(() => {});
|
|
}
|
|
|
|
relayStopBtn.addEventListener('click', () => {
|
|
relayStopRequested = true;
|
|
activeRelayCount = 0;
|
|
if (heartbeatTimer) {
|
|
clearInterval(heartbeatTimer);
|
|
heartbeatTimer = null;
|
|
}
|
|
fetch('/api/dash/relay/stop', { method: 'POST' }).catch(() => {});
|
|
relayStatusEl.textContent = 'Relay stopped - the Pi will resume its own uploads shortly.';
|
|
relayStopBtn.disabled = true;
|
|
});
|
|
|
|
// Best-effort: try to release the pause lease immediately if the page is
|
|
// closed mid-relay (it also self-expires after ~45s either way).
|
|
window.addEventListener('beforeunload', () => {
|
|
if (heartbeatTimer && navigator.sendBeacon) {
|
|
navigator.sendBeacon('/api/dash/relay/stop');
|
|
}
|
|
});
|
|
|
|
async function fetchChunkWithRetry(relativePath, start, end, attempts = 3) {
|
|
let lastErr;
|
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
try {
|
|
const response = await fetch(`/api/dash/file/${encodeURIComponent(relativePath)}`, {
|
|
headers: { Range: `bytes=${start}-${end}` },
|
|
});
|
|
if (!response.ok && response.status !== 206) {
|
|
throw new Error(`download HTTP ${response.status}`);
|
|
}
|
|
return await response.blob();
|
|
} catch (err) {
|
|
lastErr = err;
|
|
await new Promise(r => setTimeout(r, 1000 * attempt));
|
|
}
|
|
}
|
|
throw lastErr;
|
|
}
|
|
|
|
async function uploadChunkWithRetry(config, relativePath, sha256, chunkIndex, totalChunks, blob, attempts = 3) {
|
|
let lastErr;
|
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
try {
|
|
const form = new FormData();
|
|
form.append('relative_path', relativePath);
|
|
form.append('sha256', sha256);
|
|
form.append('chunk_index', String(chunkIndex));
|
|
form.append('total_chunks', String(totalChunks));
|
|
form.append('file', blob, `chunk_${chunkIndex}`);
|
|
const response = await fetch(`${config.server_url}/api/v1/dashcam/upload/chunk`, {
|
|
method: 'POST',
|
|
headers: { 'X-Device-Name': config.device_name, 'X-Api-Key': config.api_key },
|
|
body: form,
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`upload HTTP ${response.status}`);
|
|
}
|
|
return await response.json();
|
|
} catch (err) {
|
|
lastErr = err;
|
|
await new Promise(r => setTimeout(r, 1000 * attempt));
|
|
}
|
|
}
|
|
throw lastErr;
|
|
}
|
|
|
|
async function relayOneFile(file, statusEl) {
|
|
const config = await getRelayConfig();
|
|
if (!config.configured) {
|
|
statusEl.textContent = 'Remote server not configured on this device.';
|
|
return;
|
|
}
|
|
|
|
relayStopRequested = false;
|
|
startHeartbeat();
|
|
try {
|
|
const statusResp = await fetch(
|
|
`${config.server_url}/api/v1/dashcam/upload/status?relative_path=${encodeURIComponent(file.relative_path)}&sha256=${file.sha256}`,
|
|
{ headers: { 'X-Device-Name': config.device_name, 'X-Api-Key': config.api_key } }
|
|
);
|
|
if (!statusResp.ok) throw new Error(`status check HTTP ${statusResp.status}`);
|
|
const status = await statusResp.json();
|
|
|
|
if (status.status !== 'already_uploaded') {
|
|
const received = new Set(status.received_chunk_indices || []);
|
|
const totalChunks = Math.max(1, Math.ceil(file.size / config.chunk_size));
|
|
|
|
for (let idx = 0; idx < totalChunks; idx++) {
|
|
if (relayStopRequested) {
|
|
statusEl.textContent = 'Stopped.';
|
|
return;
|
|
}
|
|
if (received.has(idx)) continue;
|
|
|
|
const start = idx * config.chunk_size;
|
|
const end = Math.min(start + config.chunk_size, file.size) - 1;
|
|
statusEl.textContent = `Relaying... chunk ${idx + 1}/${totalChunks}`;
|
|
|
|
const blob = await fetchChunkWithRetry(file.relative_path, start, end);
|
|
await uploadChunkWithRetry(config, file.relative_path, file.sha256, idx, totalChunks, blob);
|
|
}
|
|
}
|
|
|
|
// Update the Pi's own local dedupe table right away so the pending
|
|
// list (and its own background uploader) reflect this immediately,
|
|
// instead of waiting for its next full backlog scan.
|
|
await fetch('/api/dash/relay/mark_uploaded', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ relative_path: file.relative_path, sha256: file.sha256 }),
|
|
}).catch(() => {});
|
|
|
|
statusEl.textContent = 'Done.';
|
|
pendingFiles = pendingFiles.filter(f => f.relative_path !== file.relative_path);
|
|
} catch (err) {
|
|
statusEl.textContent = 'Error: ' + err;
|
|
throw err;
|
|
} finally {
|
|
stopHeartbeatIfIdle();
|
|
}
|
|
}
|
|
|
|
function renderPendingFiles() {
|
|
relayFileList.innerHTML = '';
|
|
relayRowEls.clear();
|
|
|
|
if (pendingFiles.length === 0) {
|
|
relayStatusEl.textContent = 'No pending dashcam files to relay.';
|
|
relayAllBtn.disabled = true;
|
|
return;
|
|
}
|
|
|
|
relayStatusEl.textContent = `${pendingFiles.length} file(s) pending upload.`;
|
|
relayAllBtn.disabled = false;
|
|
|
|
pendingFiles.forEach(file => {
|
|
const li = document.createElement('li');
|
|
li.className = 'list-group-item d-flex justify-content-between align-items-center flex-wrap gap-2';
|
|
|
|
const label = document.createElement('span');
|
|
label.textContent = `${file.relative_path} (${humanSize(file.size)})`;
|
|
|
|
const statusSpan = document.createElement('span');
|
|
statusSpan.className = 'text-muted small';
|
|
|
|
const btn = document.createElement('button');
|
|
btn.className = 'btn btn-sm btn-outline-primary';
|
|
btn.type = 'button';
|
|
btn.textContent = 'Relay';
|
|
btn.addEventListener('click', () => {
|
|
btn.disabled = true;
|
|
relayOneFile(file, statusSpan).finally(() => {
|
|
btn.disabled = false;
|
|
});
|
|
});
|
|
|
|
li.appendChild(label);
|
|
li.appendChild(statusSpan);
|
|
li.appendChild(btn);
|
|
relayFileList.appendChild(li);
|
|
relayRowEls.set(file.relative_path, { statusSpan, btn });
|
|
});
|
|
}
|
|
|
|
function refreshPendingFiles() {
|
|
relayStatusEl.textContent = 'Checking pending files...';
|
|
fetch('/api/dash/pending')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
pendingFiles = data.files || [];
|
|
renderPendingFiles();
|
|
})
|
|
.catch(err => {
|
|
relayStatusEl.textContent = 'Error while checking pending files: ' + err;
|
|
});
|
|
}
|
|
|
|
relayRefreshBtn.addEventListener('click', refreshPendingFiles);
|
|
|
|
relayAllBtn.addEventListener('click', async () => {
|
|
relayStopRequested = false;
|
|
relayAllBtn.disabled = true;
|
|
for (const file of pendingFiles.slice()) {
|
|
if (relayStopRequested) break;
|
|
const row = relayRowEls.get(file.relative_path);
|
|
const statusSpan = row ? row.statusSpan : relayStatusEl;
|
|
if (row) row.btn.disabled = true;
|
|
try {
|
|
await relayOneFile(file, statusSpan);
|
|
} catch (err) {
|
|
// Keep going with the remaining files even if one fails.
|
|
}
|
|
}
|
|
relayAllBtn.disabled = pendingFiles.length === 0;
|
|
});
|
|
|
|
refreshPendingFiles();
|