diff --git a/modules/app_upload.py b/modules/app_upload.py index c077a15..51b1621 100644 --- a/modules/app_upload.py +++ b/modules/app_upload.py @@ -31,6 +31,7 @@ from threading import Thread import requests from dotenv import load_dotenv +from requests_toolbelt import MultipartEncoder from modules import app_sql, app_dash, app_cellular @@ -175,12 +176,26 @@ def _upload_dashcam_files(): if app_sql.dashcam_file_marked_uploaded(rel_path, sha256): continue + # Stream the multipart body via MultipartEncoder instead of requests' + # plain files=/data= kwargs. requests builds a non-streamed files= + # upload by reading the *entire* file into memory up front to + # assemble the multipart body - fine for small files, but real + # dashcam clips run 300+MB each, and on a ~1GB-RAM Pi 3 that alone is + # enough to trigger the OOM killer (confirmed against real hardware: + # RSS spiked in lockstep with dashcam file size). MultipartEncoder + # reads the file in small chunks as the request is sent instead. with open(abs_path, "rb") as f: + encoder = MultipartEncoder( + fields={ + "relative_path": rel_path, + "sha256": sha256, + "file": (os.path.basename(abs_path), f, "application/octet-stream"), + } + ) resp = requests.post( f"{SERVER_URL}/api/v1/dashcam/upload", - data={"relative_path": rel_path, "sha256": sha256}, - files={"file": f}, - headers=_headers(), + data=encoder, + headers={**_headers(), "Content-Type": encoder.content_type}, timeout=300, ) resp.raise_for_status() diff --git a/requirements.txt b/requirements.txt index df821e5..7c39194 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ pymongo python-dotenv pyserial obd -requests \ No newline at end of file +requests +requests-toolbelt \ No newline at end of file