"""Pluggable transfer sources and byte-copy primitives. ``Source`` is the seam for different download protocols: a local share of 7z archives, a remote HTTPS store server, etc. """ import json import logging import os import shutil import tempfile import urllib.parse import urllib.request from pathlib import Path from .store import TransferCancelled, extract_iso, peek_archive, scan log = logging.getLogger("transfers") _CHUNK = 1 << 22 # 4 MiB read chunks def copy_stream(src, dst, progress_cb=None, cancel_event=None): """Stream-copy ``src`` -> ``dst``, reporting fraction done. Returns bytes.""" src, dst = Path(src), Path(dst) total = src.stat().st_size dst.parent.mkdir(parents=True, exist_ok=True) done = 0 with open(src, "rb") as fsrc, open(dst, "wb") as fdst: while True: if cancel_event is not None and cancel_event.is_set(): raise TransferCancelled() block = fsrc.read(_CHUNK) if not block: break fdst.write(block) done += len(block) if progress_cb is not None and total: progress_cb("Copying", min(done / total, 1.0), "") return done def move_with_progress(src, dst, progress_cb=None, cancel_event=None): """Move a file: rename if possible, else stream-copy then unlink.""" src, dst = Path(src), Path(dst) dst.parent.mkdir(parents=True, exist_ok=True) if dst.exists(): dst.unlink() try: os.replace(src, dst) if progress_cb is not None: progress_cb("Moving", 1.0, "") return dst except OSError: pass total = src.stat().st_size done = 0 with open(src, "rb") as fsrc, open(dst, "wb") as fdst: while True: if cancel_event is not None and cancel_event.is_set(): fdst.close() try: dst.unlink() except OSError: pass raise TransferCancelled() block = fsrc.read(_CHUNK) if not block: break fdst.write(block) done += len(block) if progress_cb is not None and total: progress_cb("Moving", min(done / total, 1.0), "") try: src.unlink() except OSError: pass return dst def download_url(url, dest, headers=None, progress_cb=None, cancel_event=None): """Stream-download ``url`` to ``dest`` with progress + cancel. Returns dest.""" dest = Path(dest) dest.parent.mkdir(parents=True, exist_ok=True) req = urllib.request.Request(url, headers=headers or {}) resp = None try: resp = urllib.request.urlopen(req, timeout=30) total = int(resp.headers.get("Content-Length") or 0) done = 0 with open(dest, "wb") as out: while True: if cancel_event is not None and cancel_event.is_set(): raise TransferCancelled() chunk = resp.read(_CHUNK) if not chunk: break out.write(chunk) done += len(chunk) if progress_cb is not None and total: progress_cb("Downloading", min(done / total, 1.0), "") except Exception: try: dest.unlink() except OSError: pass raise finally: if resp is not None: resp.close() return dest def remove_path(path): """Delete a file or directory, using the desktop trash when available.""" path = Path(path) if not path.exists() and not path.is_symlink(): return True # already gone try: import send2trash # optional dependency; best-effort trash send2trash.send2trash(str(path)) return True except Exception: # noqa: BLE001 pass try: if path.is_dir() and not path.is_symlink(): shutil.rmtree(path) else: path.unlink() return True except OSError: return False def copy_tree(src, dst, progress_cb=None, cancel_event=None): """Replace ``dst`` with a copy of directory ``src`` (atomic; safe on cancel).""" src, dst = Path(src), Path(dst) if not src.is_dir(): raise RuntimeError(f"not a directory: {src}") files = [p for p in sorted(src.rglob("*")) if p.is_file()] total = sum(p.stat().st_size for p in files) or 1 done = 0 tmp = dst.with_name(dst.name + ".tmp-sync") if tmp.exists(): shutil.rmtree(tmp, ignore_errors=True) tmp.mkdir(parents=True, exist_ok=True) try: for p in files: if cancel_event is not None and cancel_event.is_set(): raise TransferCancelled() rel = p.relative_to(src) target = tmp / rel target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(p, target) done += p.stat().st_size if progress_cb is not None: progress_cb("Syncing", min(done / total, 1.0), str(rel)) if dst.exists(): shutil.rmtree(dst, ignore_errors=True) os.replace(tmp, dst) except Exception: shutil.rmtree(tmp, ignore_errors=True) raise return dst class Source: """A pluggable source of disc images (local share, HTTP, SFTP, ...).""" label = "source" def scan(self, root=None): raise NotImplementedError def download(self, item, dest_dir, progress_cb=None, cancel_event=None): raise NotImplementedError class FileCopySource(Source): """Copy a single local file (library -> PSP or PSP -> library). The queue passes an ``item`` dict that may carry a ``dest_name`` override; otherwise the source's own filename is used. """ label = "Local file" def __init__(self, path): self.path = Path(path) def scan(self, root=None): return [] def download(self, item, dest_dir, progress_cb=None, cancel_event=None): dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) dest_name = (item or {}).get("dest_name") or self.path.name dest = dest_dir / dest_name if dest.exists(): raise FileExistsError(str(dest)) copy_stream(self.path, dest, progress_cb, cancel_event) return dest class TreeCopySource(Source): """Copy a whole directory (save-data folder) into a destination directory.""" label = "Save folder" def __init__(self, path): self.path = Path(path) def scan(self, root=None): return [] def download(self, item, dest_dir, progress_cb=None, cancel_event=None): name = (item or {}).get("dest_name") or self.path.name dest = Path(dest_dir) / name return copy_tree(self.path, dest, progress_cb, cancel_event) class RemoteStoreSource(Source): """Browse and download 7z archives from a remote HTTPS store server.""" label = "Remote store" def __init__(self, base_url, token=""): self.base_url = (base_url or "").rstrip("/") self.token = token or "" def _headers(self): headers = {"User-Agent": "psp-preserve/0.1"} if self.token: headers["Authorization"] = f"Bearer {self.token}" return headers def scan(self, root=None): if not self.base_url: return [] req = urllib.request.Request(f"{self.base_url}/api/store", headers=self._headers()) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) return data.get("items", []) if isinstance(data, dict) else [] def download(self, item, dest_dir, progress_cb=None, cancel_event=None): dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) # The listing's "name" has no extension; "filename" carries the real # archive filename (e.g. "Game (USA).7z") used for the download URL. name = item.get("filename") or item["name"] tmp = Path(tempfile.mkdtemp(prefix="psp-store-")) archive = tmp / name try: url = f"{self.base_url}/api/store/download/{urllib.parse.quote(name, safe='')}" download_url(url, archive, self._headers(), progress_cb, cancel_event) iso_name = peek_archive(archive, cancel_event) if not iso_name: raise RuntimeError("No disc image found inside the archive") dest = dest_dir / iso_name if dest.exists(): raise FileExistsError(str(dest)) iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event) return move_with_progress(iso_path, dest, progress_cb, cancel_event) finally: shutil.rmtree(tmp, ignore_errors=True) def _archive_item_identifier(value): """Normalize an archive.org input to a bare item identifier.""" v = (value or "").strip().rstrip("/") if not v: return "" if "://" in v: v = v.split("://", 1)[1] for prefix in ("archive.org/download/", "archive.org/details/", "archive.org/metadata/"): if prefix in v: return v.split(prefix, 1)[1].split("/", 1)[0] return v class ArchiveOrgSource(Source): """Browse a public archive.org item holding the same Redump 7z set.""" label = "Archive.org" def __init__(self, item, sig="", user=""): self.item = _archive_item_identifier(item) self.sig = (sig or "").strip() self.user = (user or "").strip() def _base(self): return f"https://archive.org/download/{self.item}" def _headers(self): headers = {"User-Agent": "psp-preserve/0.1"} cookie = "; ".join( p for p in ( f"logged-in-sig={self.sig}" if self.sig else "", f"logged-in-user={self.user}" if self.user else "", ) if p ) if cookie: headers["Cookie"] = cookie return headers def scan(self, root=None): if not self.item: return [] url = f"https://archive.org/metadata/{self.item}" req = urllib.request.Request(url, headers=self._headers()) with urllib.request.urlopen(req, timeout=60) as resp: data = json.loads(resp.read()) items = [] archive_count = 0 private_count = 0 has_auth = bool(self.sig or self.user) for f in (data.get("files") or []) if isinstance(data, dict) else []: if not isinstance(f, dict): continue name = f.get("name") or "" if not name.lower().endswith((".7z", ".zip")): continue archive_count += 1 if str(f.get("private")).lower() == "true": private_count += 1 if not has_auth: continue try: size = int(f.get("size") or 0) except (TypeError, ValueError): size = 0 try: mtime_ns = int(float(f.get("mtime") or 0)) * 1_000_000_000 except (TypeError, ValueError): mtime_ns = 0 items.append({ "name": Path(name).stem, "filename": name, "size": size, "mtime_ns": mtime_ns, }) if private_count: log.warning("archive.org: %d of %d archive(s) are private%s", private_count, archive_count, "" if has_auth else " (login required)") if not items and archive_count and private_count == archive_count and not has_auth: raise RuntimeError( f"archive.org item '{self.item}': all {archive_count} archive(s) are private " "(login required) — try a different item." ) return items def download(self, item, dest_dir, progress_cb=None, cancel_event=None): dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) name = item.get("filename") or item["name"] tmp = Path(tempfile.mkdtemp(prefix="psp-archive-")) archive = tmp / Path(name).name try: url = f"{self._base()}/{urllib.parse.quote(name, safe='/')}" try: download_url(url, archive, self._headers(), progress_cb, cancel_event) except urllib.error.HTTPError as exc: if exc.code in (401, 403): raise RuntimeError( f"Archive.org returned {exc.code} for '{name}' — the file is likely " "private (login required) and can't be downloaded anonymously." ) from exc raise iso_name = peek_archive(archive, cancel_event) if not iso_name: raise RuntimeError("No disc image found inside the archive") dest = dest_dir / iso_name if dest.exists(): raise FileExistsError(str(dest)) iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event) return move_with_progress(iso_path, dest, progress_cb, cancel_event) finally: shutil.rmtree(tmp, ignore_errors=True) class LocalShareSource(Source): """A local (or mounted) folder of 7z archives.""" label = "Local share" def __init__(self, root): self.root = Path(root) def scan(self, root=None): return scan(root or self.root) def download(self, item, dest_dir, progress_cb=None, cancel_event=None): archive = Path(item["path"]) dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) iso_name = peek_archive(archive, cancel_event) if not iso_name: raise RuntimeError("No disc image found inside the archive") dest = dest_dir / iso_name if dest.exists(): raise FileExistsError(str(dest)) tmp = Path(tempfile.mkdtemp(prefix="psp-store-")) try: local = tmp / archive.name copy_stream(archive, local, progress_cb, cancel_event) iso_path = extract_iso(local, iso_name, tmp, progress_cb, cancel_event) move_with_progress(iso_path, dest, progress_cb, cancel_event) return dest finally: shutil.rmtree(tmp, ignore_errors=True)