"""Pluggable transfer sources and byte-copy primitives. ``Source`` is the seam for different download protocols: a local share of 7z archives, a remote HTTPS store server, etc. """ import json import logging import os import re import shutil import tempfile import urllib.parse import urllib.request from pathlib import Path from .store import TransferCancelled, extract_iso, peek_archive, scan log = logging.getLogger("transfers") _CHUNK = 1 << 22 # 4 MiB read chunks def copy_stream(src, dst, progress_cb=None, cancel_event=None): """Stream-copy ``src`` -> ``dst``, reporting fraction done. Returns bytes.""" src, dst = Path(src), Path(dst) total = src.stat().st_size dst.parent.mkdir(parents=True, exist_ok=True) done = 0 with open(src, "rb") as fsrc, open(dst, "wb") as fdst: while True: if cancel_event is not None and cancel_event.is_set(): raise TransferCancelled() block = fsrc.read(_CHUNK) if not block: break fdst.write(block) done += len(block) if progress_cb is not None and total: progress_cb("Copying", min(done / total, 1.0), "") return done def move_with_progress(src, dst, progress_cb=None, cancel_event=None): """Move a file: rename if possible, else stream-copy then unlink.""" src, dst = Path(src), Path(dst) dst.parent.mkdir(parents=True, exist_ok=True) if dst.exists(): dst.unlink() try: os.replace(src, dst) if progress_cb is not None: progress_cb("Moving", 1.0, "") return dst except OSError: pass total = src.stat().st_size done = 0 with open(src, "rb") as fsrc, open(dst, "wb") as fdst: while True: if cancel_event is not None and cancel_event.is_set(): fdst.close() try: dst.unlink() except OSError: pass raise TransferCancelled() block = fsrc.read(_CHUNK) if not block: break fdst.write(block) done += len(block) if progress_cb is not None and total: progress_cb("Moving", min(done / total, 1.0), "") try: src.unlink() except OSError: pass return dst def download_url(url, dest, headers=None, progress_cb=None, cancel_event=None): """Stream-download ``url`` to ``dest`` with progress + cancel. Returns dest.""" dest = Path(dest) dest.parent.mkdir(parents=True, exist_ok=True) req = urllib.request.Request(url, headers=headers or {}) resp = None try: resp = urllib.request.urlopen(req, timeout=30) total = int(resp.headers.get("Content-Length") or 0) done = 0 with open(dest, "wb") as out: while True: if cancel_event is not None and cancel_event.is_set(): raise TransferCancelled() chunk = resp.read(_CHUNK) if not chunk: break out.write(chunk) done += len(chunk) if progress_cb is not None and total: progress_cb("Downloading", min(done / total, 1.0), "") except Exception: try: dest.unlink() except OSError: pass raise finally: if resp is not None: resp.close() return dest def remove_path(path): """Delete a file or directory, using the desktop trash when available.""" path = Path(path) if not path.exists() and not path.is_symlink(): return True # already gone try: import send2trash # optional dependency; best-effort trash send2trash.send2trash(str(path)) return True except Exception: # noqa: BLE001 pass try: if path.is_dir() and not path.is_symlink(): shutil.rmtree(path) else: path.unlink() return True except OSError: return False def copy_tree(src, dst, progress_cb=None, cancel_event=None): """Replace ``dst`` with a copy of directory ``src`` (atomic; safe on cancel).""" src, dst = Path(src), Path(dst) if not src.is_dir(): raise RuntimeError(f"not a directory: {src}") files = [p for p in sorted(src.rglob("*")) if p.is_file()] total = sum(p.stat().st_size for p in files) or 1 done = 0 tmp = dst.with_name(dst.name + ".tmp-sync") if tmp.exists(): shutil.rmtree(tmp, ignore_errors=True) tmp.mkdir(parents=True, exist_ok=True) try: for p in files: if cancel_event is not None and cancel_event.is_set(): raise TransferCancelled() rel = p.relative_to(src) target = tmp / rel target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(p, target) done += p.stat().st_size if progress_cb is not None: progress_cb("Syncing", min(done / total, 1.0), str(rel)) if dst.exists(): shutil.rmtree(dst, ignore_errors=True) os.replace(tmp, dst) except Exception: shutil.rmtree(tmp, ignore_errors=True) raise return dst class Source: """A pluggable source of disc images (local share, HTTP, SFTP, ...).""" label = "source" def scan(self, root=None): raise NotImplementedError def download(self, item, dest_dir, progress_cb=None, cancel_event=None): raise NotImplementedError class FileCopySource(Source): """Copy a single local file (library -> PSP or PSP -> library). The queue passes an ``item`` dict that may carry a ``dest_name`` override; otherwise the source's own filename is used. """ label = "Local file" def __init__(self, path): self.path = Path(path) def scan(self, root=None): return [] def download(self, item, dest_dir, progress_cb=None, cancel_event=None): dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) dest_name = (item or {}).get("dest_name") or self.path.name dest = dest_dir / dest_name if dest.exists(): raise FileExistsError(str(dest)) copy_stream(self.path, dest, progress_cb, cancel_event) return dest class Ps1ConvertSource(Source): """Convert a PS1 disc image (.cue/.iso) into an EBOOT.PBP. ``item`` must carry ``source`` (path to the .cue or single image), ``title`` and ``game_id``. ``base_pbp`` is a PSN PS1 classic EBOOT used to supply the encrypted DATA.PSP loader and STARTDAT block. """ label = "PS1 convert" def __init__(self, base_pbp): self.base_pbp = Path(base_pbp) def scan(self, root=None): return [] def download(self, item, dest_dir, progress_cb=None, cancel_event=None): dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) source = Path(item["source"]) title = str(item.get("title") or source.stem) game_id = item.get("game_id") if not game_id: raise RuntimeError("missing game_id for PS1 conversion") folder_name = re.sub(r"[^A-Za-z0-9]", "", game_id).upper() or "PS1GAME" folder = dest_dir / folder_name eboot = folder / "EBOOT.PBP" if eboot.exists(): raise FileExistsError(str(eboot)) def cb(done, total): if progress_cb is not None and total: progress_cb("Converting", min(done / total, 1.0), "") from . import covers, ps1_convert # Pull the game's real cover art (libretro / ScreenScraper / IGDB / # Wikipedia) and bake it into ICON0 + PIC0 + PIC1 so the XMB shows the # official-style art, not the default PS1 icon. Best effort. if progress_cb is not None: progress_cb("Artwork", 0.0, "") icon0, pic0, pic1 = covers.ps1_art(title, game_id) try: ps1_convert.convert_to_folder( source, title, game_id, self.base_pbp, dest_dir, progress_cb=cb, cancel_event=cancel_event, icon0=icon0, pic0=pic0, pic1=pic1, ) except ps1_convert.Cancelled: raise TransferCancelled() from None return folder class TreeCopySource(Source): """Copy a whole directory (save-data folder) into a destination directory.""" label = "Save folder" def __init__(self, path): self.path = Path(path) def scan(self, root=None): return [] def download(self, item, dest_dir, progress_cb=None, cancel_event=None): name = (item or {}).get("dest_name") or self.path.name dest = Path(dest_dir) / name return copy_tree(self.path, dest, progress_cb, cancel_event) class RemoteStoreSource(Source): """Browse and download 7z archives from a remote HTTPS store server.""" label = "Remote store" def __init__(self, base_url, token=""): self.base_url = (base_url or "").rstrip("/") self.token = token or "" def _headers(self): headers = {"User-Agent": "psppt/0.1"} if self.token: headers["Authorization"] = f"Bearer {self.token}" return headers def scan(self, root=None): if not self.base_url: return [] req = urllib.request.Request(f"{self.base_url}/api/store", headers=self._headers()) with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read()) return data.get("items", []) if isinstance(data, dict) else [] def download(self, item, dest_dir, progress_cb=None, cancel_event=None): dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) # The listing's "name" has no extension; "filename" carries the real # archive filename (e.g. "Game (USA).7z") used for the download URL. name = item.get("filename") or item["name"] tmp = Path(tempfile.mkdtemp(prefix="psp-store-")) archive = tmp / name try: url = f"{self.base_url}/api/store/download/{urllib.parse.quote(name, safe='')}" download_url(url, archive, self._headers(), progress_cb, cancel_event) iso_name = peek_archive(archive, cancel_event) if not iso_name: raise RuntimeError("No disc image found inside the archive") dest = dest_dir / iso_name if dest.exists(): raise FileExistsError(str(dest)) iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event) return move_with_progress(iso_path, dest, progress_cb, cancel_event) finally: shutil.rmtree(tmp, ignore_errors=True) def _archive_item_identifier(value): """Normalize an archive.org input to a bare item identifier.""" v = (value or "").strip().rstrip("/") if not v: return "" if "://" in v: v = v.split("://", 1)[1] for prefix in ("archive.org/download/", "archive.org/details/", "archive.org/metadata/"): if prefix in v: return v.split(prefix, 1)[1].split("/", 1)[0] return v class ArchiveOrgSource(Source): """Browse a public archive.org item holding the same Redump 7z set.""" label = "Archive.org" def __init__(self, item, sig="", user=""): self.item = _archive_item_identifier(item) self.sig = (sig or "").strip() self.user = (user or "").strip() def _base(self): return f"https://archive.org/download/{self.item}" def _headers(self): headers = {"User-Agent": "psppt/0.1"} cookie = "; ".join( p for p in ( f"logged-in-sig={self.sig}" if self.sig else "", f"logged-in-user={self.user}" if self.user else "", ) if p ) if cookie: headers["Cookie"] = cookie return headers def scan(self, root=None): if not self.item: return [] url = f"https://archive.org/metadata/{self.item}" req = urllib.request.Request(url, headers=self._headers()) with urllib.request.urlopen(req, timeout=60) as resp: data = json.loads(resp.read()) items = [] archive_count = 0 private_count = 0 has_auth = bool(self.sig or self.user) for f in (data.get("files") or []) if isinstance(data, dict) else []: if not isinstance(f, dict): continue name = f.get("name") or "" if not name.lower().endswith((".7z", ".zip")): continue archive_count += 1 if str(f.get("private")).lower() == "true": private_count += 1 if not has_auth: continue try: size = int(f.get("size") or 0) except (TypeError, ValueError): size = 0 try: mtime_ns = int(float(f.get("mtime") or 0)) * 1_000_000_000 except (TypeError, ValueError): mtime_ns = 0 items.append({ "name": Path(name).stem, "filename": name, "size": size, "mtime_ns": mtime_ns, }) if private_count: log.warning("archive.org: %d of %d archive(s) are private%s", private_count, archive_count, "" if has_auth else " (login required)") if not items and archive_count and private_count == archive_count and not has_auth: raise RuntimeError( f"archive.org item '{self.item}': all {archive_count} archive(s) are private " "(login required) — try a different item." ) return items def download(self, item, dest_dir, progress_cb=None, cancel_event=None): dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) name = item.get("filename") or item["name"] tmp = Path(tempfile.mkdtemp(prefix="psp-archive-")) archive = tmp / Path(name).name try: url = f"{self._base()}/{urllib.parse.quote(name, safe='/')}" try: download_url(url, archive, self._headers(), progress_cb, cancel_event) except urllib.error.HTTPError as exc: if exc.code in (401, 403): raise RuntimeError( f"Archive.org returned {exc.code} for '{name}' — the file is likely " "private (login required) and can't be downloaded anonymously." ) from exc raise iso_name = peek_archive(archive, cancel_event) if not iso_name: raise RuntimeError("No disc image found inside the archive") dest = dest_dir / iso_name if dest.exists(): raise FileExistsError(str(dest)) iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event) return move_with_progress(iso_path, dest, progress_cb, cancel_event) finally: shutil.rmtree(tmp, ignore_errors=True) class LocalShareSource(Source): """A local (or mounted) folder of 7z archives.""" label = "Local share" def __init__(self, root): self.root = Path(root) def scan(self, root=None): return scan(root or self.root) def download(self, item, dest_dir, progress_cb=None, cancel_event=None): archive = Path(item["path"]) dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) iso_name = peek_archive(archive, cancel_event) if not iso_name: raise RuntimeError("No disc image found inside the archive") dest = dest_dir / iso_name if dest.exists(): raise FileExistsError(str(dest)) tmp = Path(tempfile.mkdtemp(prefix="psp-store-")) try: local = tmp / archive.name copy_stream(archive, local, progress_cb, cancel_event) iso_path = extract_iso(local, iso_name, tmp, progress_cb, cancel_event) move_with_progress(iso_path, dest, progress_cb, cancel_event) return dest finally: shutil.rmtree(tmp, ignore_errors=True) class NoPayStationSource(Source): """Browse NoPayStation .pkg lists and install PSP/PSX PSN content. ``content_type`` is ``"psp"`` (PSP_GAMES.tsv), ``"psx"`` (PSX_GAMES.tsv), ``"theme"`` (PSP_THEMES.tsv) or ``"dlc"`` (PSP_DLCS.tsv). Downloading decrypts the ``.pkg`` with the bundled pure-Python decryptor and writes a ``/`` game folder into ``dest_dir``. """ label = "NoPayStation" def __init__(self, content_type, tsv_url="", force=False): from . import nopaystation self.content_type = content_type self.tsv_url = tsv_url or nopaystation.tsv_url(content_type) self.force = force def scan(self, root=None): from . import nopaystation return nopaystation.fetch_items(self.content_type, self.tsv_url, force=self.force) def download(self, item, dest_dir, progress_cb=None, cancel_event=None): from . import nopaystation dest_dir = Path(dest_dir) dest_dir.mkdir(parents=True, exist_ok=True) # The bundled Python decryptor handles PSP/PSX/themes/DLC directly. title_id = item.get("title_id") or item.get("content_id") or "" pkg_name = re.sub(r"[^A-Za-z0-9_-]", "", title_id) or "pkg" tmp = Path(tempfile.mkdtemp(prefix="nps-download-")) pkg = tmp / f"{pkg_name}.pkg" try: if progress_cb is not None: progress_cb("Downloading", 0.0, "") download_url(item["pkg_url"], pkg, None, progress_cb, cancel_event) kind_hint = self.content_type if self.content_type in ("theme", "dlc") else None folder = nopaystation.extract_pkg(pkg, title_id, dest_dir, kind_hint, progress_cb, cancel_event) if self.content_type in ("theme", "dlc"): _write_sidecar(folder, item) return folder finally: shutil.rmtree(tmp, ignore_errors=True) def _write_sidecar(folder, item): """Save the store display name beside a downloaded theme/DLC (for the library).""" try: meta = {"name": item.get("name") or "", "title_id": item.get("title_id") or "", "content_id": item.get("content_id") or ""} (Path(folder) / "meta.json").write_text(json.dumps(meta, ensure_ascii=False), encoding="utf-8") except OSError: pass