524 lines
19 KiB
Python
524 lines
19 KiB
Python
"""Pluggable transfer sources and byte-copy primitives.
|
|
|
|
``Source`` is the seam for different download protocols: a local share of 7z
|
|
archives, a remote HTTPS store server, etc.
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from .store import TransferCancelled, extract_iso, peek_archive, scan
|
|
|
|
log = logging.getLogger("transfers")
|
|
|
|
_CHUNK = 1 << 22 # 4 MiB read chunks
|
|
|
|
|
|
def copy_stream(src, dst, progress_cb=None, cancel_event=None):
|
|
"""Stream-copy ``src`` -> ``dst``, reporting fraction done. Returns bytes."""
|
|
src, dst = Path(src), Path(dst)
|
|
total = src.stat().st_size
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
done = 0
|
|
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
|
while True:
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
raise TransferCancelled()
|
|
block = fsrc.read(_CHUNK)
|
|
if not block:
|
|
break
|
|
fdst.write(block)
|
|
done += len(block)
|
|
if progress_cb is not None and total:
|
|
progress_cb("Copying", min(done / total, 1.0), "")
|
|
return done
|
|
|
|
|
|
def move_with_progress(src, dst, progress_cb=None, cancel_event=None):
|
|
"""Move a file: rename if possible, else stream-copy then unlink."""
|
|
src, dst = Path(src), Path(dst)
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
if dst.exists():
|
|
dst.unlink()
|
|
try:
|
|
os.replace(src, dst)
|
|
if progress_cb is not None:
|
|
progress_cb("Moving", 1.0, "")
|
|
return dst
|
|
except OSError:
|
|
pass
|
|
total = src.stat().st_size
|
|
done = 0
|
|
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
|
while True:
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
fdst.close()
|
|
try:
|
|
dst.unlink()
|
|
except OSError:
|
|
pass
|
|
raise TransferCancelled()
|
|
block = fsrc.read(_CHUNK)
|
|
if not block:
|
|
break
|
|
fdst.write(block)
|
|
done += len(block)
|
|
if progress_cb is not None and total:
|
|
progress_cb("Moving", min(done / total, 1.0), "")
|
|
try:
|
|
src.unlink()
|
|
except OSError:
|
|
pass
|
|
return dst
|
|
|
|
|
|
def download_url(url, dest, headers=None, progress_cb=None, cancel_event=None):
|
|
"""Stream-download ``url`` to ``dest`` with progress + cancel. Returns dest."""
|
|
dest = Path(dest)
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
req = urllib.request.Request(url, headers=headers or {})
|
|
resp = None
|
|
try:
|
|
resp = urllib.request.urlopen(req, timeout=30)
|
|
total = int(resp.headers.get("Content-Length") or 0)
|
|
done = 0
|
|
with open(dest, "wb") as out:
|
|
while True:
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
raise TransferCancelled()
|
|
chunk = resp.read(_CHUNK)
|
|
if not chunk:
|
|
break
|
|
out.write(chunk)
|
|
done += len(chunk)
|
|
if progress_cb is not None and total:
|
|
progress_cb("Downloading", min(done / total, 1.0), "")
|
|
except Exception:
|
|
try:
|
|
dest.unlink()
|
|
except OSError:
|
|
pass
|
|
raise
|
|
finally:
|
|
if resp is not None:
|
|
resp.close()
|
|
return dest
|
|
|
|
|
|
def remove_path(path):
|
|
"""Delete a file or directory, using the desktop trash when available."""
|
|
path = Path(path)
|
|
if not path.exists() and not path.is_symlink():
|
|
return True # already gone
|
|
try:
|
|
import send2trash # optional dependency; best-effort trash
|
|
send2trash.send2trash(str(path))
|
|
return True
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
try:
|
|
if path.is_dir() and not path.is_symlink():
|
|
shutil.rmtree(path)
|
|
else:
|
|
path.unlink()
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def copy_tree(src, dst, progress_cb=None, cancel_event=None):
|
|
"""Replace ``dst`` with a copy of directory ``src`` (atomic; safe on cancel)."""
|
|
src, dst = Path(src), Path(dst)
|
|
if not src.is_dir():
|
|
raise RuntimeError(f"not a directory: {src}")
|
|
files = [p for p in sorted(src.rglob("*")) if p.is_file()]
|
|
total = sum(p.stat().st_size for p in files) or 1
|
|
done = 0
|
|
tmp = dst.with_name(dst.name + ".tmp-sync")
|
|
if tmp.exists():
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
tmp.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
for p in files:
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
raise TransferCancelled()
|
|
rel = p.relative_to(src)
|
|
target = tmp / rel
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(p, target)
|
|
done += p.stat().st_size
|
|
if progress_cb is not None:
|
|
progress_cb("Syncing", min(done / total, 1.0), str(rel))
|
|
if dst.exists():
|
|
shutil.rmtree(dst, ignore_errors=True)
|
|
os.replace(tmp, dst)
|
|
except Exception:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
raise
|
|
return dst
|
|
|
|
|
|
class Source:
|
|
"""A pluggable source of disc images (local share, HTTP, SFTP, ...)."""
|
|
label = "source"
|
|
|
|
def scan(self, root=None):
|
|
raise NotImplementedError
|
|
|
|
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
|
raise NotImplementedError
|
|
|
|
|
|
class FileCopySource(Source):
|
|
"""Copy a single local file (library -> PSP or PSP -> library).
|
|
|
|
The queue passes an ``item`` dict that may carry a ``dest_name`` override;
|
|
otherwise the source's own filename is used.
|
|
"""
|
|
|
|
label = "Local file"
|
|
|
|
def __init__(self, path):
|
|
self.path = Path(path)
|
|
|
|
def scan(self, root=None):
|
|
return []
|
|
|
|
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
|
dest_dir = Path(dest_dir)
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
dest_name = (item or {}).get("dest_name") or self.path.name
|
|
dest = dest_dir / dest_name
|
|
if dest.exists():
|
|
raise FileExistsError(str(dest))
|
|
copy_stream(self.path, dest, progress_cb, cancel_event)
|
|
return dest
|
|
|
|
|
|
class Ps1ConvertSource(Source):
|
|
"""Convert a PS1 disc image (.cue/.iso) into an EBOOT.PBP.
|
|
|
|
``item`` must carry ``source`` (path to the .cue or single image),
|
|
``title`` and ``game_id``. ``base_pbp`` is a PSN PS1 classic EBOOT used to
|
|
supply the encrypted DATA.PSP loader and STARTDAT block.
|
|
"""
|
|
|
|
label = "PS1 convert"
|
|
|
|
def __init__(self, base_pbp):
|
|
self.base_pbp = Path(base_pbp)
|
|
|
|
def scan(self, root=None):
|
|
return []
|
|
|
|
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
|
dest_dir = Path(dest_dir)
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
source = Path(item["source"])
|
|
title = str(item.get("title") or source.stem)
|
|
game_id = item.get("game_id")
|
|
if not game_id:
|
|
raise RuntimeError("missing game_id for PS1 conversion")
|
|
|
|
folder_name = re.sub(r"[^A-Za-z0-9]", "", game_id).upper() or "PS1GAME"
|
|
folder = dest_dir / folder_name
|
|
eboot = folder / "EBOOT.PBP"
|
|
if eboot.exists():
|
|
raise FileExistsError(str(eboot))
|
|
|
|
def cb(done, total):
|
|
if progress_cb is not None and total:
|
|
progress_cb("Converting", min(done / total, 1.0), "")
|
|
|
|
from . import covers, ps1_convert
|
|
|
|
# Pull the game's real cover art (libretro / ScreenScraper / IGDB /
|
|
# Wikipedia) and bake it into ICON0 + PIC0 + PIC1 so the XMB shows the
|
|
# official-style art, not the default PS1 icon. Best effort.
|
|
if progress_cb is not None:
|
|
progress_cb("Artwork", 0.0, "")
|
|
icon0, pic0, pic1 = covers.ps1_art(title, game_id)
|
|
|
|
try:
|
|
ps1_convert.convert_to_folder(
|
|
source, title, game_id, self.base_pbp, dest_dir,
|
|
progress_cb=cb, cancel_event=cancel_event,
|
|
icon0=icon0, pic0=pic0, pic1=pic1,
|
|
)
|
|
except ps1_convert.Cancelled:
|
|
raise TransferCancelled() from None
|
|
return folder
|
|
|
|
|
|
class TreeCopySource(Source):
|
|
"""Copy a whole directory (save-data folder) into a destination directory."""
|
|
|
|
label = "Save folder"
|
|
|
|
def __init__(self, path):
|
|
self.path = Path(path)
|
|
|
|
def scan(self, root=None):
|
|
return []
|
|
|
|
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
|
name = (item or {}).get("dest_name") or self.path.name
|
|
dest = Path(dest_dir) / name
|
|
return copy_tree(self.path, dest, progress_cb, cancel_event)
|
|
|
|
|
|
class RemoteStoreSource(Source):
|
|
"""Browse and download 7z archives from a remote HTTPS store server."""
|
|
|
|
label = "Remote store"
|
|
|
|
def __init__(self, base_url, token=""):
|
|
self.base_url = (base_url or "").rstrip("/")
|
|
self.token = token or ""
|
|
|
|
def _headers(self):
|
|
headers = {"User-Agent": "psp-preserve/0.1"}
|
|
if self.token:
|
|
headers["Authorization"] = f"Bearer {self.token}"
|
|
return headers
|
|
|
|
def scan(self, root=None):
|
|
if not self.base_url:
|
|
return []
|
|
req = urllib.request.Request(f"{self.base_url}/api/store", headers=self._headers())
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
data = json.loads(resp.read())
|
|
return data.get("items", []) if isinstance(data, dict) else []
|
|
|
|
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
|
dest_dir = Path(dest_dir)
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
# The listing's "name" has no extension; "filename" carries the real
|
|
# archive filename (e.g. "Game (USA).7z") used for the download URL.
|
|
name = item.get("filename") or item["name"]
|
|
tmp = Path(tempfile.mkdtemp(prefix="psp-store-"))
|
|
archive = tmp / name
|
|
try:
|
|
url = f"{self.base_url}/api/store/download/{urllib.parse.quote(name, safe='')}"
|
|
download_url(url, archive, self._headers(), progress_cb, cancel_event)
|
|
iso_name = peek_archive(archive, cancel_event)
|
|
if not iso_name:
|
|
raise RuntimeError("No disc image found inside the archive")
|
|
dest = dest_dir / iso_name
|
|
if dest.exists():
|
|
raise FileExistsError(str(dest))
|
|
iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event)
|
|
return move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
|
|
def _archive_item_identifier(value):
|
|
"""Normalize an archive.org input to a bare item identifier."""
|
|
v = (value or "").strip().rstrip("/")
|
|
if not v:
|
|
return ""
|
|
if "://" in v:
|
|
v = v.split("://", 1)[1]
|
|
for prefix in ("archive.org/download/", "archive.org/details/", "archive.org/metadata/"):
|
|
if prefix in v:
|
|
return v.split(prefix, 1)[1].split("/", 1)[0]
|
|
return v
|
|
|
|
|
|
class ArchiveOrgSource(Source):
|
|
"""Browse a public archive.org item holding the same Redump 7z set."""
|
|
|
|
label = "Archive.org"
|
|
|
|
def __init__(self, item, sig="", user=""):
|
|
self.item = _archive_item_identifier(item)
|
|
self.sig = (sig or "").strip()
|
|
self.user = (user or "").strip()
|
|
|
|
def _base(self):
|
|
return f"https://archive.org/download/{self.item}"
|
|
|
|
def _headers(self):
|
|
headers = {"User-Agent": "psp-preserve/0.1"}
|
|
cookie = "; ".join(
|
|
p for p in (
|
|
f"logged-in-sig={self.sig}" if self.sig else "",
|
|
f"logged-in-user={self.user}" if self.user else "",
|
|
) if p
|
|
)
|
|
if cookie:
|
|
headers["Cookie"] = cookie
|
|
return headers
|
|
|
|
def scan(self, root=None):
|
|
if not self.item:
|
|
return []
|
|
url = f"https://archive.org/metadata/{self.item}"
|
|
req = urllib.request.Request(url, headers=self._headers())
|
|
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
data = json.loads(resp.read())
|
|
items = []
|
|
archive_count = 0
|
|
private_count = 0
|
|
has_auth = bool(self.sig or self.user)
|
|
for f in (data.get("files") or []) if isinstance(data, dict) else []:
|
|
if not isinstance(f, dict):
|
|
continue
|
|
name = f.get("name") or ""
|
|
if not name.lower().endswith((".7z", ".zip")):
|
|
continue
|
|
archive_count += 1
|
|
if str(f.get("private")).lower() == "true":
|
|
private_count += 1
|
|
if not has_auth:
|
|
continue
|
|
try:
|
|
size = int(f.get("size") or 0)
|
|
except (TypeError, ValueError):
|
|
size = 0
|
|
try:
|
|
mtime_ns = int(float(f.get("mtime") or 0)) * 1_000_000_000
|
|
except (TypeError, ValueError):
|
|
mtime_ns = 0
|
|
items.append({
|
|
"name": Path(name).stem,
|
|
"filename": name,
|
|
"size": size,
|
|
"mtime_ns": mtime_ns,
|
|
})
|
|
if private_count:
|
|
log.warning("archive.org: %d of %d archive(s) are private%s",
|
|
private_count, archive_count, "" if has_auth else " (login required)")
|
|
if not items and archive_count and private_count == archive_count and not has_auth:
|
|
raise RuntimeError(
|
|
f"archive.org item '{self.item}': all {archive_count} archive(s) are private "
|
|
"(login required) — try a different item."
|
|
)
|
|
return items
|
|
|
|
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
|
dest_dir = Path(dest_dir)
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
name = item.get("filename") or item["name"]
|
|
tmp = Path(tempfile.mkdtemp(prefix="psp-archive-"))
|
|
archive = tmp / Path(name).name
|
|
try:
|
|
url = f"{self._base()}/{urllib.parse.quote(name, safe='/')}"
|
|
try:
|
|
download_url(url, archive, self._headers(), progress_cb, cancel_event)
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code in (401, 403):
|
|
raise RuntimeError(
|
|
f"Archive.org returned {exc.code} for '{name}' — the file is likely "
|
|
"private (login required) and can't be downloaded anonymously."
|
|
) from exc
|
|
raise
|
|
iso_name = peek_archive(archive, cancel_event)
|
|
if not iso_name:
|
|
raise RuntimeError("No disc image found inside the archive")
|
|
dest = dest_dir / iso_name
|
|
if dest.exists():
|
|
raise FileExistsError(str(dest))
|
|
iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event)
|
|
return move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
|
|
class LocalShareSource(Source):
|
|
"""A local (or mounted) folder of 7z archives."""
|
|
|
|
label = "Local share"
|
|
|
|
def __init__(self, root):
|
|
self.root = Path(root)
|
|
|
|
def scan(self, root=None):
|
|
return scan(root or self.root)
|
|
|
|
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
|
archive = Path(item["path"])
|
|
dest_dir = Path(dest_dir)
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
iso_name = peek_archive(archive, cancel_event)
|
|
if not iso_name:
|
|
raise RuntimeError("No disc image found inside the archive")
|
|
dest = dest_dir / iso_name
|
|
if dest.exists():
|
|
raise FileExistsError(str(dest))
|
|
tmp = Path(tempfile.mkdtemp(prefix="psp-store-"))
|
|
try:
|
|
local = tmp / archive.name
|
|
copy_stream(archive, local, progress_cb, cancel_event)
|
|
iso_path = extract_iso(local, iso_name, tmp, progress_cb, cancel_event)
|
|
move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
|
return dest
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
|
|
class NoPayStationSource(Source):
|
|
"""Browse NoPayStation .pkg lists and install PSP/PSX PSN content.
|
|
|
|
``content_type`` is ``"psp"`` (PSP_GAMES.tsv), ``"psx"`` (PSX_GAMES.tsv),
|
|
``"theme"`` (PSP_THEMES.tsv) or ``"dlc"`` (PSP_DLCS.tsv). Downloading
|
|
decrypts the ``.pkg`` with the bundled pure-Python decryptor and writes a
|
|
``<TITLE_ID>/`` game folder into ``dest_dir``.
|
|
"""
|
|
|
|
label = "NoPayStation"
|
|
|
|
def __init__(self, content_type, tsv_url="", pkg2zip_path=""):
|
|
from . import nopaystation
|
|
self.content_type = content_type
|
|
self.tsv_url = tsv_url or nopaystation.tsv_url(content_type)
|
|
self.pkg2zip = pkg2zip_path or ""
|
|
|
|
def scan(self, root=None):
|
|
from . import nopaystation
|
|
return nopaystation.fetch_items(self.content_type, self.tsv_url)
|
|
|
|
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
|
from . import nopaystation
|
|
dest_dir = Path(dest_dir)
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# The bundled Python decryptor handles PSP/PSX/themes/DLC directly;
|
|
# pkg2zip is only an optional fallback if a binary happens to be
|
|
# configured (and only supports games/PSX).
|
|
pkg2zip = nopaystation.find_pkg2zip(self.pkg2zip)
|
|
title_id = item.get("title_id") or item.get("content_id") or ""
|
|
pkg_name = re.sub(r"[^A-Za-z0-9_-]", "", title_id) or "pkg"
|
|
tmp = Path(tempfile.mkdtemp(prefix="nps-download-"))
|
|
pkg = tmp / f"{pkg_name}.pkg"
|
|
try:
|
|
if progress_cb is not None:
|
|
progress_cb("Downloading", 0.0, "")
|
|
download_url(item["pkg_url"], pkg, None, progress_cb, cancel_event)
|
|
kind_hint = self.content_type if self.content_type in ("theme", "dlc") else None
|
|
folder = nopaystation.extract_pkg(pkg, title_id, dest_dir, pkg2zip,
|
|
kind_hint, progress_cb, cancel_event)
|
|
if self.content_type in ("theme", "dlc"):
|
|
_write_sidecar(folder, item)
|
|
return folder
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
|
|
def _write_sidecar(folder, item):
|
|
"""Save the store display name beside a downloaded theme/DLC (for the library)."""
|
|
try:
|
|
meta = {"name": item.get("name") or "", "title_id": item.get("title_id") or "",
|
|
"content_id": item.get("content_id") or ""}
|
|
(Path(folder) / "meta.json").write_text(json.dumps(meta, ensure_ascii=False), encoding="utf-8")
|
|
except OSError:
|
|
pass
|