Initial commit
This commit is contained in:
@@ -0,0 +1,408 @@
|
||||
"""Pluggable transfer sources and byte-copy primitives.
|
||||
|
||||
``Source`` is the seam for different download protocols: a local share of 7z
|
||||
archives, a remote HTTPS store server, etc.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from .store import TransferCancelled, extract_iso, peek_archive, scan
|
||||
|
||||
log = logging.getLogger("transfers")
|
||||
|
||||
_CHUNK = 1 << 22 # 4 MiB read chunks
|
||||
|
||||
|
||||
def copy_stream(src, dst, progress_cb=None, cancel_event=None):
|
||||
"""Stream-copy ``src`` -> ``dst``, reporting fraction done. Returns bytes."""
|
||||
src, dst = Path(src), Path(dst)
|
||||
total = src.stat().st_size
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
done = 0
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
raise TransferCancelled()
|
||||
block = fsrc.read(_CHUNK)
|
||||
if not block:
|
||||
break
|
||||
fdst.write(block)
|
||||
done += len(block)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb("Copying", min(done / total, 1.0), "")
|
||||
return done
|
||||
|
||||
|
||||
def move_with_progress(src, dst, progress_cb=None, cancel_event=None):
|
||||
"""Move a file: rename if possible, else stream-copy then unlink."""
|
||||
src, dst = Path(src), Path(dst)
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dst.exists():
|
||||
dst.unlink()
|
||||
try:
|
||||
os.replace(src, dst)
|
||||
if progress_cb is not None:
|
||||
progress_cb("Moving", 1.0, "")
|
||||
return dst
|
||||
except OSError:
|
||||
pass
|
||||
total = src.stat().st_size
|
||||
done = 0
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
fdst.close()
|
||||
try:
|
||||
dst.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise TransferCancelled()
|
||||
block = fsrc.read(_CHUNK)
|
||||
if not block:
|
||||
break
|
||||
fdst.write(block)
|
||||
done += len(block)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb("Moving", min(done / total, 1.0), "")
|
||||
try:
|
||||
src.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
return dst
|
||||
|
||||
|
||||
def download_url(url, dest, headers=None, progress_cb=None, cancel_event=None):
|
||||
"""Stream-download ``url`` to ``dest`` with progress + cancel. Returns dest."""
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
req = urllib.request.Request(url, headers=headers or {})
|
||||
resp = None
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=30)
|
||||
total = int(resp.headers.get("Content-Length") or 0)
|
||||
done = 0
|
||||
with open(dest, "wb") as out:
|
||||
while True:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
raise TransferCancelled()
|
||||
chunk = resp.read(_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
done += len(chunk)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb("Downloading", min(done / total, 1.0), "")
|
||||
except Exception:
|
||||
try:
|
||||
dest.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
finally:
|
||||
if resp is not None:
|
||||
resp.close()
|
||||
return dest
|
||||
|
||||
|
||||
def remove_path(path):
|
||||
"""Delete a file or directory, using the desktop trash when available."""
|
||||
path = Path(path)
|
||||
if not path.exists() and not path.is_symlink():
|
||||
return True # already gone
|
||||
try:
|
||||
import send2trash # optional dependency; best-effort trash
|
||||
send2trash.send2trash(str(path))
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
try:
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
path.unlink()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def copy_tree(src, dst, progress_cb=None, cancel_event=None):
|
||||
"""Replace ``dst`` with a copy of directory ``src`` (atomic; safe on cancel)."""
|
||||
src, dst = Path(src), Path(dst)
|
||||
if not src.is_dir():
|
||||
raise RuntimeError(f"not a directory: {src}")
|
||||
files = [p for p in sorted(src.rglob("*")) if p.is_file()]
|
||||
total = sum(p.stat().st_size for p in files) or 1
|
||||
done = 0
|
||||
tmp = dst.with_name(dst.name + ".tmp-sync")
|
||||
if tmp.exists():
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
tmp.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
for p in files:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
raise TransferCancelled()
|
||||
rel = p.relative_to(src)
|
||||
target = tmp / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, target)
|
||||
done += p.stat().st_size
|
||||
if progress_cb is not None:
|
||||
progress_cb("Syncing", min(done / total, 1.0), str(rel))
|
||||
if dst.exists():
|
||||
shutil.rmtree(dst, ignore_errors=True)
|
||||
os.replace(tmp, dst)
|
||||
except Exception:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
raise
|
||||
return dst
|
||||
|
||||
|
||||
class Source:
|
||||
"""A pluggable source of disc images (local share, HTTP, SFTP, ...)."""
|
||||
label = "source"
|
||||
|
||||
def scan(self, root=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FileCopySource(Source):
|
||||
"""Copy a single local file (library -> PSP or PSP -> library).
|
||||
|
||||
The queue passes an ``item`` dict that may carry a ``dest_name`` override;
|
||||
otherwise the source's own filename is used.
|
||||
"""
|
||||
|
||||
label = "Local file"
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = Path(path)
|
||||
|
||||
def scan(self, root=None):
|
||||
return []
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
dest_dir = Path(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest_name = (item or {}).get("dest_name") or self.path.name
|
||||
dest = dest_dir / dest_name
|
||||
if dest.exists():
|
||||
raise FileExistsError(str(dest))
|
||||
copy_stream(self.path, dest, progress_cb, cancel_event)
|
||||
return dest
|
||||
|
||||
|
||||
class TreeCopySource(Source):
|
||||
"""Copy a whole directory (save-data folder) into a destination directory."""
|
||||
|
||||
label = "Save folder"
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = Path(path)
|
||||
|
||||
def scan(self, root=None):
|
||||
return []
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
name = (item or {}).get("dest_name") or self.path.name
|
||||
dest = Path(dest_dir) / name
|
||||
return copy_tree(self.path, dest, progress_cb, cancel_event)
|
||||
|
||||
|
||||
class RemoteStoreSource(Source):
|
||||
"""Browse and download 7z archives from a remote HTTPS store server."""
|
||||
|
||||
label = "Remote store"
|
||||
|
||||
def __init__(self, base_url, token=""):
|
||||
self.base_url = (base_url or "").rstrip("/")
|
||||
self.token = token or ""
|
||||
|
||||
def _headers(self):
|
||||
headers = {"User-Agent": "psp-preserve/0.1"}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"Bearer {self.token}"
|
||||
return headers
|
||||
|
||||
def scan(self, root=None):
|
||||
if not self.base_url:
|
||||
return []
|
||||
req = urllib.request.Request(f"{self.base_url}/api/store", headers=self._headers())
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read())
|
||||
return data.get("items", []) if isinstance(data, dict) else []
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
dest_dir = Path(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
# The listing's "name" has no extension; "filename" carries the real
|
||||
# archive filename (e.g. "Game (USA).7z") used for the download URL.
|
||||
name = item.get("filename") or item["name"]
|
||||
tmp = Path(tempfile.mkdtemp(prefix="psp-store-"))
|
||||
archive = tmp / name
|
||||
try:
|
||||
url = f"{self.base_url}/api/store/download/{urllib.parse.quote(name, safe='')}"
|
||||
download_url(url, archive, self._headers(), progress_cb, cancel_event)
|
||||
iso_name = peek_archive(archive, cancel_event)
|
||||
if not iso_name:
|
||||
raise RuntimeError("No disc image found inside the archive")
|
||||
dest = dest_dir / iso_name
|
||||
if dest.exists():
|
||||
raise FileExistsError(str(dest))
|
||||
iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event)
|
||||
return move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def _archive_item_identifier(value):
|
||||
"""Normalize an archive.org input to a bare item identifier."""
|
||||
v = (value or "").strip().rstrip("/")
|
||||
if not v:
|
||||
return ""
|
||||
if "://" in v:
|
||||
v = v.split("://", 1)[1]
|
||||
for prefix in ("archive.org/download/", "archive.org/details/", "archive.org/metadata/"):
|
||||
if prefix in v:
|
||||
return v.split(prefix, 1)[1].split("/", 1)[0]
|
||||
return v
|
||||
|
||||
|
||||
class ArchiveOrgSource(Source):
|
||||
"""Browse a public archive.org item holding the same Redump 7z set."""
|
||||
|
||||
label = "Archive.org"
|
||||
|
||||
def __init__(self, item, sig="", user=""):
|
||||
self.item = _archive_item_identifier(item)
|
||||
self.sig = (sig or "").strip()
|
||||
self.user = (user or "").strip()
|
||||
|
||||
def _base(self):
|
||||
return f"https://archive.org/download/{self.item}"
|
||||
|
||||
def _headers(self):
|
||||
headers = {"User-Agent": "psp-preserve/0.1"}
|
||||
cookie = "; ".join(
|
||||
p for p in (
|
||||
f"logged-in-sig={self.sig}" if self.sig else "",
|
||||
f"logged-in-user={self.user}" if self.user else "",
|
||||
) if p
|
||||
)
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
return headers
|
||||
|
||||
def scan(self, root=None):
|
||||
if not self.item:
|
||||
return []
|
||||
url = f"https://archive.org/metadata/{self.item}"
|
||||
req = urllib.request.Request(url, headers=self._headers())
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
data = json.loads(resp.read())
|
||||
items = []
|
||||
archive_count = 0
|
||||
private_count = 0
|
||||
has_auth = bool(self.sig or self.user)
|
||||
for f in (data.get("files") or []) if isinstance(data, dict) else []:
|
||||
if not isinstance(f, dict):
|
||||
continue
|
||||
name = f.get("name") or ""
|
||||
if not name.lower().endswith((".7z", ".zip")):
|
||||
continue
|
||||
archive_count += 1
|
||||
if str(f.get("private")).lower() == "true":
|
||||
private_count += 1
|
||||
if not has_auth:
|
||||
continue
|
||||
try:
|
||||
size = int(f.get("size") or 0)
|
||||
except (TypeError, ValueError):
|
||||
size = 0
|
||||
try:
|
||||
mtime_ns = int(float(f.get("mtime") or 0)) * 1_000_000_000
|
||||
except (TypeError, ValueError):
|
||||
mtime_ns = 0
|
||||
items.append({
|
||||
"name": Path(name).stem,
|
||||
"filename": name,
|
||||
"size": size,
|
||||
"mtime_ns": mtime_ns,
|
||||
})
|
||||
if private_count:
|
||||
log.warning("archive.org: %d of %d archive(s) are private%s",
|
||||
private_count, archive_count, "" if has_auth else " (login required)")
|
||||
if not items and archive_count and private_count == archive_count and not has_auth:
|
||||
raise RuntimeError(
|
||||
f"archive.org item '{self.item}': all {archive_count} archive(s) are private "
|
||||
"(login required) — try a different item."
|
||||
)
|
||||
return items
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
dest_dir = Path(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
name = item.get("filename") or item["name"]
|
||||
tmp = Path(tempfile.mkdtemp(prefix="psp-archive-"))
|
||||
archive = tmp / Path(name).name
|
||||
try:
|
||||
url = f"{self._base()}/{urllib.parse.quote(name, safe='/')}"
|
||||
try:
|
||||
download_url(url, archive, self._headers(), progress_cb, cancel_event)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (401, 403):
|
||||
raise RuntimeError(
|
||||
f"Archive.org returned {exc.code} for '{name}' — the file is likely "
|
||||
"private (login required) and can't be downloaded anonymously."
|
||||
) from exc
|
||||
raise
|
||||
iso_name = peek_archive(archive, cancel_event)
|
||||
if not iso_name:
|
||||
raise RuntimeError("No disc image found inside the archive")
|
||||
dest = dest_dir / iso_name
|
||||
if dest.exists():
|
||||
raise FileExistsError(str(dest))
|
||||
iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event)
|
||||
return move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
class LocalShareSource(Source):
|
||||
"""A local (or mounted) folder of 7z archives."""
|
||||
|
||||
label = "Local share"
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = Path(root)
|
||||
|
||||
def scan(self, root=None):
|
||||
return scan(root or self.root)
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
archive = Path(item["path"])
|
||||
dest_dir = Path(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
iso_name = peek_archive(archive, cancel_event)
|
||||
if not iso_name:
|
||||
raise RuntimeError("No disc image found inside the archive")
|
||||
dest = dest_dir / iso_name
|
||||
if dest.exists():
|
||||
raise FileExistsError(str(dest))
|
||||
tmp = Path(tempfile.mkdtemp(prefix="psp-store-"))
|
||||
try:
|
||||
local = tmp / archive.name
|
||||
copy_stream(archive, local, progress_cb, cancel_event)
|
||||
iso_path = extract_iso(local, iso_name, tmp, progress_cb, cancel_event)
|
||||
move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
||||
return dest
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
Reference in New Issue
Block a user