Source code for mga_format.archive
"""A near-optimal set as one archive, for publishing and downloading.
A set is a directory with subdirectories (``sets/``, ``outputs/``), but places
such as GitHub releases hold only flat files. So a set is published as one zip
archive of its directory: :func:`pack` creates it, :func:`unpack` restores the
directory, and :func:`fetch` downloads and unpacks one. Both check the set with
:func:`~mga_format.near_optimal_set.read` before handing it over.
"""
from __future__ import annotations
import shutil
import tempfile
import urllib.request
import zipfile
from pathlib import Path, PurePosixPath
from .near_optimal_set import MANIFEST_NAME, NearOptimalSet, NearOptimalSetError, read
[docs]
def pack(directory: str | Path, archive: str | Path) -> Path:
"""Write the set in `directory` to the zip file `archive`, after checking
it. Files are stored uncompressed: Parquet files are already compressed."""
directory, archive = Path(directory), Path(archive)
read(directory)
with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_STORED) as zf:
for path in sorted(p for p in directory.rglob("*") if p.is_file()):
zf.write(path, path.relative_to(directory).as_posix())
return archive
def _safe_members(zf: zipfile.ZipFile) -> list[str]:
names = zf.namelist()
for name in names:
parts = PurePosixPath(name).parts
if name.startswith("/") or ".." in parts or (parts and ":" in parts[0]):
raise NearOptimalSetError(f"archive member {name!r} points outside the set")
if MANIFEST_NAME not in names:
raise NearOptimalSetError(f"archive has no {MANIFEST_NAME}")
return names
[docs]
def unpack(archive: str | Path, directory: str | Path) -> NearOptimalSet:
"""Unpack the zip file `archive` into `directory`, replacing whatever is
there, and return the set read from it. The set is unpacked and checked
next to `directory` first, so a broken archive leaves `directory` as it was."""
archive, directory = Path(archive), Path(directory)
directory.parent.mkdir(parents=True, exist_ok=True)
staging = Path(tempfile.mkdtemp(prefix=".unpack-", dir=directory.parent))
try:
with zipfile.ZipFile(archive) as zf:
zf.extractall(staging, members=_safe_members(zf))
read(staging)
if directory.exists():
shutil.rmtree(directory)
staging.rename(directory)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return read(directory)
[docs]
def fetch(url: str, directory: str | Path, timeout: float = 600) -> NearOptimalSet:
"""Download the archive at `url` and unpack it into `directory`, replacing
whatever is there, and return the set."""
directory = Path(directory)
directory.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=directory.parent, suffix=".zip", delete=False) as tmp:
download = Path(tmp.name)
try:
with urllib.request.urlopen(url, timeout=timeout) as response, open(download, "wb") as fh:
shutil.copyfileobj(response, fh)
return unpack(download, directory)
finally:
download.unlink(missing_ok=True)