Source code for mga_surveyor.export

"""Export a `generate_mga_files` run as a near-optimal set (``mga_format``).

`export_near_optimal_set` reads the result files `generate_mga_files` wrote to
its `output_dir` and writes them in the format MGA Compass reads: one point per
result file (the optimum, the box points and the nearest points of every slack
level), with its cost and duals and the cache key in its file name (``key``), plus the PoI definitions, the optimal cost,
each slack level's run summary and the network's fingerprint.
"""
from __future__ import annotations

import hashlib
import json
from datetime import datetime, timezone
from importlib.metadata import PackageNotFoundError, version
from pathlib import Path

import numpy as np
import pandas as pd
import pypsa

from mga_format import Cost, Fingerprint, Manifest, NearOptimalSet, Poi, SlackLevel, write

from .pypsa_ext import RUN_SUMMARY_NAME, _read_meta

_PACKAGES = ("mga-surveyor", "mga-format", "pypsa", "linopy")


[docs] def network_fingerprint(n: pypsa.Network) -> str: """The sha256 of `n`'s input data: its snapshots and their weightings, and every component's static attributes and time series that PyPSA marks as input. Optimization results are left out, so solving `n` again does not change the fingerprint, while changing any cost, limit or time series does.""" digest = hashlib.sha256() def add(label: str, df: pd.DataFrame) -> None: digest.update(label.encode()) digest.update(json.dumps([str(c) for c in df.columns]).encode()) if not df.empty: digest.update(pd.util.hash_pandas_object(df, index=True).to_numpy().tobytes()) add("snapshot_weightings", n.snapshot_weightings.sort_index(axis=1)) for component in n.components: attrs = component.attrs inputs = set(attrs.index[attrs["status"].str.startswith("Input")]) static = component.static add(f"{component.name}.static", static[sorted(c for c in static.columns if c in inputs)]) for attr in sorted(component.dynamic): series = component.dynamic[attr] if attr in inputs and not series.empty: add(f"{component.name}.{attr}", series.sort_index(axis=1)) return digest.hexdigest()
def _slack_dirs(output_dir: Path) -> list[tuple[float, Path]]: dirs = [(float(p.name.removeprefix("slack_")), p) for p in output_dir.glob("slack_*") if p.is_dir()] return sorted(dirs) def _key(path: Path) -> str | None: """The cache key in a result file's name, ``mga_*_result_<key>.nc``; None for ``optimal.nc``.""" _, separator, key = path.stem.partition("_result_") return key if separator else None def _row(meta: dict, keys: list[str], coordinates_key: str) -> tuple[list[float], list[float], float]: coordinates = [float(meta[coordinates_key][k]) for k in keys] duals = [float(meta["duals"][k]) for k in keys] if "duals" in meta else [np.nan] * len(keys) return coordinates, duals, float(meta["obj"]) def _software() -> dict[str, str]: versions = {} for package in _PACKAGES: try: versions[package] = version(package) except PackageNotFoundError: pass return versions
[docs] def export_near_optimal_set( output_dir: str | Path, directory: str | Path, *, units: dict[str, str], cost_unit: str, cost_name: str = "cost", descriptions: dict[str, str] | None = None, ) -> Manifest: """Write the results of a `generate_mga_files` run in `output_dir` as a near-optimal set in the new directory `directory`, and return its manifest. The set has no outputs; to add some, use :func:`collect_near_optimal_set` and write the set with ``mga_format.write`` yourself. `units` gives every PoI's unit, `cost_unit` the cost's; `descriptions` can describe the PoIs. The cost column is named `cost_name`. :raises FileNotFoundError: If `output_dir` has no `optimal.nc`, or a slack level has no run summary (its run did not finish; resuming `generate_mga_files` writes it). :raises ValueError: If a PoI has no unit, or a result has no duals (it was solved with `resolve_main_objective=False`). """ nos, _ = collect_near_optimal_set( output_dir, units=units, cost_unit=cost_unit, cost_name=cost_name, descriptions=descriptions, ) return write(directory, nos)
[docs] def collect_near_optimal_set( output_dir: str | Path, *, units: dict[str, str], cost_unit: str, cost_name: str = "cost", descriptions: dict[str, str] | None = None, ) -> tuple[NearOptimalSet, list[Path]]: """The results of a `generate_mga_files` run in `output_dir` as a near-optimal set in memory, and the result file of each point, in point order (``files[p]`` is the solved network behind point ``p``), for reading detailed outputs. Arguments and errors as for :func:`export_near_optimal_set`. """ output_dir = Path(output_dir) descriptions = descriptions or {} optimal_nc = output_dir / "optimal.nc" if not optimal_nc.exists(): msg = f"{optimal_nc} not found -- run generate_mga_files first" raise FileNotFoundError(msg) optimal = _read_meta(optimal_nc) definitions = optimal["dimensions"] keys = list(definitions) if missing := [k for k in keys if k not in units]: msg = f"no unit given for PoIs {missing}" raise ValueError(msg) rows = [(0.0, "optimum", *_row(optimal, keys, "coordinates"))] files = [optimal_nc] slack_levels = [] for slack, slack_dir in _slack_dirs(output_dir): summary_path = slack_dir / RUN_SUMMARY_NAME if not summary_path.exists(): msg = f"{summary_path} not found -- the run at slack {slack} did not finish; resume generate_mga_files" raise FileNotFoundError(msg) slack_levels.append(SlackLevel.model_validate(json.loads(summary_path.read_text(encoding="utf-8")))) for path in sorted(slack_dir.glob("mga_furthest_result_*.nc")): rows.append((slack, "box", *_row(_read_meta(path), keys, "coordinates"))) files.append(path) for path in sorted(slack_dir.glob("mga_closest_result_*.nc")): rows.append((slack, "nearest", *_row(_read_meta(path), keys, "point"))) files.append(path) index = pd.Index(range(len(rows)), name="point") points = pd.DataFrame([r[2] for r in rows], columns=keys, index=index) points.insert(0, "key", [_key(path) for path in files]) points.insert(0, "kind", [r[1] for r in rows]) points.insert(0, "slack", [r[0] for r in rows]) points[cost_name] = [r[4] for r in rows] duals = pd.DataFrame([r[3] for r in rows], columns=keys, index=index) if duals.isna().any().any(): missing_points = duals.index[duals.isna().any(axis=1)].tolist() msg = ( f"no duals for points {missing_points[:10]}: their results were solved with " "resolve_main_objective=False, so they lack duals and their cost is not the cheapest " "for their PoI values; solve them again with resolve_main_objective=True" ) raise ValueError(msg) manifest = Manifest( format="mga-near-optimal-set", version=1, created=datetime.now(timezone.utc), software=_software(), cost=Cost(name=cost_name, unit=cost_unit, optimum=float(optimal["obj"])), pois=[ Poi(name=k, unit=units[k], description=descriptions.get(k, ""), definition=definitions[k]) for k in keys ], slack_levels=slack_levels, fingerprint=Fingerprint(network=network_fingerprint(pypsa.Network(optimal_nc))), ) nos = NearOptimalSet(manifest=manifest, points=points, duals=duals) nos.validate() return nos, files