"""Direct GIE AGSI example; no OilPriceAPI storage route is implied.
Documented against GIE API manual v007; live authenticated run required.
"""
import argparse
import csv
import json
import math
import os
import time
from datetime import date, datetime, timedelta, timezone
from pathlib import Path


def numeric(value):
    if value is None or value == "":
        return None
    if isinstance(value, bool):
        raise ValueError("Boolean where a number was expected")
    number = float(value)
    if not math.isfinite(number):
        raise ValueError("Non-finite numeric value")
    return number


def normalize(rows, start, end, retrieved):
    result, seen = [], set()
    for row in rows:
        if str(row.get("code", "")).lower() != "eu":
            raise ValueError("Unexpected aggregate; expected EU")
        day = date.fromisoformat(row["gasDayStart"])
        if not start <= day <= end:
            raise ValueError("Response date outside requested range")
        if day in seen:
            raise ValueError("Duplicate gas day; inspect raw source records")
        seen.add(day)
        inv, cap = numeric(row.get("gasInStorage")), numeric(row.get("workingGasVolume"))
        if inv is not None and inv < 0 or cap is not None and cap <= 0:
            raise ValueError("Invalid inventory or capacity")
        result.append({"gas_day": str(day), "inventory_twh": inv,
                       "capacity_twh": cap, "fill_pct": numeric(row.get("full")),
                       "status": row.get("status") or "unknown",
                       "retrieved_at_utc": retrieved})
    return sorted(result, key=lambda x: x["gas_day"])


def fetch_pages(session, key, start, end, folder, sleep=time.sleep):
    rows, page, last = [], 1, 1
    while page <= last:
        for attempt in range(4):
            response = session.get("https://agsi.gie.eu/api",
                params={"type": "eu", "from": str(start), "to": str(end), "page": page, "size": 300},
                headers={"x-key": key}, timeout=30)
            if response.status_code == 429 or response.status_code >= 500:
                if attempt == 3:
                    response.raise_for_status()
                sleep(60 if response.status_code == 429 else 2 ** attempt)
                continue
            response.raise_for_status()
            break
        payload = response.json()
        if not isinstance(payload, dict) or not isinstance(payload.get("data"), list):
            raise ValueError("Unexpected response shape")
        raw_last = payload.get("last_page")
        if isinstance(raw_last, bool) or not str(raw_last).isdigit():
            raise ValueError("Invalid pagination")
        declared = int(raw_last)
        if not 1 <= declared <= 10000:
            raise ValueError("Pagination outside safe bounds")
        if page > 1 and declared != last:
            raise ValueError("Page count changed during fetch; restart with a fresh output folder")
        last = declared
        (folder / f"page-{page:04d}.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
        if page < last and not payload["data"]:
            raise ValueError("Empty page before end of dataset")
        rows.extend(payload["data"])
        page += 1
        if page <= last:
            sleep(1.2)
    return rows


def main():
    import requests
    parser = argparse.ArgumentParser()
    parser.add_argument("--start", required=True, type=date.fromisoformat)
    parser.add_argument("--end", required=True, type=date.fromisoformat)
    parser.add_argument("--output", required=True, type=Path)
    args = parser.parse_args()
    if args.end < args.start:
        parser.error("end precedes start")
    key = os.environ.get("GIE_API_KEY")
    if not key:
        parser.error("Set GIE_API_KEY in the environment")
    args.output.mkdir(parents=True, exist_ok=False)
    retrieved = datetime.now(timezone.utc).isoformat()
    with requests.Session() as session:
        # Do not automatically retry connection exceptions: rerun into a new directory.
        rows = fetch_pages(session, key, args.start, args.end, args.output)
    normalized = normalize(rows, args.start, args.end, retrieved)
    if not normalized:
        raise ValueError("No observations returned; inspect raw response")
    with (args.output / "storage.csv").open("w", newline="", encoding="utf-8") as output:
        writer = csv.DictWriter(output, fieldnames=list(normalized[0]))
        writer.writeheader()
        writer.writerows(normalized)
    seen = {r["gas_day"] for r in normalized}
    days = [str(args.start + timedelta(days=i)) for i in range((args.end-args.start).days+1)]
    report = {"source": "GIE AGSI", "scope": "EU aggregate", "retrieved_at_utc": retrieved,
              "missing_gas_days": [d for d in days if d not in seen],
              "non_confirmed_days": [r["gas_day"] for r in normalized if r["status"] != "C"],
              "missing_inventory_days": [r["gas_day"] for r in normalized if r["inventory_twh"] is None],
              "rows": len(normalized)}
    (args.output / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
    print(json.dumps(report, indent=2))


if __name__ == "__main__":
    main()