"""Download the CAA's monthly UK flight punctuality statistics.

    uv run caa.py              # fetch new or republished months since January 2018
    uv run caa.py --refresh    # fetch every month again

The CAA publishes each month's statistics on a page for its year. The download
links contain a file number that cannot be derived from the month, so each year's
page is read for its links. Two files are kept for each month in data/raw/:

    YYYYMM.csv   "Punctuality Statistics Full Analysis Arrival Departure": one row
                 for each airport, origin or destination, airline, direction
                 (arrival or departure) and type of service (scheduled or charter);
    YYYYMM.xlsx  "On Time Performance Tables" (from October 2024 only): the CAA's
                 own totals for each airport and for its 50 largest airlines, used
                 to check ours.

data/raw/manifest.json records each file's link. A month is fetched again when
its link changes, which is how a republished (revised) month shows itself.
"""

import argparse
import datetime
import hashlib
import html
import json
import os
import re
import sys
import time
import urllib.request
from pathlib import Path

BASE_URL = "https://www.caa.co.uk"
YEAR_URL = (
    BASE_URL + "/data-and-analysis/uk-aviation-market/flight-punctuality/"
    "uk-flight-punctuality-statistics/{year}/"
)
FIRST_YEAR = 2018  # the first year with today's columns (see build_data.py)
USER_AGENT = "scipython-flight-punctuality/0.1"
RAW_DIR = Path(__file__).parent / "data" / "raw"
MANIFEST = RAW_DIR / "manifest.json"

LINK = re.compile(r'<a[^>]+href="(?P<href>[^"]*/Documents/Download/[^"]+)"[^>]*>(?P<text>.*?)</a>', re.S)
# Monthly files only: the annual files begin with a four-digit year.
KINDS = {
    "csv": re.compile(r"^(?P<period>\d{6}) Punctuality Statistics Full Analysis Arrival Departure \(CSV"),
    "xlsx": re.compile(r"^(?P<period>\d{6}) On Time Performance Tables \(XLSX"),
}  # fmt: skip
# The columns that build_data.py relies on, checked in every file fetched.
EXPECTED_COLUMNS = (
    "run_date,reporting_period,reporting_airport,origin_destination_country,"
    "origin_destination,airline_name,arrival_departure,scheduled_charter,"
    "number_flights_matched,actual_flights_unmatched,number_flights_cancelled,"
    "flights_more_than_15_minutes_early_percent,"
    "flights_15_minutes_early_to_1_minute_early_percent,"
    "flights_0_to_15_minutes_late_percent"
)


def get(url: str) -> bytes:
    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    with urllib.request.urlopen(request, timeout=60) as response:
        return response.read()


def list_files(year: int) -> dict[tuple[str, str], str]:
    """Map (period, kind) to the download URL of each monthly file for year."""
    page = get(YEAR_URL.format(year=year)).decode("utf-8")
    files = {}
    for match in LINK.finditer(page):
        text = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", html.unescape(match["text"]))).strip()
        for kind, pattern in KINDS.items():
            if m := pattern.match(text):
                files[m["period"], kind] = BASE_URL + html.unescape(match["href"])
    return files


def decode(content: bytes) -> str:
    """Decode a CAA CSV file: UTF-8 with a byte-order mark, but a few months are
    in Windows-1252 (with non-breaking spaces in some names)."""
    try:
        return content.decode("utf-8-sig")
    except UnicodeDecodeError:
        return content.decode("cp1252")


def check_csv(content: bytes, period: str) -> None:
    text = decode(content)
    header, _, rest = text.partition("\n")
    if not header.strip().startswith(EXPECTED_COLUMNS):
        raise ValueError(f"{period}: unexpected columns: {header[:200]}")
    first = rest.split("\n", 1)[0].split(",")
    if first[1] != period:
        raise ValueError(f"{period}: the file holds period {first[1]}")


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    parser.add_argument("--refresh", action="store_true", help="fetch every month again")
    args = parser.parse_args()

    RAW_DIR.mkdir(parents=True, exist_ok=True)
    manifest = json.loads(MANIFEST.read_text()) if MANIFEST.exists() else {}
    fetched = []
    for year in range(FIRST_YEAR, datetime.date.today().year + 1):
        for (period, kind), url in sorted(list_files(year).items()):
            key = f"{period}.{kind}"
            path = RAW_DIR / key
            if not args.refresh and path.exists() and manifest.get(key, {}).get("url") == url:
                continue
            content = get(url)
            if kind == "csv":
                check_csv(content, period)
            path.write_bytes(content)
            manifest[key] = {
                "url": url,
                "sha256": hashlib.sha256(content).hexdigest(),
                "fetched": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"),
            }
            MANIFEST.write_text(json.dumps(dict(sorted(manifest.items())), indent=1) + "\n")
            fetched.append(key)
            print(f"fetched {key} ({len(content) / 1e6:.1f} MB)", file=sys.stderr)
            time.sleep(0.5)  # be gentle with the CAA's server
    MANIFEST.write_text(json.dumps(dict(sorted(manifest.items())), indent=1) + "\n")
    months = sorted(k[:6] for k in manifest if k.endswith(".csv"))
    print(f"{len(fetched)} files fetched; {len(months)} months held, {months[0]} to {months[-1]}", file=sys.stderr)
    # In GitHub Actions, tell the workflow whether anything is new.
    if output := os.environ.get("GITHUB_OUTPUT"):
        with open(output, "a") as f:
            f.write(f"fetched={len(fetched)}\nlatest={months[-1]}\n")


if __name__ == "__main__":
    main()
