#!/usr/bin/env python3
"""
fetch_players.py  --  Pull the latest NFL rosters (name, team, position, headshot)
from ESPN's free public API, enrich them with real fantasy ADP and team logos,
add one draftable D/ST per team, and write players.json next to draftboard.html.

Run this on draft day (double-click 1_fetch_players.bat on Windows, or `python fetch_players.py`).
No API key required.

Data sources (all free, no key):
  * ESPN rosters + team logos ....... site.api.espn.com
  * Fantasy ADP ..................... fantasyfootballcalculator.com public API

Only offensive/kicking skill positions relevant to fantasy are kept by default, plus a
D/ST for every team. Flip KEEP_ALL = True to include everyone (IDP leagues, etc.).
"""
import concurrent.futures, json, os, random, re, sys, time, urllib.error, urllib.request

KEEP_ALL = False  # set True to include defense/OL/etc.
FETCH_NEWS = True  # pull a short recent-news blurb for ranked players (threaded; set False to skip)
VALIDATE_HEADSHOTS = True  # HEAD-check headshots and blank the missing ones (kills console 404s; threaded)
FANTASY_POS = {"QB", "RB", "WR", "TE", "K"}
# ESPN's abbreviations don't all match standard fantasy labels. Normalize before
# filtering/storing so the board's position chips (QB/RB/WR/TE/K) and colors match.
#   PK -> K  : ESPN calls place-kickers "PK"; the board expects "K".
#   FB -> RB : the league has no fullback slot, so cluster fullbacks with running backs.
POS_MAP = {"PK": "K", "FB": "RB"}
TIMEOUT = 20

TEAMS_URL = "https://site.api.espn.com/apis/site/v2/sports/football/nfl/teams"
ROSTER_URL = "https://site.api.espn.com/apis/site/v2/sports/football/nfl/teams/{tid}/roster"
HEADSHOT = "https://a.espncdn.com/i/headshots/nfl/players/full/{pid}.png"
LOGO = "https://a.espncdn.com/i/teamlogos/nfl/500/{abbr}.png"

# Fantasy Football Calculator public ADP API. Change "ppr" to "half-ppr" or "standard"
# to match your league's scoring; "year" should be the upcoming season.
ADP_URL = "https://fantasyfootballcalculator.com/api/v1/adp/ppr?teams=12&year=2026"
ADP_UNRANKED = 999.9  # players with no ADP sort after every ranked player
NEWS_URL = "https://site.web.api.espn.com/apis/common/v3/sports/football/nfl/athletes/{pid}/overview"

# FFC uses a few team codes that differ from ESPN's abbreviations.
TEAM_FIX = {"WAS": "WSH", "JAC": "JAX", "LA": "LAR", "OAK": "LV", "SD": "LAC", "STL": "LAR"}


# ESPN's edge keeps changing which User-Agents it accepts: it 403'd "DraftBoard/1.0", then
# started 403'ing browser-style "Mozilla/5.0" too, while urllib's own default sails through.
# Don't hard-code one -- try a list of header profiles, lock onto whichever works, and re-probe
# the others automatically if that one starts getting refused. Self-healing beats guessing.
UA_PROFILES = [
    {},                                                   # urllib default (works 2026-08)
    {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                   "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"},
    {"User-Agent": "Mozilla/5.0"},
    {"User-Agent": "DraftBoard/1.0"},
]
_profile = 0          # index of the last profile that worked


def _open(url, headers=None, timeout=TIMEOUT, method=None, rounds=3):
    """urlopen that survives BOTH of ESPN's defences:
       * shifting User-Agent blocking -> rotate header profiles, lock onto what works
       * rolling rate limits on sustained volume -> back off and retry the whole set
    """
    global _profile
    last = None
    for rnd in range(rounds):
        order = sorted(range(len(UA_PROFILES)), key=lambda i: 0 if i == _profile else 1)
        for i in order:
            h = dict(UA_PROFILES[i])
            h.update(headers or {})
            try:
                req = urllib.request.Request(url, headers=h, method=method)
                r = urllib.request.urlopen(req, timeout=timeout)
                _profile = i
                return r
            except urllib.error.HTTPError as e:
                last = e
                if e.code in (403, 429):
                    continue          # blocked or throttled -> next profile
                raise
        if rnd < rounds - 1:
            time.sleep(1.0 + 2.0 * rnd + random.random())   # throttled: wait, then try again
    raise last


def get(url):
    with _open(url) as r:
        return json.load(r)


def norm_name(s):
    """Normalize a player name so it matches across ESPN and the ADP source."""
    s = (s or "").lower().replace(".", "").replace("'", "").replace("-", " ")
    s = re.sub(r"\b(jr|sr|ii|iii|iv|v)\b", "", s)   # drop generational suffixes
    s = re.sub(r"[^a-z ]", "", s)
    return " ".join(s.split())


def fetch_adp():
    """Return (by_name, by_def, by_bye): name->adp, team->defense adp, team->bye week."""
    by_name, by_def, by_bye = {}, {}, {}
    try:
        data = get(ADP_URL)
    except Exception as e:
        print(f"  !! ADP fetch failed ({e}); players will load unranked")
        return by_name, by_def, by_bye
    for p in data.get("players", []):
        tm = TEAM_FIX.get(p.get("team", ""), p.get("team", ""))
        if p.get("bye") and tm:
            by_bye[tm] = p.get("bye")           # bye weeks are per-team
        adp = p.get("adp")
        if adp is None:
            continue
        if p.get("position") == "DEF":
            by_def[tm] = adp
        else:
            key = norm_name(p.get("name"))
            if key and (key not in by_name or adp < by_name[key]):
                by_name[key] = adp   # keep the best (earliest) ADP on name collisions
    print(f"  ADP loaded: {len(by_name)} players + {len(by_def)} defenses + {len(by_bye)} byes")
    return by_name, by_def, by_bye


def player_news(pid):
    """Return {'news': headline, 'newsDate': 'YYYY-MM-DD'} for one athlete, or None."""
    try:
        with _open(NEWS_URL.format(pid=pid), timeout=12) as r:
            d = json.load(r)
    except Exception:
        return None
    rw = d.get("rotowire") or {}
    head, date = rw.get("headline"), rw.get("published")   # rotowire = fantasy-relevant notes
    if not head:
        arr = d.get("news") or []
        if arr:
            head, date = arr[0].get("headline"), arr[0].get("published")
    if not head:
        return None
    head = " ".join(head.split())
    if len(head) > 180:
        head = head[:177].rstrip() + "…"
    return {"news": head, "newsDate": (date or "")[:10]}


def add_news(players):
    """Attach recent news to ranked, real-athlete players using a small thread pool."""
    targets = [p for p in players if p["adp"] < ADP_UNRANKED and str(p["id"]).isdigit()]
    print(f"Fetching news for {len(targets)} ranked players ...")
    got = 0
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as ex:
        futs = {ex.submit(player_news, p["id"]): p for p in targets}
        for fut in concurrent.futures.as_completed(futs):
            res = fut.result()
            if res:
                futs[fut]["news"], futs[fut]["newsDate"] = res["news"], res["newsDate"]
                got += 1
    miss = len(targets) - got
    print(f"  got news for {got} players" + (f"  (!! {miss} failed — ESPN throttling; re-run to fill in)" if miss else ""))


def headshot_ok(url):
    """True if the headshot exists; only a definite 404 counts as missing."""
    try:
        with _open(url, timeout=8, method="HEAD") as r:
            return getattr(r, "status", 200) == 200
    except urllib.error.HTTPError as e:
        return e.code != 404
    except Exception:
        return True  # network hiccup — keep the image rather than wrongly blanking it


def validate_headshots(players):
    """Blank headshots that 404 so the board doesn't spew console errors / broken-image flicker."""
    targets = [p for p in players if str(p["id"]).isdigit() and p.get("img")]
    print(f"Validating {len(targets)} headshots ...")
    removed, blanked = 0, set()
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
        futs = {ex.submit(headshot_ok, p["img"]): p for p in targets}
        for fut in concurrent.futures.as_completed(futs):
            if not fut.result():
                futs[fut]["img"] = ""
                blanked.add(str(futs[fut]["id"]))     # a definite 404 -- do NOT restore this one
                removed += 1
    print(f"  blanked {removed} missing headshots")
    return blanked


def load_prior():
    """Previous players.json keyed by id, so a THROTTLED re-run never loses data it already had."""
    try:
        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "players.json"),
                  encoding="utf-8") as f:
            return {str(p["id"]): p for p in json.load(f)["players"]}
    except Exception:
        return {}


def merge_prior(players, prior, blanked=(), failed_teams=()):
    """NEVER lose data. Anything this run failed to pull (ESPN throttles sustained volume, or a
    whole team's roster request can fail) falls back to what we already had, so re-running only
    ever ADDS. The one exception: headshots proven missing by a 404 THIS run stay blank."""
    if not prior:
        return players
    CARRY = ("news", "newsDate", "logo", "bye", "inj", "injDetail", "injRet",
             "age", "exp", "jersey", "teamName", "team", "name")
    filled = {k: 0 for k in CARRY}
    img_back = adp_back = 0
    have = set()
    for p in players:
        have.add(str(p["id"]))
        old = prior.get(str(p["id"]))
        if not old:
            continue
        for k in CARRY:
            if not p.get(k) and old.get(k):
                p[k] = old[k]
                filled[k] += 1
        # headshot: restore unless we proved it's a 404 on this run
        if not p.get("img") and old.get("img") and str(p["id"]) not in blanked:
            p["img"] = old["img"]
            img_back += 1
        # ADP is the big one: if the ADP feed was down, every player would drop to unranked
        if p.get("adp", ADP_UNRANKED) >= ADP_UNRANKED and old.get("adp", ADP_UNRANKED) < ADP_UNRANKED:
            p["adp"] = old["adp"]
            adp_back += 1
    # a team whose roster request failed would silently vanish -- bring its players back
    readded = 0
    for pid, old in prior.items():
        if pid in have:
            continue
        if old.get("team") in failed_teams:
            players.append(dict(old))
            readded += 1
    bits = [f"{v} {k}" for k, v in filled.items() if v]
    if img_back:
        bits.append(f"{img_back} headshots")
    if adp_back:
        bits.append(f"{adp_back} ADP values")
    if readded:
        bits.append(f"{readded} players from failed rosters")
    print("  carried forward from the last run: " + (", ".join(bits) if bits else "nothing needed"))
    return players


def main():
    prior = load_prior()
    print("Fetching team list from ESPN ...")
    tdata = get(TEAMS_URL)
    teams = tdata["sports"][0]["leagues"][0]["teams"]
    print(f"  found {len(teams)} teams")

    print("Fetching fantasy ADP ...")
    adp_by_name, adp_by_def, bye_by_team = fetch_adp()

    players = []
    seen = set()
    failed_teams = set()
    team_meta = []  # (abbr, tname, logo) for building D/ST entries afterward
    for t in teams:
        team = t["team"]
        tid = team["id"]
        abbr = team.get("abbreviation", "")
        tname = team.get("displayName", "")
        logo = (team.get("logos") or [{}])[0].get("href") or LOGO.format(abbr=abbr.lower())
        team_meta.append((abbr, tname, logo))
        try:
            rdata = get(ROSTER_URL.format(tid=tid))
        except Exception as e:
            print(f"  !! {tname}: {e}  (will reuse last run's players for this team)")
            failed_teams.add(abbr)
            continue
        cnt = 0
        for grp in rdata.get("athletes", []):
            # skip practice squad / IR / suspended so unavailable guys don't clutter the board
            if grp.get("position") in ("injuredReserveOrOut", "suspended", "practiceSquad"):
                continue
            for a in grp.get("items", []):
                pid = a.get("id")
                raw_pos = (a.get("position") or {}).get("abbreviation", "")
                pos = POS_MAP.get(raw_pos, raw_pos)
                if not KEEP_ALL and pos not in FANTASY_POS:
                    continue
                if pid in seen:
                    continue
                seen.add(pid)
                name = a.get("fullName") or a.get("displayName", "")
                # Injury flag for the draft guide: prefer the injuries[] status ("Questionable",
                # "Out", ...); fall back to a non-Active roster status ("Injured Reserve", ...).
                inj, inj_detail, inj_ret = "", "", ""
                for j in (a.get("injuries") or []):
                    st = (j.get("status") or "").strip()
                    if st:
                        inj = st
                        det = j.get("details") or {}
                        bits = [det.get("side"), det.get("type"), det.get("detail")]
                        inj_detail = " ".join(str(b) for b in bits if b and str(b).lower() != "not specified")
                        inj_ret = (det.get("returnDate") or "")
                        break
                if not inj:
                    st = ((a.get("status") or {}).get("name") or "").strip()
                    if st and st.lower() != "active":
                        inj = st
                players.append({
                    "id": pid,
                    "name": name,
                    "team": abbr,
                    "teamName": tname,
                    "pos": pos,
                    "jersey": a.get("jersey", ""),
                    "img": HEADSHOT.format(pid=pid),
                    "logo": logo,
                    "adp": adp_by_name.get(norm_name(name), ADP_UNRANKED),
                    "bye": bye_by_team.get(abbr, ""),
                    "inj": inj,
                    "injDetail": inj_detail,
                    "injRet": inj_ret,
                    "age": a.get("age") or "",
                    "exp": (a.get("experience") or {}).get("years", ""),
                })
                cnt += 1
        print(f"  {abbr:>3} {tname:<26} {cnt} players")
        time.sleep(0.15)  # be polite to ESPN

    # Add one draftable D/ST per team (defenses aren't in ESPN's roster feed as units).
    for abbr, tname, logo in team_meta:
        players.append({
            "id": "DST_" + abbr,
            "name": tname + " D/ST",
            "team": abbr,
            "teamName": tname,
            "pos": "DST",
            "jersey": "",
            "img": logo,
            "logo": logo,
            "adp": adp_by_def.get(abbr, ADP_UNRANKED),
            "bye": bye_by_team.get(abbr, ""),
            "inj": "",
        })

    if FETCH_NEWS:
        add_news(players)
    blanked = validate_headshots(players) if VALIDATE_HEADSHOTS else set()
    players = merge_prior(players, prior, blanked, failed_teams)

    # Primary sort by ADP so the board opens ranked; unranked fall through to pos+name.
    players.sort(key=lambda p: (p["adp"], p["pos"], p["name"]))
    ranked = sum(1 for p in players if p["adp"] < ADP_UNRANKED)
    out = {
        "generated": time.strftime("%Y-%m-%d %H:%M:%S"),
        "season": "2026-2027",
        "count": len(players),
        "adpRanked": ranked,
        "players": players,
    }
    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "players.json")
    with open(path, "w", encoding="utf-8") as f:
        json.dump(out, f, indent=1)
    print(f"\nWrote {len(players)} players ({ranked} with ADP) -> {path}")
    print("Now open draftboard.html in your browser and click 'Load / Refresh Players'.")


if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        print("ERROR:", e)
        sys.exit(1)