#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Game cache manifest generator / verifier (agent C1).

Builds api/cache_manifest.json: one entry per file of the game cache with its
exact byte size, SHA-256 and CRC-32, plus where the client downloads it from.
The iOS / Android client (native/src/core/GameCache*.cpp) checks the local
cache against it and re-downloads ONLY the files whose size or hash differ.

Layouts (how the files sit on S3, --layout):

  filezip  (default, the one s3mobile.firsts3.ru/gamecache/zip uses today, same
           as Lit Mobile's launcher): every file is its own zip,
           <baseUrl>/<path>.zip, holding a single entry named like the file.
           --probe-remote reads every zip's central directory over HTTP Range
           (a few KB per file), records "zipSize" and checks that the zipped
           file's CRC-32 and size are the ones of the local file.
  files    S3 mirrors the tree 1:1: <baseUrl>/<path>.
  zips     files are inside bigger zips. --zip-map <json> says which:
             { "zips": [ { "url": "https://.../data.zip",      (absolute, or
                                                                 relative to baseUrl)
                           "local": "D:/upload/data.zip",        (optional: read the
                                                                 directory from this
                                                                 copy instead of S3)
                           "prefix": "data/" } ] }               (entry name ->
                                                                 cache path prefix)
           Every file found in a zip gets "archive": {url, entryName,
           compressedSize, method, localHeaderOffset, dataOffset}; the client
           then fetches exactly that entry's bytes with one Range request.

Per-file "url" overrides: --url-overrides <json> {"path": "https://..."}.

Usage:
  python generate_manifest.py C:\\cachevarf --probe-remote
  python generate_manifest.py C:\\cachevarf --layout files --base-url https://host/cache
  python generate_manifest.py --verify C:\\cachevarf [--manifest api/cache_manifest.json]
  python generate_manifest.py --check-remote [--manifest api/cache_manifest.json]
"""

from __future__ import annotations

import argparse
import concurrent.futures as cf
import datetime as dt
import fnmatch
import hashlib
import json
import os
import struct
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import zlib

MANIFEST_FORMAT = 1
DEFAULT_BASE_URL = "https://s3mobile.firsts3.ru/gamecache/zip"
HERE = os.path.dirname(os.path.abspath(__file__))
DEFAULT_OUT = os.path.join(HERE, "cache_manifest.json")
# The copy that ships inside the app (iOS bundle / APK assets -> game dir), used
# by the client only when no manifest server answers and it has never synced.
DEFAULT_BUNDLE_OUT = os.path.join(os.path.dirname(HERE), "gamefiles", "cache_manifest.bundled.json")
# The engine's own overrides that ship inside the app (GXT texts, fonts_r.txd,
# gamecontrollerdb.txt, ...): the app copies them into the game directory
# itself, so the manifest must not describe them - the client would otherwise
# "repair" the override back to the cache's version.
DEFAULT_BUNDLED_DIR = os.path.join(os.path.dirname(HERE), "gamefiles")

# Never part of the cache: the client's own bookkeeping, logs, saves, partial
# downloads. Matched case-insensitively against the posix relative path.
DEFAULT_EXCLUDES = [
    ".cache_*", "*.part", "*.zpart", "*.zip.part", "cache_manifest.bundled.json",
    "gamelog.txt", "log.txt", "crash_log.txt", "logcat.txt",
    "userfiles/*", "save/*",
    "thumbs.db", "desktop.ini", ".ds_store", "*/.ds_store",
]

# Files the game (or the player) writes to. The client only downloads them when
# they are MISSING and never "repairs" them on a hash mismatch - Lit Mobile's
# NOT_CHECK_BY_HASH_FILES, plus our own per-user ini.
DEFAULT_MUTABLE = [
    "gta_sa.set", "gta_vc.set", "GTASAMP10.b", "Adjustable.cfg", "gtasatelem.set",
    "SAMP/settings.ini", "SAMP/game_settings.ini", "SAMP/savedposition.txt",
    "SAMP/server_config.json", "SAMP/music_favorites.json", "SAMP/tablet_position.json",
    "CINFO.BIN", "models/MINFO.BIN",
    "audio/CONFIG/AudioEventHistory.txt",
    "Text/userfiles/*", "*/reVC.ini", "reVC.ini",
    # SA's own config: written by the game, and the S3 copy (3561 bytes) is not
    # the one in C:\cachevarf (3411 bytes) - see api/README.md.
    "gtasa.ini",
]

CHUNK = 1 << 20
_print_lock = threading.Lock()


def log(msg: str) -> None:
    with _print_lock:
        sys.stderr.write(msg + "\n")
        sys.stderr.flush()


def human(n: float) -> str:
    for unit in ("B", "KB", "MB", "GB", "TB"):
        if abs(n) < 1024 or unit == "TB":
            return f"{n:.2f} {unit}" if unit != "B" else f"{int(n)} B"
        n /= 1024.0
    return f"{n:.2f} TB"


def match_any(path: str, patterns: list[str]) -> bool:
    p = path.lower()
    for pat in patterns:
        pat = pat.lower()
        if fnmatch.fnmatchcase(p, pat):
            return True
        # "dir/*" also covers everything deeper.
        if pat.endswith("/*") and p.startswith(pat[:-1]):
            return True
    return False


# --------------------------------------------------------------------------
# hashing
# --------------------------------------------------------------------------

def hash_file(path: str, progress=None) -> tuple[int, str, str]:
    """(size, sha256 hex, crc32 hex) in one read of the file."""
    sha = hashlib.sha256()
    crc = 0
    size = 0
    with open(path, "rb", buffering=0) as f:
        while True:
            buf = f.read(CHUNK)
            if not buf:
                break
            sha.update(buf)
            crc = zlib.crc32(buf, crc)
            size += len(buf)
            if progress:
                progress(len(buf))
    return size, sha.hexdigest(), f"{crc & 0xFFFFFFFF:08x}"


class Progress:
    def __init__(self, total_bytes: int, total_files: int, label: str):
        self.total_bytes = max(1, total_bytes)
        self.total_files = total_files
        self.label = label
        self.bytes = 0
        self.files = 0
        self.t0 = time.time()
        self.last = 0.0
        self.lock = threading.Lock()

    def add_bytes(self, n: int) -> None:
        with self.lock:
            self.bytes += n
            self._maybe_print()

    def file_done(self) -> None:
        with self.lock:
            self.files += 1
            self._maybe_print(force=self.files == self.total_files)

    def _maybe_print(self, force: bool = False) -> None:
        now = time.time()
        if not force and now - self.last < 1.0:
            return
        self.last = now
        el = max(0.001, now - self.t0)
        speed = self.bytes / el
        eta = (self.total_bytes - self.bytes) / speed if speed > 0 else 0
        log(f"[{self.label}] {self.files}/{self.total_files} files  "
            f"{human(self.bytes)} / {human(self.total_bytes)}  "
            f"{human(speed)}/s  eta {int(eta)} s")


# --------------------------------------------------------------------------
# zip central directory (local file or HTTP Range)
# --------------------------------------------------------------------------

EOCD_SIG = b"PK\x05\x06"
EOCD64_LOC_SIG = b"PK\x06\x07"
EOCD64_SIG = b"PK\x06\x06"
CDH_SIG = b"PK\x01\x02"
LFH_SIG = b"PK\x03\x04"


class ZipEntry:
    __slots__ = ("name", "method", "crc32", "csize", "usize", "lho", "flags")

    def __init__(self, name, method, crc32, csize, usize, lho, flags):
        self.name, self.method, self.crc32 = name, method, crc32
        self.csize, self.usize, self.lho, self.flags = csize, usize, lho, flags


def parse_central_directory(tail: bytes, tail_offset: int, total_size: int, read_at) -> list[ZipEntry]:
    """tail = the last bytes of the archive, starting at absolute tail_offset.
    read_at(offset, length) fetches any other range when the directory does
    not fit in the tail."""
    i = tail.rfind(EOCD_SIG)
    if i < 0:
        raise ValueError("no end-of-central-directory record")
    (_, _disk, _cd_disk, _n_disk, n_total, cd_size, cd_off, _clen) = struct.unpack_from("<4sHHHHIIH", tail, i)
    if n_total == 0xFFFF or cd_size == 0xFFFFFFFF or cd_off == 0xFFFFFFFF:
        j = tail.rfind(EOCD64_LOC_SIG, 0, i)
        if j < 0:
            raise ValueError("zip64 locator missing")
        (_, _d, eocd64_off, _nd) = struct.unpack_from("<4sIQI", tail, j)
        if eocd64_off >= tail_offset:
            rec = tail[eocd64_off - tail_offset: eocd64_off - tail_offset + 56]
        else:
            rec = read_at(eocd64_off, 56)
        if rec[:4] != EOCD64_SIG:
            raise ValueError("bad zip64 EOCD")
        (_, _sz, _vm, _vn, _d1, _d2, _n1, n_total, cd_size, cd_off) = struct.unpack_from("<4sQHHIIQQQQ", rec, 0)
    if cd_off >= tail_offset and cd_off + cd_size <= tail_offset + len(tail):
        cd = tail[cd_off - tail_offset: cd_off - tail_offset + cd_size]
    else:
        cd = read_at(cd_off, cd_size)
    out = []
    p = 0
    for _ in range(n_total):
        if cd[p:p + 4] != CDH_SIG:
            raise ValueError("bad central directory header")
        (_, _vm, _vn, flags, method, _t, _d, crc, csize, usize, nlen, elen, clen,
         _dn, _ia, _ea, lho) = struct.unpack_from("<4sHHHHHHIIIHHHHHII", cd, p)
        name = cd[p + 46: p + 46 + nlen].decode("utf-8" if flags & 0x800 else "cp437")
        extra = cd[p + 46 + nlen: p + 46 + nlen + elen]
        if usize == 0xFFFFFFFF or csize == 0xFFFFFFFF or lho == 0xFFFFFFFF:
            q = 0
            while q + 4 <= len(extra):
                hid, hlen = struct.unpack_from("<HH", extra, q)
                if hid == 1:
                    vals = extra[q + 4: q + 4 + hlen]
                    k = 0
                    if usize == 0xFFFFFFFF:
                        usize = struct.unpack_from("<Q", vals, k)[0]; k += 8
                    if csize == 0xFFFFFFFF:
                        csize = struct.unpack_from("<Q", vals, k)[0]; k += 8
                    if lho == 0xFFFFFFFF:
                        lho = struct.unpack_from("<Q", vals, k)[0]; k += 8
                    break
                q += 4 + hlen
        out.append(ZipEntry(name, method, crc, csize, usize, lho, flags))
        p += 46 + nlen + elen + clen
    return out


def http_range(url: str, start: int | None, end: int | None, suffix: int | None = None,
               timeout: float = 30.0, tries: int = 4) -> tuple[int, bytes, int]:
    """(status, body, total_size or -1)."""
    last = None
    for attempt in range(tries):
        req = urllib.request.Request(url)
        if suffix is not None:
            req.add_header("Range", f"bytes=-{suffix}")
        elif start is not None:
            req.add_header("Range", f"bytes={start}-{'' if end is None else end}")
        req.add_header("Accept-Encoding", "identity")
        try:
            with urllib.request.urlopen(req, timeout=timeout) as r:
                body = r.read()
                total = -1
                cr = r.headers.get("Content-Range")
                if cr and "/" in cr:
                    t = cr.rsplit("/", 1)[1]
                    total = int(t) if t.isdigit() else -1
                elif r.status == 200:
                    total = len(body)
                return r.status, body, total
        except urllib.error.HTTPError as e:
            if e.code in (403, 404):
                return e.code, b"", -1
            last = e
        except Exception as e:  # noqa: BLE001 - network hiccup, retry
            last = e
        time.sleep(0.5 * (2 ** attempt))
    raise RuntimeError(f"{url}: {last}")


def remote_zip_directory(url: str) -> tuple[int, list[ZipEntry]]:
    status, tail, total = http_range(url, None, None, suffix=65557)
    if status not in (200, 206):
        raise FileNotFoundError(f"HTTP {status}")
    if total < 0:
        total = len(tail)
    tail_off = total - len(tail)

    def read_at(off, n):
        st, body, _ = http_range(url, off, off + n - 1)
        if st not in (200, 206):
            raise IOError(f"HTTP {st}")
        return body

    return total, parse_central_directory(tail, tail_off, total, read_at)


def local_zip_directory(path: str) -> tuple[int, list[ZipEntry]]:
    total = os.path.getsize(path)
    with open(path, "rb") as f:
        n = min(total, 65557)
        f.seek(total - n)
        tail = f.read(n)

        def read_at(off, ln):
            f.seek(off)
            return f.read(ln)

        return total, parse_central_directory(tail, total - n, total, read_at)


def local_header_data_offset(read_at, lho: int) -> int:
    hdr = read_at(lho, 30)
    if hdr[:4] != LFH_SIG:
        raise ValueError("bad local file header")
    nlen, elen = struct.unpack_from("<HH", hdr, 26)
    return lho + 30 + nlen + elen


def join_url(base: str, rel: str) -> str:
    return base.rstrip("/") + "/" + urllib.parse.quote(rel, safe="/-_.~")


# --------------------------------------------------------------------------
# generate
# --------------------------------------------------------------------------

def walk(root: str, excludes: list[str]) -> list[str]:
    out = []
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames.sort()
        for fn in sorted(filenames):
            full = os.path.join(dirpath, fn)
            rel = os.path.relpath(full, root).replace(os.sep, "/")
            if match_any(rel, excludes):
                continue
            out.append(rel)
    out.sort(key=lambda s: s.lower())
    return out


def generate(args) -> int:
    root = os.path.abspath(args.root)
    if not os.path.isdir(root):
        log(f"not a directory: {root}")
        return 2
    excludes = DEFAULT_EXCLUDES + (args.exclude or [])
    mutable = DEFAULT_MUTABLE + (args.mutable or [])
    base_url = args.base_url.rstrip("/")

    rels = walk(root, excludes)
    if args.bundled_dir and os.path.isdir(args.bundled_dir):
        owned = {r.lower() for r in walk(args.bundled_dir, excludes)}
        skipped = [r for r in rels if r.lower() in owned]
        if skipped:
            log(f"{len(skipped)} file(s) ship inside the app ({args.bundled_dir}) and are left out: "
                + ", ".join(skipped))
        rels = [r for r in rels if r.lower() not in owned]
    sizes = {r: os.path.getsize(os.path.join(root, r)) for r in rels}
    total = sum(sizes.values())
    log(f"{len(rels)} files, {human(total)} under {root}")

    # Case-insensitive duplicates break on Android's FUSE storage and on the
    # iPhone's case-sensitive one differently - refuse them outright.
    seen = {}
    for r in rels:
        k = r.lower()
        if k in seen:
            log(f"ERROR: '{r}' and '{seen[k]}' differ only in case")
            return 2
        seen[k] = r

    prog = Progress(total, len(rels), "hash")
    results: dict[str, tuple[int, str, str]] = {}

    def job(rel):
        res = hash_file(os.path.join(root, rel), prog.add_bytes)
        prog.file_done()
        return rel, res

    with cf.ThreadPoolExecutor(max_workers=args.jobs) as ex:
        for rel, res in ex.map(job, rels):
            results[rel] = res

    overrides = {}
    if args.url_overrides:
        with open(args.url_overrides, "r", encoding="utf-8") as f:
            overrides = json.load(f)

    entries = []
    for rel in rels:
        size, sha, crc = results[rel]
        e = {"path": rel, "size": size, "sha256": sha, "crc32": crc}
        if match_any(rel, mutable):
            e["mutable"] = True
        if rel in overrides:
            e["url"] = overrides[rel]
        entries.append(e)
    by_path = {e["path"]: e for e in entries}

    problems = []
    if args.layout == "zips":
        if not args.zip_map:
            log("--layout zips needs --zip-map")
            return 2
        problems += apply_zip_map(args.zip_map, base_url, by_path)
        missing = [e["path"] for e in entries if "archive" not in e and "url" not in e]
        if missing:
            problems.append(f"{len(missing)} files are in no zip of the map, e.g. {missing[:5]}")
    elif args.layout == "filezip" and args.probe_remote:
        problems += probe_filezip(entries, base_url, args.jobs_net, args.remote_wins)
    elif args.layout == "files" and args.probe_remote:
        problems += probe_files(entries, base_url, args.jobs_net)

    version = args.version or int(dt.datetime.utcnow().strftime("%Y%m%d%H%M"))
    total = sum(e["size"] for e in entries)  # --remote-wins may have changed some
    download_total = 0
    for e in entries:
        if "archive" in e:
            download_total += e["archive"]["compressedSize"]
        elif "zipSize" in e:
            download_total += e["zipSize"]
        else:
            download_total += e["size"]
    manifest = {
        "format": MANIFEST_FORMAT,
        "name": args.name,
        "version": version,
        "generatedAt": dt.datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
        "baseUrl": base_url,
        "layout": args.layout,
        "totalSize": total,
        "allFilesSize": total,  # Lit Mobile's GameFileInfo name for the same number
        "totalDownloadSize": download_total,
        "fileCount": len(entries),
        "files": entries,
    }
    out = os.path.abspath(args.out)
    tmp = out + ".tmp"
    with open(tmp, "w", encoding="utf-8", newline="\n") as f:
        json.dump(manifest, f, ensure_ascii=False, separators=(",", ":"), indent=None)
        f.write("\n")
    os.replace(tmp, out)
    with open(out, "rb") as f:
        digest = hashlib.sha256(f.read()).hexdigest()
    with open(os.path.splitext(out)[0] + ".sha256", "w", encoding="ascii", newline="\n") as f:
        f.write(f"{digest}  {os.path.basename(out)}\n")
    if args.bundle_out and os.path.isdir(os.path.dirname(os.path.abspath(args.bundle_out))):
        import shutil
        shutil.copyfile(out, args.bundle_out)
        log(f"copied to {args.bundle_out} (ships inside the app)")
    log(f"wrote {out}: version {version}, {len(entries)} files, {human(total)} "
        f"(download {human(download_total)}), sha256 {digest}")
    if problems:
        log(f"\n{len(problems)} PROBLEM(S) with the remote copy:")
        for p in problems[:200]:
            log("  " + p)
        if len(problems) > 200:
            log(f"  ... and {len(problems) - 200} more")
        return 0 if args.allow_remote_mismatch else 1
    return 0


def remote_file_hash(url: str, entry_name: str) -> tuple[int, str, str]:
    """Downloads a whole per-file zip and hashes the file inside it."""
    import io
    import zipfile
    st, body, _ = http_range(url, None, None, timeout=300)
    if st != 200:
        raise IOError(f"HTTP {st}")
    with zipfile.ZipFile(io.BytesIO(body)) as zf:
        names = [n for n in zf.namelist() if not n.endswith("/")]
        name = entry_name if entry_name in names else names[0]
        data = zf.read(name)
    return len(data), hashlib.sha256(data).hexdigest(), f"{zlib.crc32(data) & 0xFFFFFFFF:08x}"


def probe_filezip(entries, base_url, jobs, remote_wins=False) -> list[str]:
    """Reads the central directory of <base>/<path>.zip for every entry and
    checks it holds this exact file (CRC-32 + size). With remote_wins a file
    that differs is described as S3 has it (the client downloads from S3, so
    the manifest must describe S3's bytes)."""
    prog = Progress(len(entries), len(entries), "probe")
    problems = []
    lock = threading.Lock()

    def job(e):
        url = e.get("url") or join_url(base_url, e["path"] + ".zip")
        try:
            zsize, ents = remote_zip_directory(url)
        except FileNotFoundError as ex:
            with lock:
                problems.append(f"{e['path']}: {url} -> {ex}")
            return
        except Exception as ex:  # noqa: BLE001
            with lock:
                problems.append(f"{e['path']}: {url} -> {ex}")
            return
        finally:
            prog.add_bytes(1)
            prog.file_done()
        files = [z for z in ents if not z.name.endswith("/")]
        want = e["path"].rsplit("/", 1)[-1]
        z = next((z for z in files if z.name == want), None) or \
            next((z for z in files if z.name.rsplit("/", 1)[-1].lower() == want.lower()), None) or \
            (files[0] if len(files) == 1 else None)
        if z is None:
            with lock:
                problems.append(f"{e['path']}: no entry '{want}' in {url}")
            return
        e["zipSize"] = zsize
        if z.name != want:
            e["zipEntry"] = z.name
        if z.usize != e["size"] or f"{z.crc32:08x}" != e["crc32"]:
            msg = (f"{e['path']}: S3 has size {z.usize} crc32 {z.crc32:08x}, "
                   f"local cache has size {e['size']} crc32 {e['crc32']}")
            if remote_wins:
                try:
                    size, sha, crc = remote_file_hash(url, z.name)
                except Exception as ex:  # noqa: BLE001
                    with lock:
                        problems.append(f"{msg}; could not fetch it: {ex}")
                    return
                e["size"], e["sha256"], e["crc32"] = size, sha, crc
                log(f"WARNING {msg} -> manifest describes the S3 copy (--remote-wins)")
            else:
                with lock:
                    problems.append(msg)

    with cf.ThreadPoolExecutor(max_workers=jobs) as ex:
        list(ex.map(job, entries))
    return problems


def probe_files(entries, base_url, jobs) -> list[str]:
    problems = []
    lock = threading.Lock()

    def job(e):
        url = e.get("url") or join_url(base_url, e["path"])
        try:
            st, _, total = http_range(url, 0, 0)
        except Exception as ex:  # noqa: BLE001
            st, total = str(ex), -1
        if st not in (200, 206) or total != e["size"]:
            with lock:
                problems.append(f"{e['path']}: {url} -> status {st}, size {total} (want {e['size']})")

    with cf.ThreadPoolExecutor(max_workers=jobs) as ex:
        list(ex.map(job, entries))
    return problems


def apply_zip_map(map_path, base_url, by_path) -> list[str]:
    with open(map_path, "r", encoding="utf-8") as f:
        zmap = json.load(f)
    problems = []
    lower = {k.lower(): v for k, v in by_path.items()}
    for z in zmap.get("zips", []):
        url = z["url"] if "://" in z["url"] else join_url(base_url, z["url"])
        prefix = z.get("prefix", "")
        if z.get("local"):
            _, ents = local_zip_directory(z["local"])
            fh = open(z["local"], "rb")

            def read_at(off, n, fh=fh):
                fh.seek(off)
                return fh.read(n)
        else:
            _, ents = remote_zip_directory(url)
            fh = None

            def read_at(off, n, url=url):
                st, body, _ = http_range(url, off, off + n - 1)
                if st not in (200, 206):
                    raise IOError(f"HTTP {st}")
                return body
        for ze in ents:
            if ze.name.endswith("/"):
                continue
            path = prefix + ze.name
            e = by_path.get(path) or lower.get(path.lower())
            if e is None:
                continue
            if ze.method not in (0, 8):
                problems.append(f"{path}: zip method {ze.method} (only store/deflate)")
                continue
            if ze.usize != e["size"] or f"{ze.crc32:08x}" != e["crc32"]:
                problems.append(f"{path}: {url} has size {ze.usize} crc32 {ze.crc32:08x}, "
                                f"local {e['size']} {e['crc32']}")
            e["archive"] = {
                "url": url,
                "entryName": ze.name,
                "compressedSize": ze.csize,
                "method": ze.method,
                "localHeaderOffset": ze.lho,
                "dataOffset": local_header_data_offset(read_at, ze.lho),
            }
        if fh:
            fh.close()
    return problems


# --------------------------------------------------------------------------
# verify (same rules as the client)
# --------------------------------------------------------------------------

def load_manifest(path: str) -> dict:
    with open(path, "r", encoding="utf-8") as f:
        m = json.load(f)
    if m.get("format") != MANIFEST_FORMAT or not isinstance(m.get("files"), list):
        raise ValueError(f"{path}: not a format-{MANIFEST_FORMAT} manifest")
    return m


def verify(args) -> int:
    m = load_manifest(args.manifest)
    root = os.path.abspath(args.verify)
    files = m["files"]
    total = sum(e["size"] for e in files)
    prog = Progress(total, len(files), "verify")
    bad: list[str] = []
    lock = threading.Lock()

    def job(e):
        full = os.path.join(root, *e["path"].split("/"))
        try:
            st = os.stat(full)
        except FileNotFoundError:
            with lock:
                bad.append(f"MISSING  {e['path']}")
            prog.file_done()
            return
        if e.get("mutable"):
            prog.add_bytes(e["size"])
            prog.file_done()
            return
        if st.st_size != e["size"]:
            with lock:
                bad.append(f"SIZE     {e['path']}  {st.st_size} != {e['size']}")
            prog.add_bytes(e["size"])
            prog.file_done()
            return
        size, sha, crc = hash_file(full, prog.add_bytes)
        prog.file_done()
        if crc != e["crc32"]:
            with lock:
                bad.append(f"CRC32    {e['path']}  {crc} != {e['crc32']}")
        elif sha != e["sha256"]:
            with lock:
                bad.append(f"SHA256   {e['path']}")

    with cf.ThreadPoolExecutor(max_workers=args.jobs) as ex:
        list(ex.map(job, files))

    known = {e["path"].lower() for e in files}
    extra = [r for r in walk(root, DEFAULT_EXCLUDES) if r.lower() not in known]
    if extra:
        log(f"{len(extra)} file(s) not in the manifest (left alone, like the client does):")
        for r in extra[:50]:
            log(f"  EXTRA    {r}")
    if bad:
        log(f"\nFAILED: {len(bad)} of {len(files)} files would be re-downloaded:")
        for b in sorted(bad):
            log("  " + b)
        return 1
    log(f"\nOK: all {len(files)} files match manifest version {m.get('version')} "
        f"({human(total)})")
    return 0


def check_remote(args) -> int:
    m = load_manifest(args.manifest)
    entries = m["files"]
    base = m.get("baseUrl", DEFAULT_BASE_URL)
    layout = m.get("layout", "filezip")
    if layout == "filezip":
        before = {e["path"]: e.get("zipSize") for e in entries}
        problems = probe_filezip(entries, base, args.jobs_net)
        changed = [e["path"] for e in entries if before[e["path"]] not in (None, e.get("zipSize"))]
        if changed:
            problems.append(f"{len(changed)} zip(s) changed size since the manifest was made, e.g. {changed[:5]}")
    elif layout == "files":
        problems = probe_files(entries, base, args.jobs_net)
    else:
        problems = []
        for e in entries:
            a = e.get("archive")
            if not a:
                continue
            st, body, _ = http_range(a["url"], a["dataOffset"], a["dataOffset"] + min(a["compressedSize"], 64) - 1)
            if st not in (200, 206):
                problems.append(f"{e['path']}: {a['url']} -> HTTP {st}")
    if problems:
        for p in problems:
            log("  " + p)
        log(f"FAILED: {len(problems)} problem(s)")
        return 1
    log(f"OK: all {len(entries)} remote objects match")
    return 0


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("root", nargs="?", help="game cache directory to describe")
    ap.add_argument("--out", default=DEFAULT_OUT)
    ap.add_argument("--layout", choices=["filezip", "files", "zips"], default="filezip")
    ap.add_argument("--base-url", default=DEFAULT_BASE_URL)
    ap.add_argument("--zip-map", help="json for --layout zips")
    ap.add_argument("--url-overrides", help='json {"path": "absolute url"}')
    ap.add_argument("--exclude", action="append", help="extra glob to leave out (repeatable)")
    ap.add_argument("--mutable", action="append", help="extra glob: download if missing, never re-check")
    ap.add_argument("--version", type=int, help="manifest version (default UTC yyyymmddHHMM)")
    ap.add_argument("--name", default="santa-mobile-gamecache")
    ap.add_argument("--probe-remote", action="store_true", help="check every file against S3")
    ap.add_argument("--allow-remote-mismatch", action="store_true")
    ap.add_argument("--remote-wins", action="store_true",
                    help="filezip: describe a file that differs on S3 as S3 has it")
    ap.add_argument("--jobs", type=int, default=max(2, min(8, (os.cpu_count() or 4))))
    ap.add_argument("--jobs-net", type=int, default=16)
    ap.add_argument("--verify", metavar="DIR", help="check DIR against --manifest")
    ap.add_argument("--check-remote", action="store_true", help="check S3 against --manifest")
    ap.add_argument("--manifest", default=DEFAULT_OUT)
    ap.add_argument("--bundle-out", default=DEFAULT_BUNDLE_OUT,
                    help="also copy the manifest here (default: gamefiles/, bundled into the app)")
    ap.add_argument("--no-bundle", dest="bundle_out", action="store_const", const=None)
    ap.add_argument("--bundled-dir", default=DEFAULT_BUNDLED_DIR,
                    help="files the app ships itself (default: gamefiles/); left out of the manifest")
    ap.add_argument("--no-exclude-bundled", dest="bundled_dir", action="store_const", const=None)
    args = ap.parse_args()
    if args.verify:
        return verify(args)
    if args.check_remote:
        return check_remote(args)
    if not args.root:
        ap.error("root directory required")
    return generate(args)


if __name__ == "__main__":
    sys.exit(main())