Read OCP

import json
import sys
from collections import defaultdict

STREAMS = ("input_tokens", "cached_input_tokens", "output_tokens")
TERMINAL = {"done", "failed", "rejected", "canceled", "settled_unverified", "lost"}

observed = defaultdict(int)  # tokens seen, per model
unknown = defaultdict(int)   # attempts without a cost record, per model
outcomes = defaultdict(lambda: defaultdict(int))


def cache_writes(cost):
    # the total when given, else the retention buckets, never both
    if cost.get("cache_creation_tokens") is not None:
        return cost["cache_creation_tokens"]
    return (cost.get("cache_creation_5m_tokens") or 0) + (cost.get("cache_creation_1h_tokens") or 0)


for path in sys.argv[1:]:
    with open(path, encoding="utf-8") as f:
        doc = json.load(f)
    if doc.get("ocp") not in ("0.1", "0.2", "0.3"):
        print(f"skip {path}: unknown version {doc.get('ocp')!r}")
        continue
    for a in doc.get("attempts") or []:
        model = a.get("model") or {}
        name = model.get("id") or model.get("raw") or "unknown"  # canonical id when known, else the raw label
        cost = a.get("cost")
        if isinstance(cost, dict):
            # reasoning_tokens are left out: output_tokens already includes them
            observed[name] += sum(cost.get(k) or 0 for k in STREAMS) + cache_writes(cost)
        else:
            unknown[name] += 1  # an absent cost is unknown, not zero
        if a.get("status") in TERMINAL:
            outcome = a.get("outcome") or {}
            tier = outcome.get("evidence") or "asserted"  # missing evidence is the weakest tier
            outcomes[a.get("harness") or "unknown"][(a["status"], tier)] += 1

for name in sorted(set(observed) | set(unknown), key=lambda m: -observed.get(m, 0)):
    print(f"{name:24} {observed.get(name, 0):>12,} tokens observed, {unknown.get(name, 0)} attempts without cost")
for harness, counts in outcomes.items():
    print(harness, dict(counts))