Codex provider: decode each rollout once (token-totals seam + fork-prefix fan-out)

.gitignore

@@ -190,9 +190,14 @@ test/test_data/*.html
 local.ps1
 
 
-# Integration test data - exclude generated cache/HTML, keep JSONL files
+# Integration test data - exclude generated cache/HTML/images, keep JSONL files
 test/test_data/real_projects/*/cache/
 test/test_data/real_projects/*/*.html
 test/test_data/real_projects/index.html
+# Referenced-image export writes images/ next to the HTML it references, so a
+# render pointed at this corpus leaves binaries here. Same generated class as
+# the two rules above, which predate that export mode. Use `git add -f` if an
+# image ever needs to be a real fixture.
+test/test_data/real_projects/*/images/
 # SQLite cache the CLI drops into a projects dir when run against test data
 claude-code-log-cache.db

claude_code_log/converter.py

@@ -22,7 +22,10 @@
 import dateparser
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from .cache import CacheManager
+    from .providers.base import ProviderTokenTotals
 
 from .utils import (
     coalesce_trunk_session_id,
@@ -2931,6 +2934,65 @@ def _wholesale_should_render(
     return stale
 
 
+def _sum_provider_token_totals(
+    totals: "Iterable[Optional[ProviderTokenTotals]]",
+) -> dict[str, int]:
+    """Sum per-session cumulative totals into the four index-summary token
+    keys. ``None`` sessions (no ``token_count``) contribute nothing.
+
+    Each :class:`ProviderTokenTotals` is ALREADY a whole-session cumulative
+    figure, so summing across the DISTINCT sessions of a project is correct —
+    what must never be summed is the per-step deltas WITHIN one session, which
+    is why the provider returns one cumulative value per session, not a stream.
+
+    ``total_cache_creation_tokens`` is kept at 0 purely so the emitted dict is
+    shape-identical to the Claude path (the drift pin's contract). Codex has no
+    cache-creation concept and the renderer omits a zero column, so nothing
+    labelled "Cache Creation" ever renders — absent, not "0".
+    """
+    total_input = 0
+    total_output = 0
+    total_cache_read = 0
+    for totals_item in totals:
+        if totals_item is None:
+            continue
+        total_input += totals_item.input_tokens
+        total_output += totals_item.output_tokens
+        total_cache_read += totals_item.cache_read_tokens
+    return {
+        "total_input_tokens": total_input,
+        "total_output_tokens": total_output,
+        "total_cache_creation_tokens": 0,
+        "total_cache_read_tokens": total_cache_read,
+    }
+
+
+# The four token keys a project card carries, in the shape
+# ``_sum_provider_token_totals`` emits. Named once so the provider path and
+# the per-message fallback cannot drift apart.
+_PROJECT_TOKEN_KEYS = (
+    "total_input_tokens",
+    "total_output_tokens",
+    "total_cache_creation_tokens",
+    "total_cache_read_tokens",
+)
+
+
+def _project_token_totals_from_messages(
+    messages: List[TranscriptEntry],
+) -> dict[str, int]:
+    """Per-message project token totals, in the project-card key shape.
+
+    The fallback for a provider with no cumulative ``session_token_totals``
+    seam: its usage lives on the messages, exactly as Claude's does, so the
+    ordinary accumulator is the right source. Kept to the four token keys so
+    it can be spread into the card dict interchangeably with
+    ``_sum_provider_token_totals``.
+    """
+    aggregates = compute_project_aggregates(messages)
+    return {key: int(aggregates.get(key, 0) or 0) for key in _PROJECT_TOKEN_KEYS}
+
+
 def render_provider_wholesale(
     provider_name: str,
     sessions_root: Optional[Path],
@@ -3061,13 +3123,29 @@ def render_provider_wholesale(
         # Phase 1 — load every session in the project fresh. v1 always re-parses
         # rollouts (cache-backed load is a documented deferral); only rendering
         # is skipped when unchanged.
+        # Entries and cumulative token totals come back from ONE provider call:
+        # a provider reading both from the same file (Codex) would otherwise
+        # re-parse it for the totals, which measured +118 rollout decodes and
+        # +478 MB re-parsed over a 34-rollout archive. The base implementation
+        # of the seam is the old call pair, so providers that don't override it
+        # behave exactly as before.
+        #
+        # The totals ride along with the entries rather than being collected
+        # separately, because they must stay subject to the SAME survival test:
+        # a session emptied by --from-date/--to-date contributes no messages and
+        # must likewise contribute no tokens. Hoisting the totals out of this
+        # filter would let a filtered-out session inflate the project totals —
+        # a behaviour change that no decode count would reveal.
         loaded: list[tuple[SessionInfo, list[TranscriptEntry]]] = []
+        loaded_totals: dict[str, Optional[ProviderTokenTotals]] = {}
         for info in group_infos:
-            messages = list(provider.load_session_under(sessions_root, info.session_id))
+            session = provider.load_session_with_totals(sessions_root, info.session_id)
+            messages = session.entries
             if from_date or to_date:
                 messages = filter_messages_by_date(messages, from_date, to_date)
             if messages:
                 loaded.append((info, messages))
+                loaded_totals[info.session_id] = session.token_totals
 
         if not loaded:
             continue  # everything in this project was empty / filtered out
@@ -3076,6 +3154,31 @@ def render_provider_wholesale(
             m for _info, msgs in loaded for m in msgs
         ]
 
+        # Token accounting (#296 deferral). Codex-style providers record
+        # cumulative session totals in the rollout rather than per-assistant-
+        # message ``usage``, so the message-usage accumulators
+        # (compute_session_data / compute_project_aggregates) see zero here.
+        # Pull each session's cumulative total from the provider seam and apply
+        # it directly — a cumulative figure must bypass that per-message
+        # summation, never flow through it (that path would double-count). The
+        # default seam returns None, so a provider without session-level totals
+        # leaves every surface exactly as before.
+        session_totals: dict[str, Optional[ProviderTokenTotals]] = {
+            info.session_id: loaded_totals[info.session_id] for info, _ in loaded
+        }
+        project_token_totals = _sum_provider_token_totals(session_totals.values())
+        # Did the provider actually supply cumulative totals? When it did not —
+        # every seam returned ``None``, which is the DEFAULT — the sum above is
+        # an all-zero dict, and applying it would REPLACE a provider's real
+        # per-message aggregates with zeros. The session-level override below
+        # is already gated on ``session_total is not None``; the two
+        # project-level uses must be gated symmetrically, or a provider that
+        # reports usage per message but has no cumulative seam silently loses
+        # its project totals.
+        has_provider_token_totals = any(
+            total is not None for total in session_totals.values()
+        )
+
         # Phase 2 — populate the cache and capture the pre-render modified set.
         cache: Optional[CacheManager] = None
         modified_sources: set[Path] = set()
@@ -3105,10 +3208,28 @@ def render_provider_wholesale(
                             info.source_path, messages, subagents_fp=""
                         )
                     merged_session_data.update(compute_session_data(messages))
+                # Replace the zero message-usage token totals with the
+                # provider's cumulative session totals (see the seam above).
+                # Keyed by session_id — Codex messages carry sessionId, so
+                # compute_session_data already keys each session that way.
+                for info, _ in loaded:
+                    session_total = session_totals.get(info.session_id)
+                    session_datum = merged_session_data.get(info.session_id)
+                    if session_total is not None and session_datum is not None:
+                        session_datum.total_input_tokens = session_total.input_tokens
+                        session_datum.total_output_tokens = session_total.output_tokens
+                        session_datum.total_cache_creation_tokens = 0
+                        session_datum.total_cache_read_tokens = (
+                            session_total.cache_read_tokens
+                        )
                 cache.update_session_cache(merged_session_data)
-                cache.update_project_aggregates(
-                    **compute_project_aggregates(combined_messages)
-                )
+                project_aggregates = compute_project_aggregates(combined_messages)
+                # Cumulative project totals override the (zero) per-message sum
+                # — but only when the provider supplied any; see
+                # ``has_provider_token_totals``.
+                if has_provider_token_totals:
+                    project_aggregates.update(project_token_totals)
+                cache.update_project_aggregates(**project_aggregates)
             session_counts = {
                 sid: sd.message_count for sid, sd in merged_session_data.items()
             }
@@ -3172,6 +3293,12 @@ def render_provider_wholesale(
                     "first_user_message": first_user
                     or "[No user message found in session.]",
                     "file": f"{rel_dest}/session-{session_key}{suffix}.{ext}",
+                    # NOTE: no per-session token_summary key here — the Claude
+                    # index project-card session list carries none either, and
+                    # the drift pin (test_index_summary_dict_shape_matches_claude_path)
+                    # locks the two session-dict shapes together. Per-session
+                    # cumulative totals are stored on the session cache instead
+                    # (durability); the project card shows the rolled-up total.
                 }
             )
 
@@ -3212,14 +3339,20 @@ def render_provider_wholesale(
                 "jsonl_count": len(session_dicts),
                 "message_count": len(combined_messages),
                 "last_modified": last_modified,
-                # Codex has no token accounting yet; emit zero totals so the
-                # index-summary dict is shape-identical to the Claude path
-                # (the contract the drift pin locks) rather than relying on the
-                # renderer's ``.get(..., 0)`` fallbacks.
-                "total_input_tokens": 0,
-                "total_output_tokens": 0,
-                "total_cache_creation_tokens": 0,
-                "total_cache_read_tokens": 0,
+                # Project-card token totals — the cumulative session totals
+                # summed across the project's sessions (#296 deferral closed).
+                # ``_sum_provider_token_totals`` keeps the dict shape-identical
+                # to the Claude path (all four keys, cache_creation pinned 0 and
+                # never displayed) so the drift pin's contract still holds.
+                # Sibling of the cache-side override above and gated the same
+                # way: with no cumulative seam this dict is all zeros, so fall
+                # back to the per-message aggregate rather than zeroing the
+                # card. Computed lazily — the fallback never runs for Codex.
+                **(
+                    project_token_totals
+                    if has_provider_token_totals
+                    else _project_token_totals_from_messages(combined_messages)
+                ),
                 "latest_timestamp": last_ts_all or "",
                 "earliest_timestamp": first_ts_all or "",
                 "working_directories": working_directories,

claude_code_log/providers/base.py

@@ -18,6 +18,49 @@
 )
 
 
+@dataclass(frozen=True)
+class ProviderTokenTotals:
+    """Cumulative session token totals surfaced by a provider that records
+    them at the session level (e.g. Codex ``token_count`` events), as opposed
+    to the Claude path which sums per-assistant-message ``usage``.
+
+    Mapped onto the same four columns the index renders, minus one:
+    ``cache_creation`` is deliberately ABSENT, not zero. Codex has no
+    cache-creation concept, and an omitted column ("we don't record this") is
+    a different, honest claim than a zero one ("we recorded zero of it").
+
+    ``total_tokens`` is the record's own authoritative total — never
+    recomputed from the components. For the well-formed cumulative records
+    that back session totals the identity ``input + cache_read + output ==
+    total`` holds, but degenerate records (all components zero, non-zero
+    total) do occur in the per-step stream, and there the stored total is the
+    only trustworthy figure. It is **currently unconsumed by the render/cache
+    paths** (the four displayed columns come from input/cache_read/output); it
+    is kept as the reconstruction anchor the tests validate and the reserve a
+    future per-turn layer would need.
+    """
+
+    input_tokens: int  # billable non-cached input = input_tokens - cached
+    cache_read_tokens: int  # cached_input_tokens
+    output_tokens: int  # output_tokens, which already includes reasoning
+    total_tokens: int  # record's authoritative total; never recomputed
+
+
+@dataclass(frozen=True)
+class LoadedSession:
+    """One session's rendered entries together with its cumulative token
+    totals, as returned by :meth:`BaseProvider.load_session_with_totals`.
+
+    The pair travels together because the caller needs both and a provider may
+    be able to produce both from a single parse. ``token_totals`` is ``None``
+    for the providers (and the sessions) that record none — omitted, never
+    zeroed, since a zero total is a different claim from an absent one.
+    """
+
+    entries: list[TranscriptEntry]
+    token_totals: Optional[ProviderTokenTotals]
+
+
 @dataclass
 class SessionInfo:
     provider: str
@@ -281,3 +324,45 @@ def load_session_under(
 
     def get_session_stats(self, session_id: str) -> dict[str, Any]:
         return {}
+
+    def session_token_totals(
+        self, root: Path, session_id: str
+    ) -> Optional[ProviderTokenTotals]:
+        """Cumulative session token totals for the session ``session_id`` under
+        ``root``, or ``None`` when the provider records none.
+
+        The default is ``None``: providers whose token accounting is
+        per-assistant-message ``usage`` (Claude) leave this alone — those
+        totals flow through the message-usage accumulators in ``converter``,
+        not this seam. A provider that records session-level cumulative totals
+        (Codex) overrides this so the wholesale/index path can surface them
+        directly, bypassing the per-message summation that would otherwise
+        double-count a cumulative figure.
+
+        Still the seam for a totals-only lookup. The wholesale walker uses
+        :meth:`load_session_with_totals` instead, so that a provider whose
+        totals live in the same source it just parsed need not re-read it.
+        """
+        return None
+
+    def load_session_with_totals(
+        self, root: Path, session_id: str, max_messages: Optional[int] = None
+    ) -> LoadedSession:
+        """Entries *and* cumulative token totals for one session, in one call.
+
+        The wholesale walker needs both, and for a provider that reads them
+        from the same file this is the difference between parsing that file
+        once and parsing it twice — the second parse being work the first
+        already did and discarded, not a recomputation worth caching (the
+        decoded records of one real archive reach 124 MB for a single session,
+        so any cache here would need a byte budget rather than an entry count).
+
+        **The default is exactly the pair of calls the walker used to make**,
+        so a provider that does not override this cannot change behaviour by
+        the seam existing. Override it only when the two can genuinely share
+        work; leave it alone otherwise.
+        """
+        return LoadedSession(
+            entries=list(self.load_session_under(root, session_id, max_messages)),
+            token_totals=self.session_token_totals(root, session_id),
+        )