fix(tier3): exclude failed skill reads from execution credit

CHANGELOG.md

@@ -93,6 +93,13 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ### Fixed
 
+- Preserve explicit Claude and OpenCode read failure metadata in reconstructed
+  ATIF trajectories. Failed isolated SKILL.md reads and failed native Skill
+  calls lose credit; compound shell failures and shell reads whose path may
+  expand to several files (braces, globs, `$` or backticks) remain ambiguous
+  and retain prior credit in host and verifier grading. Unattributed errors do
+  not cancel successful reads, and shell reads count dash-prefixed filenames
+  after `--`.
 - Tier 1 pyproject dependency audits now inspect static target declarations
   rather than the evaluator environment. Exact direct pins (including optional
   groups) are queried without installation or build-script execution;

src/skillevaluator/tier3/eval_core/atif_helpers.py

@@ -17,7 +17,11 @@
 from typing import Any
 
 from skillevaluator.evidence import evidence_ref_identity
-from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import atif_content_has_text, atif_content_text
+from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import (
+    atif_content_has_text,
+    atif_content_text,
+    normalized_tool_call_is_error,
+)
 from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import (
     iter_normalized_tool_calls as iter_tool_calls,
 )
@@ -53,11 +57,15 @@ def get_all_tool_calls(traj: dict[str, Any]) -> list[dict[str, Any]]:
 def get_skill_tool_calls(traj: dict[str, Any]) -> list[str]:
     """Get skill names from Claude Code's native ``Skill`` tool invocations."""
     skills: list[str] = []
-    for tc in get_all_tool_calls(traj):
-        if tc["fn"].lower() == "skill":
-            name = tc["args"].get("skill", tc["args"].get("name", ""))
-            if name:
-                skills.append(str(name))
+    for step, tc in iter_tool_calls(traj):
+        if (tc.get("function_name") or "").lower() != "skill":
+            continue
+        if normalized_tool_call_is_error(step, tc):
+            continue
+        args = tc.get("arguments") or {}
+        name = args.get("skill", args.get("name", ""))
+        if name:
+            skills.append(str(name))
     return skills
 
 
@@ -1054,6 +1062,8 @@ def extract_tool_calls_as_dicts(traj: dict[str, Any]) -> list[dict[str, Any]]:
                 "action_input": tc.get("arguments") or {},
                 "observation": _tool_call_observation(step, tc),
             }
+            if normalized_tool_call_is_error(step, tc):
+                call["is_error"] = True
             if status := tc.get("_atif_normalization_status"):
                 call["normalization_status"] = status
             if status := tc.get("_atif_observation_status"):

src/skillevaluator/tier3/eval_core/checks.py

@@ -640,6 +640,38 @@ def _command_reads_skill_md_arg(command: list[str], cmd_idx: int, assignments: d
     return False
 
 
+# Brace, glob, parameter and command expansion can turn one shell word into
+# several files. Quotes are gone after tokenizing, so any operand holding one of
+# these characters is treated as possibly naming more than one file.
+_SHELL_EXPANSION_CHARS = frozenset("{*?[$`")
+
+
+def _failed_single_skill_read(call):
+    """A shell failure only proves the skill read failed for an isolated read.
+
+    Compound commands, multi-file reads and operands that the shell may expand
+    into several files can print SKILL.md successfully before another file or
+    operation fails.
+    """
+    if call.get("is_error") is not True:
+        return False
+    tokens = _shell_tokens(_command_text(call))
+    if not tokens or any(token in _SHELL_SEPARATORS for token in tokens):
+        return False
+    command = tokens[0].rsplit("/", 1)[-1].lower()
+    if command not in {"cat", "nl", "bat", "less", "more"}:
+        return False
+    files = []
+    options_ended = False
+    for arg in tokens[1:]:
+        arg = arg.removeprefix(_QUOTED_SYNTAX_MARK)
+        if not options_ended and arg == "--":
+            options_ended = True
+        elif options_ended or arg == "-" or not arg.startswith("-"):
+            files.append(arg)
+    return len(files) == 1 and not _SHELL_EXPANSION_CHARS.intersection(files[0]) and _skill_md_arg(files[0], {})
+
+
 def _cmd_reads_skill_md(cmd) -> bool:
     """True if a shell command reads a SKILL.md via a file-view utility.
 
@@ -2018,7 +2050,7 @@ def check_activation(
                 }
 
     # Check 2: read_file / read with SKILL.md in path
-    read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower()]
+    read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower() and tc.get("is_error") is not True]
     for call in read_calls:
         path_arg = _extract_path(call)
         if "SKILL.md" not in path_arg:
@@ -2037,7 +2069,7 @@ def check_activation(
             }
 
     # Check 3: shell read of SKILL.md (cat/sed/head/...)
-    exec_calls = [tc for tc in tool_calls if _is_execution_action(str(tc["action"]))]
+    exec_calls = [tc for tc in tool_calls if _is_execution_action(str(tc["action"])) and not _failed_single_skill_read(tc)]
     for call in exec_calls:
         cmd = _command_text(call)
         if _cmd_reads_skill_md(cmd):
@@ -2073,7 +2105,10 @@ def check_activation(
         has_skill_read_evidence = ("read" in action.lower() and "SKILL.md" in str(tc.get("action_input", ""))) or (
             _is_execution_action(action) and _cmd_reads_skill_md(cmd)
         )
-        if not has_skill_read_evidence:
+        if not has_skill_read_evidence or (
+            tc.get("is_error") is True
+            and (not _is_execution_action(action) or _failed_single_skill_read(tc))
+        ):
             continue
         obs = str(tc.get("observation", "")).lower()
         if "skill.md" in obs:
@@ -5107,7 +5142,10 @@ def check_workflow_order(
         cmd = _command_text(call)
 
         if ("read" in action and "SKILL.md" in args_str) or (_is_execution_action(action) and _cmd_reads_skill_md(cmd)):
-            sequence.append("read_skill")
+            if call.get("is_error") is not True or (
+                _is_execution_action(action) and not _failed_single_skill_read(call)
+            ):
+                sequence.append("read_skill")
         elif _is_execution_action(action) and cmd and "--help" not in cmd and "which " not in cmd:
             sequence.append("execution")
 
@@ -5365,7 +5403,7 @@ def check_routing(
 ) -> dict[str, Any]:
     """Check the agent read only expected/allowed workspace skill docs."""
     unsupported_native_codex_call = _has_unsupported_native_codex_call(tool_calls)
-    read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower()]
+    read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower() and tc.get("is_error") is not True]
 
     skills_read: list[str] = []
     wrong_skills: list[str] = []
@@ -5398,7 +5436,7 @@ def check_routing(
     for call in tool_calls:
         action = call["action"].lower()
         cmd = _command_text(call)
-        if not (_is_execution_action(action) and _cmd_reads_skill_md(cmd)):
+        if not (_is_execution_action(action) and _cmd_reads_skill_md(cmd)) or _failed_single_skill_read(call):
             continue
 
         skills_read.append(cmd)

src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py

@@ -101,6 +101,22 @@ def normalized_tool_call_observation(step: dict[str, Any], tool_call: dict[str,
     )
 
 
+def normalized_tool_call_is_error(step: dict[str, Any], tool_call: dict[str, Any]) -> bool:
+    """Preserve explicit failure metadata only for an attributable tool result."""
+    if tool_call.get("_atif_observation_status") not in (None, MAPPED_OUTER_EXEC_OBSERVATION):
+        return False
+    tool_call_id = tool_call.get("tool_call_id")
+    for result in (step.get("observation") or {}).get("results") or []:
+        source_call_id = result.get("source_call_id")
+        # Unscoped observations cannot prove that this particular call failed.
+        if not tool_call_id or source_call_id != tool_call_id:
+            continue
+        extra = result.get("extra") or {}
+        if isinstance(extra, dict) and extra.get("tool_result_is_error") is True:
+            return True
+    return False
+
+
 def normalized_tool_call_wrapper_observation(step: dict[str, Any], tool_call: dict[str, Any]) -> str:
     """Return an unattributed wrapper observation once for security scanning."""
     if tool_call.get("_atif_observation_status") not in (

src/skillevaluator/tier3/eval_core/log_converters.py

@@ -54,20 +54,21 @@ def _stringify_tool_result_content(content: Any) -> str:
     return str(content)
 
 
-def _attach_tool_result(steps: list[dict[str, Any]], tool_use_id: str, content_text: str) -> None:
+def _attach_tool_result(
+    steps: list[dict[str, Any]], tool_use_id: str, content_text: str, *, is_error: bool = False
+) -> None:
     for step in reversed(steps):
         if step.get("source") != "agent":
             continue
         for tc in step.get("tool_calls") or []:
             if tc.get("tool_call_id") == tool_use_id:
                 step.setdefault("observation", {"results": []})
                 step["observation"].setdefault("results", [])
-                step["observation"]["results"].append(
-                    {
-                        "source_call_id": tool_use_id,
-                        "content": content_text,
-                    }
-                )
+                result: dict[str, Any] = {"source_call_id": tool_use_id, "content": content_text}
+                if is_error:
+                    # Match Harbor's native Claude Code ATIF result metadata.
+                    result["extra"] = {"tool_result_is_error": True}
+                step["observation"]["results"].append(result)
                 return
 
 
@@ -158,7 +159,7 @@ def synthetic_trajectory_from_claude_stream_jsonl(text: str) -> dict[str, Any] |
                 tid = str(block.get("tool_use_id") or "")
                 res_text = _stringify_tool_result_content(block.get("content"))
                 if tid:
-                    _attach_tool_result(steps, tid, res_text)
+                    _attach_tool_result(steps, tid, res_text, is_error=block.get("is_error") is True)
 
         elif et == "result":
             res = evt.get("result")
@@ -564,13 +565,12 @@ def synthetic_trajectory_from_opencode_json(text: str) -> dict[str, Any] | None:
             "observation": {"results": []},
         }
         observation_text = _opencode_tool_observation(state)
-        if observation_text:
-            step["observation"]["results"].append(
-                {
-                    "source_call_id": call_id,
-                    "content": observation_text,
-                }
-            )
+        failed_read = status == "error" and function_name.lower() in {"read", "read_file"}
+        if observation_text or failed_read:
+            result = {"source_call_id": call_id, "content": observation_text}
+            if failed_read:
+                result["extra"] = {"tool_result_is_error": True}
+            step["observation"]["results"].append(result)
         steps.append(step)
 
     if not saw_content or not steps:

src/skillevaluator/tier3/harbor/templates/eval.py

@@ -77,6 +77,7 @@ def load_trajectory_with_fallback(trajectory_path, logs_dir=None):
         atif_content_has_text,
         atif_content_text,
         iter_normalized_tool_calls,
+        normalized_tool_call_is_error,
         normalized_tool_call_observation,
         normalized_tool_call_wrapper_observation,
     )
@@ -88,6 +89,7 @@ def load_trajectory_with_fallback(trajectory_path, logs_dir=None):
         atif_content_has_text,
         atif_content_text,
         iter_normalized_tool_calls,
+        normalized_tool_call_is_error,
         normalized_tool_call_observation,
         normalized_tool_call_wrapper_observation,
     )
@@ -403,11 +405,15 @@ def get_all_tool_calls(traj):
 
 def get_skill_tool_calls(traj):
     skills = []
-    for tc in get_all_tool_calls(traj):
-        if tc["fn"].lower() == "skill":
-            name = tc["args"].get("skill", tc["args"].get("name", ""))
-            if name:
-                skills.append(str(name))
+    for step, tc in iter_tool_calls(traj):
+        if (tc.get("function_name") or "").lower() != "skill":
+            continue
+        if normalized_tool_call_is_error(step, tc):
+            continue
+        args = tc.get("arguments") or {}
+        name = args.get("skill", args.get("name", ""))
+        if name:
+            skills.append(str(name))
     return skills
 
 
@@ -458,6 +464,8 @@ def extract_tool_calls_as_dicts(traj):
                 "action_input": tc.get("arguments") or {},
                 "observation": _tool_call_observation(step, tc),
             }
+            if normalized_tool_call_is_error(step, tc):
+                call["is_error"] = True
             if status := tc.get("_atif_normalization_status"):
                 call["normalization_status"] = status
             if status := tc.get("_atif_observation_status"):
@@ -3260,6 +3268,38 @@ def _command_reads_skill_md_arg(command, cmd_idx, assignments):
     return False
 
 
+# Brace, glob, parameter and command expansion can turn one shell word into
+# several files. Quotes are gone after tokenizing, so any operand holding one of
+# these characters is treated as possibly naming more than one file.
+_SHELL_EXPANSION_CHARS = frozenset("{*?[$`")
+
+
+def _failed_single_skill_read(call):
+    """A shell failure only proves the skill read failed for an isolated read.
+
+    Compound commands, multi-file reads and operands that the shell may expand
+    into several files can print SKILL.md successfully before another file or
+    operation fails.
+    """
+    if call.get("is_error") is not True:
+        return False
+    tokens = _shell_tokens(_command_text(call))
+    if not tokens or any(token in _SHELL_SEPARATORS for token in tokens):
+        return False
+    command = tokens[0].rsplit("/", 1)[-1].lower()
+    if command not in {"cat", "nl", "bat", "less", "more"}:
+        return False
+    files = []
+    options_ended = False
+    for arg in tokens[1:]:
+        arg = arg.removeprefix(_QUOTED_SYNTAX_MARK)
+        if not options_ended and arg == "--":
+            options_ended = True
+        elif options_ended or arg == "-" or not arg.startswith("-"):
+            files.append(arg)
+    return len(files) == 1 and not _SHELL_EXPANSION_CHARS.intersection(files[0]) and _skill_md_arg(files[0], {})
+
+
 def _cmd_reads_skill_md(cmd) -> bool:
     """True if a shell command reads a SKILL.md via a file-view utility.
 
@@ -4574,7 +4614,7 @@ def check_activation(tool_calls, expected_skill, skill_tool_names=None, acceptab
                     "reason": reason,
                     "details": {**_skill_match_details(expected_skill, acceptable_skills), **match},
                 }
-    read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower()]
+    read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower() and tc.get("is_error") is not True]
     for call in read_calls:
         path_arg = _extract_path(call)
         if "SKILL.md" not in path_arg:
@@ -4592,7 +4632,7 @@ def check_activation(tool_calls, expected_skill, skill_tool_names=None, acceptab
                 "details": {**_skill_match_details(expected_skill, acceptable_skills), **match, "path": path_arg},
             }
     for call in tool_calls:
-        if _is_execution_action(call["action"]):
+        if _is_execution_action(call["action"]) and not _failed_single_skill_read(call):
             cmd = _command_text(call)
             if _cmd_reads_skill_md(cmd):
                 match = _classify_skill_match(cmd, expected_skill, acceptable_skills, fuzzy=True)
@@ -4624,7 +4664,10 @@ def check_activation(tool_calls, expected_skill, skill_tool_names=None, acceptab
         has_skill_read_evidence = ("read" in action.lower() and "SKILL.md" in str(tc.get("action_input", ""))) or (
             _is_execution_action(action) and _cmd_reads_skill_md(cmd)
         )
-        if not has_skill_read_evidence:
+        if not has_skill_read_evidence or (
+            tc.get("is_error") is True
+            and (not _is_execution_action(action) or _failed_single_skill_read(tc))
+        ):
             continue
         obs = str(tc.get("observation", "")).lower()
         if "skill.md" in obs:
@@ -7641,7 +7684,10 @@ def check_workflow_order(tool_calls, skill_tool_names=None, expected_skill=None)
         args_str = str(call.get("action_input", ""))
         cmd = _command_text(call)
         if ("read" in action and "SKILL.md" in args_str) or (_is_execution_action(action) and _cmd_reads_skill_md(cmd)):
-            sequence.append("read_skill")
+            if call.get("is_error") is not True or (
+                _is_execution_action(action) and not _failed_single_skill_read(call)
+            ):
+                sequence.append("read_skill")
         elif _is_execution_action(action) and cmd and "--help" not in cmd and "which " not in cmd:
             sequence.append("execution")
     if not sequence:
@@ -7724,7 +7770,7 @@ def check_routing(
     acceptable_skills=None,
 ):
     unsupported_native_codex_call = _has_unsupported_native_codex_call(tool_calls)
-    read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower()]
+    read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower() and tc.get("is_error") is not True]
     skills_read, wrong_skills = [], []
     matched_expected = False
     matched_alternate = False
@@ -7752,7 +7798,7 @@ def check_routing(
     for call in tool_calls:
         action = call["action"].lower()
         cmd = _command_text(call)
-        if not (_is_execution_action(action) and _cmd_reads_skill_md(cmd)):
+        if not (_is_execution_action(action) and _cmd_reads_skill_md(cmd)) or _failed_single_skill_read(call):
             continue
         skills_read.append(cmd)
         skill_name = _skill_name_from_ref(cmd)