Key results and details output paths on the model revision

docs/source/saving-and-reading-results.mdx

@@ -6,19 +6,24 @@ Lighteval provides comprehensive logging and result management through the `Eval
 
 Lighteval automatically saves results and evaluation details in the
 directory specified with the `--output-dir` option. The results are saved in
-`{output_dir}/results/{model_name}/results_{timestamp}.json`. [Here is an
+`{output_dir}/results/{model_name}/{revision}/results_{timestamp}.json`. [Here is an
 example of a result file](#example-of-a-result-file). The output path can be
 any [fsspec](https://filesystem-spec.readthedocs.io/en/latest/index.html)
 compliant path (local, S3, Hugging Face Hub, Google Drive, FTP, etc.).
 
 To save detailed evaluation information, you can use the `--save-details`
 option. The details are saved in Parquet files at
-`{output_dir}/details/{model_name}/{timestamp}/details_{task}_{timestamp}.parquet`.
+`{output_dir}/details/{model_name}/{revision}/{timestamp}/details_{task}_{timestamp}.parquet`.
+
+The `{revision}` segment keeps evaluations of different versions of the same model apart.
+It is the `--revision` passed to the model, and defaults to `main` for backends that do not
+support pinning a revision.
 
 If you want results to be saved in a custom path structure, you can set the `results-path-template` option.
-This allows you to specify a string template for the path. The template must contain the following
-variables: `output_dir`, `model_name`, `org`. For example:
-`{output_dir}/{org}_{model}`. The template will be used to create the path for the results file.
+This allows you to specify a string template for the path. The following variables are available:
+`output_dir`, `model`, `org` and `revision`. For example:
+`{output_dir}/{org}_{model}/{revision}`. The template will be used to create the path for the results
+file. Templates that do not use every variable remain valid.
 
 ## Pushing Results to the Hugging Face Hub
 
@@ -68,16 +73,17 @@ import glob
 
 output_dir = "evals_doc"
 model_name = "HuggingFaceH4/zephyr-7b-beta"
+revision = "main"
 timestamp = "latest"
 task = "gsm8k"
 
 if timestamp == "latest":
-    path = f"{output_dir}/details/{model_name}/*/"
+    path = f"{output_dir}/details/{model_name}/{revision}/*/"
     timestamps = glob.glob(path)
     timestamp = sorted(timestamps)[-1].split("/")[-2]
     print(f"Latest timestamp: {timestamp}")
 
-details_path = f"{output_dir}/details/{model_name}/{timestamp}/details_{task}_{timestamp}.parquet"
+details_path = f"{output_dir}/details/{model_name}/{revision}/{timestamp}/details_{task}_{timestamp}.parquet"
 
 # Load the details
 details = load_dataset("parquet", data_files=details_path, split="train")
@@ -135,7 +141,7 @@ tracker = EvaluationTracker(
 ```python
 tracker = EvaluationTracker(
     output_dir="./results",
-    results_path_template="{output_dir}/custom/{org}_{model}",
+    results_path_template="{output_dir}/custom/{org}_{model}/{revision}",
     save_details=True,
     push_to_hub=True,
     push_to_tensorboard=True,

src/lighteval/cli_args.py

@@ -140,7 +140,7 @@ class Arg:
     type=Annotated[
         str | None,
         Option(
-            help="Custom template for results file path. Available variables: {output_dir}, {org}, {model}. Example: '{output_dir}/experiments/{org}_{model}' creates results in a subdirectory.",
+            help="Custom template for results file path. Available variables: {output_dir}, {org}, {model}, {revision}. Example: '{output_dir}/experiments/{org}_{model}/{revision}' creates results in a subdirectory.",
             rich_help_panel=HELP_PANEL_NAME_2,
         ),
     ],

src/lighteval/logging/evaluation_tracker.py

@@ -111,7 +111,9 @@ class EvaluationTracker:
     Args:
         output_dir (str): Local directory to save evaluation results and logs
         results_path_template (str, optional): Template for results directory structure.
-            Example: "{output_dir}/results/{org}_{model}"
+            Supports the {output_dir}, {org}, {model} and {revision} variables; a template
+            using only a subset of them stays valid.
+            Example: "{output_dir}/results/{org}_{model}/{revision}"
         save_details (bool, defaults to True): Whether to save detailed evaluation records
         push_to_hub (bool, defaults to False): Whether to push results to HF Hub
         push_to_tensorboard (bool, defaults to False): Whether to push metrics to TensorBoard
@@ -303,23 +305,61 @@ def push_to_wandb(self, results_dict: dict, details_datasets: dict) -> None:
         )
         self.wandb_run.finish()
 
+    def _get_model_revision(self) -> str:
+        """Returns the revision of the evaluated model, used to disambiguate output paths.
+
+        Only the backends able to load a specific version of a model define a `revision` on
+        their config (transformers, VLM transformers, vllm and inference endpoints). For every
+        other backend, and for a model config logged without a revision, we fall back to "main",
+        which is the default those backends themselves use.
+
+        Returns:
+            str: The model revision, or "main" if the model config does not define one.
+        """
+        revision = getattr(self.general_config_logger.model_config, "revision", None)
+        if not revision:
+            return "main"
+        # Stripped like `model_name` is by the callers: a leading or trailing separator would
+        # otherwise introduce an empty path segment.
+        return revision.strip("/") or "main"
+
     def save_results(self, date_id: str, results_dict: dict):
+        revision = self._get_model_revision()
         if self.results_path_template is not None:
             org_model_parts = self.general_config_logger.model_name.split("/")
             org = org_model_parts[0] if len(org_model_parts) >= 2 else ""
             model = org_model_parts[1] if len(org_model_parts) >= 2 else org_model_parts[0]
             output_dir = self.output_dir
-            output_dir_results = Path(self.results_path_template.format(output_dir=output_dir, org=org, model=model))
+            output_dir_results = Path(
+                self.results_path_template.format(output_dir=output_dir, org=org, model=model, revision=revision)
+            )
         else:
-            output_dir_results = Path(self.output_dir) / "results" / self.general_config_logger.model_name.strip("/")
+            output_dir_results = (
+                Path(self.output_dir) / "results" / self.general_config_logger.model_name.strip("/") / revision
+            )
         self.fs.mkdirs(output_dir_results, exist_ok=True)
         output_results_file = output_dir_results / f"results_{date_id}.json"
         logger.info(f"Saving results to {output_results_file}")
         with self.fs.open(output_results_file, "w") as f:
             f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False))
 
-    def _get_details_sub_folder(self, date_id: str):
+    def _get_details_sub_folder(self, date_id: str, use_legacy_layout: bool = False):
+        """Returns the folder holding the details of a single evaluation run.
+
+        Args:
+            date_id (str): The run timestamp, or "first"/"last" to resolve it against the
+                timestamp folders present on disk.
+            use_legacy_layout (bool): Resolve against the pre-revision layout,
+                `{output_dir}/details/{model}/`, instead of the revision-scoped one. This is only
+                ever used to *read* details written by an older lighteval; details are always
+                written to the revision-scoped path.
+
+        Returns:
+            Path: The details folder for this run.
+        """
         output_dir_details = Path(self.output_dir) / "details" / self.general_config_logger.model_name.strip("/")
+        if not use_legacy_layout:
+            output_dir_details = output_dir_details / self._get_model_revision()
         if date_id in ["first", "last"]:
             # Get all folders in output_dir_details
             if not self.fs.exists(output_dir_details):
@@ -335,8 +375,46 @@ def _get_details_sub_folder(self, date_id: str):
             date_id = max(folders) if date_id == "last" else min(folders)
         return output_dir_details / date_id
 
+    def _find_legacy_details_sub_folder(self, date_id: str) -> Path | None:
+        """Returns the pre-revision details folder for `date_id`, if it actually holds details.
+
+        Details written before results were keyed on the revision live one level up, directly
+        under `{output_dir}/details/{model}/`. We require the resolved folder to contain details
+        files so that a sibling *revision* directory is never mistaken for a timestamp folder.
+
+        Args:
+            date_id (str): The run timestamp, or "first"/"last".
+
+        Returns:
+            Path | None: The legacy details folder, or None if it holds no details.
+        """
+        try:
+            sub_folder = self._get_details_sub_folder(date_id, use_legacy_layout=True)
+        except FileNotFoundError:
+            return None
+        return sub_folder if self.fs.glob(str(sub_folder / "details_*.parquet")) else None
+
     def load_details_datasets(self, date_id: str, task_names: list[str]) -> dict[str, Dataset]:
-        output_dir_details_sub_folder = self._get_details_sub_folder(date_id)
+        try:
+            output_dir_details_sub_folder = self._get_details_sub_folder(date_id)
+            missing_error = None
+        except FileNotFoundError as e:
+            output_dir_details_sub_folder, missing_error = None, e
+
+        if output_dir_details_sub_folder is None or not self.fs.glob(
+            str(output_dir_details_sub_folder / "details_*.parquet")
+        ):
+            legacy_sub_folder = self._find_legacy_details_sub_folder(date_id)
+            if legacy_sub_folder is not None:
+                logger.warning(
+                    f"No details found for revision '{self._get_model_revision()}'. Falling back to "
+                    f"the pre-revision layout at {legacy_sub_folder}."
+                )
+                output_dir_details_sub_folder = legacy_sub_folder
+            elif missing_error is not None:
+                # Nothing in either layout: surface the original error untouched.
+                raise missing_error
+
         logger.info(f"Loading details from {output_dir_details_sub_folder}")
         date_id = output_dir_details_sub_folder.name  # Overwrite date_id in case of latest
         details_datasets = {}

tests/unit/logging/test_evaluation_tracker.py

@@ -92,7 +92,8 @@ def test_results_logging(self, mock_evaluation_tracker: EvaluationTracker):
 
         mock_evaluation_tracker.save()
 
-        results_dir = Path(mock_evaluation_tracker.output_dir) / "results" / "test_model"
+        # DummyModelConfig defines no revision, so the path falls back to "main".
+        results_dir = Path(mock_evaluation_tracker.output_dir) / "results" / "test_model" / "main"
         assert results_dir.exists()
 
         result_files = list(results_dir.glob("results_*.json"))
@@ -139,7 +140,7 @@ def test_details_logging(self, mock_evaluation_tracker, mock_datetime):
         mock_evaluation_tracker.save()
 
         date_id = mock_datetime.isoformat().replace(":", "-")
-        details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" / date_id
+        details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" / "main" / date_id
         assert details_dir.exists()
 
         for task in ["task1", "task2"]:
@@ -153,7 +154,7 @@ def test_details_logging(self, mock_evaluation_tracker, mock_datetime):
     def test_no_details_output(self, mock_evaluation_tracker: EvaluationTracker):
         mock_evaluation_tracker.save()
 
-        details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model"
+        details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" / "main"
         assert not details_dir.exists()
 
     @pytest.mark.skip(  # skipif
@@ -201,6 +202,152 @@ def test_push_to_hub_works(
         assert len(details_files) == 2
 
 
+class TestRevisionInOutputPaths:
+    """Results and details are keyed on the model revision, so that evaluating several revisions
+    of the same model does not collapse every run into a single directory.
+    """
+
+    @staticmethod
+    def _make_tracker(output_dir: str, revision: str | None, save_details: bool = False):
+        """Builds a tracker for `test/model`, logging a model config with or without a revision.
+
+        Args:
+            output_dir (str): The tracker output directory.
+            revision (str | None): The revision to evaluate. When None, a `DummyModelConfig` is
+                logged instead, which does not define a `revision` field at all.
+            save_details (bool): Whether the tracker should save details.
+
+        Returns:
+            EvaluationTracker: A tracker with aggregated metrics already populated.
+        """
+        from lighteval.models.dummy.dummy_model import DummyModelConfig
+        from lighteval.models.transformers.transformers_model import TransformersModelConfig
+
+        tracker = EvaluationTracker(output_dir=output_dir, save_details=save_details)
+        if revision is None:
+            model_config = DummyModelConfig(model_name="test/model")
+        else:
+            model_config = TransformersModelConfig(model_name="test/model", revision=revision)
+        tracker.general_config_logger.log_model_info(model_config=model_config)
+        tracker.metrics_logger.metric_aggregated = {"task1": {"accuracy": 0.8}}
+        return tracker
+
+    def test_different_revisions_write_results_to_different_directories(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            for revision in ["v1.0", "v2.0"]:
+                self._make_tracker(temp_dir, revision).save()
+
+            model_dir = Path(temp_dir) / "results" / "test" / "model"
+            assert sorted(path.name for path in model_dir.iterdir()) == ["v1.0", "v2.0"]
+            for revision in ["v1.0", "v2.0"]:
+                assert len(list((model_dir / revision).glob("results_*.json"))) == 1
+
+    def test_different_revisions_write_details_to_different_directories(self, mock_datetime):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            for revision in ["v1.0", "v2.0"]:
+                tracker = self._make_tracker(temp_dir, revision, save_details=True)
+                tracker.details_logger.details = {
+                    "task1": [DetailsLogger.CompiledDetail(hashes=None, truncated=10, padded=5)]
+                }
+                tracker.save()
+
+            date_id = mock_datetime.isoformat().replace(":", "-")
+            model_dir = Path(temp_dir) / "details" / "test" / "model"
+            assert sorted(path.name for path in model_dir.iterdir()) == ["v1.0", "v2.0"]
+            for revision in ["v1.0", "v2.0"]:
+                details_file = model_dir / revision / date_id / f"details_task1_{date_id}.parquet"
+                assert details_file.is_file()
+
+    def test_missing_revision_falls_back_to_main(self):
+        """A model config without a `revision` field must still produce a valid path."""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            self._make_tracker(temp_dir, revision=None).save()
+
+            results_dir = Path(temp_dir) / "results" / "test" / "model" / "main"
+            assert results_dir.is_dir()
+            assert len(list(results_dir.glob("results_*.json"))) == 1
+
+    def test_explicit_main_revision_matches_the_fallback(self):
+        """Evaluating `main` explicitly lands in the same place as not specifying a revision."""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            self._make_tracker(temp_dir, revision="main").save()
+
+            assert (Path(temp_dir) / "results" / "test" / "model" / "main").is_dir()
+
+    def test_results_path_template_supports_revision(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            tracker = self._make_tracker(temp_dir, "v1.0")
+            tracker.results_path_template = "{output_dir}/{org}_{model}/{revision}"
+            tracker.save()
+
+            assert len(list((Path(temp_dir) / "test_model" / "v1.0").glob("results_*.json"))) == 1
+
+    def test_results_path_template_without_revision_is_unaffected(self):
+        """Templates written before `{revision}` existed must keep resolving unchanged."""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            tracker = self._make_tracker(temp_dir, "v1.0")
+            tracker.results_path_template = "{output_dir}/{org}_{model}"
+            tracker.save()
+
+            assert len(list((Path(temp_dir) / "test_model").glob("results_*.json"))) == 1
+
+    def test_details_saved_before_this_change_are_still_readable(self):
+        """Details in the pre-revision layout stay loadable, so upgrading does not strand them."""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            tracker = self._make_tracker(temp_dir, revision="v1.0", save_details=True)
+
+            # Lay details out the way lighteval wrote them before results were keyed on the
+            # revision: directly under details/{model}/{timestamp}/, with no revision segment.
+            # The trailing "|0" is the fewshot count, which load_details_datasets strips before
+            # matching against task_names.
+            date_id = "2023-01-01T12-00-00.000000"
+            legacy_dir = Path(temp_dir) / "details" / "test" / "model" / date_id
+            legacy_dir.mkdir(parents=True)
+            Dataset.from_dict({"truncated": [10], "padded": [5]}).to_parquet(
+                str(legacy_dir / f"details_task1|0_{date_id}.parquet")
+            )
+
+            loaded = tracker.load_details_datasets(date_id, ["task1"])
+
+            assert list(loaded.keys()) == ["task1|0"]
+            assert len(loaded["task1|0"]) == 1
+
+    def test_legacy_fallback_resolves_the_last_timestamp(self):
+        """The fallback also covers the "last" alias, not just an explicit timestamp."""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            tracker = self._make_tracker(temp_dir, revision="v1.0", save_details=True)
+
+            model_dir = Path(temp_dir) / "details" / "test" / "model"
+            for date_id in ["2023-01-01T12-00-00.000000", "2023-06-01T12-00-00.000000"]:
+                legacy_dir = model_dir / date_id
+                legacy_dir.mkdir(parents=True)
+                Dataset.from_dict({"truncated": [10]}).to_parquet(
+                    str(legacy_dir / f"details_task1|0_{date_id}.parquet")
+                )
+
+            loaded = tracker.load_details_datasets("last", ["task1"])
+
+            assert list(loaded.keys()) == ["task1|0"]
+
+    def test_sibling_revision_folder_is_not_mistaken_for_legacy_details(self):
+        """A revision directory must never be picked up as though it were a timestamp folder.
+
+        `details/{model}/` is the parent of every revision directory, so a naive fallback would
+        treat `v1.0/` as a timestamp folder when asked for the details of some other revision.
+        """
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Only revision-scoped details exist, and none for the revision we ask about.
+            writer = self._make_tracker(temp_dir, revision="v1.0", save_details=True)
+            writer.details_logger.details = {
+                "task1": [DetailsLogger.CompiledDetail(hashes=None, truncated=10, padded=5)]
+            }
+            writer.save()
+
+            reader = self._make_tracker(temp_dir, revision="v2.0", save_details=True)
+            with pytest.raises((FileNotFoundError, ValueError)):
+                reader.load_details_datasets("last", ["task1"])
+
+
 class TestProperties(unittest.TestCase):
     def setUp(self):
         # In setup in case we need to reuse for future tests