From 48bbb990650bbfb003b656dea024cbc22c155d08 Mon Sep 17 00:00:00 2001 From: r37033ac Date: Sun, 30 Aug 2026 05:01:40 +0100 Subject: [PATCH] Key results and details output paths on the model revision Results and details were written to a directory keyed only on the model name, so evaluating several revisions of the same model interleaved every run in one folder, distinguished only by a timestamp. Results now go to {output_dir}/results/{model}/{revision}/ and details to {output_dir}/details/{model}/{revision}/{timestamp}/. The revision is read off the logged model config, which only the backends that can pin a model version declare it on (transformers, VLM transformers, vllm and inference endpoints); everything else falls back to "main", the default those same backends use. results_path_template gains a {revision} placeholder. Templates that do not use it are unaffected, since str.format ignores unused keyword arguments. Details written by an earlier lighteval, which have no revision segment, are still readable: when a revision-scoped folder holds no details, loading falls back to the old layout. The fallback requires the old folder to contain details files itself, so a sibling revision directory is never mistaken for a timestamp folder. --- docs/source/saving-and-reading-results.mdx | 22 ++- src/lighteval/cli_args.py | 2 +- src/lighteval/logging/evaluation_tracker.py | 88 +++++++++- tests/unit/logging/test_evaluation_tracker.py | 153 +++++++++++++++++- 4 files changed, 248 insertions(+), 17 deletions(-) diff --git a/docs/source/saving-and-reading-results.mdx b/docs/source/saving-and-reading-results.mdx index 167cf12e0..55e924f99 100644 --- a/docs/source/saving-and-reading-results.mdx +++ b/docs/source/saving-and-reading-results.mdx @@ -6,19 +6,24 @@ Lighteval provides comprehensive logging and result management through the `Eval Lighteval automatically saves results and evaluation details in the directory specified with the `--output-dir` option. The results are saved in -`{output_dir}/results/{model_name}/results_{timestamp}.json`. [Here is an +`{output_dir}/results/{model_name}/{revision}/results_{timestamp}.json`. [Here is an example of a result file](#example-of-a-result-file). The output path can be any [fsspec](https://filesystem-spec.readthedocs.io/en/latest/index.html) compliant path (local, S3, Hugging Face Hub, Google Drive, FTP, etc.). To save detailed evaluation information, you can use the `--save-details` option. The details are saved in Parquet files at -`{output_dir}/details/{model_name}/{timestamp}/details_{task}_{timestamp}.parquet`. +`{output_dir}/details/{model_name}/{revision}/{timestamp}/details_{task}_{timestamp}.parquet`. + +The `{revision}` segment keeps evaluations of different versions of the same model apart. +It is the `--revision` passed to the model, and defaults to `main` for backends that do not +support pinning a revision. If you want results to be saved in a custom path structure, you can set the `results-path-template` option. -This allows you to specify a string template for the path. The template must contain the following -variables: `output_dir`, `model_name`, `org`. For example: -`{output_dir}/{org}_{model}`. The template will be used to create the path for the results file. +This allows you to specify a string template for the path. The following variables are available: +`output_dir`, `model`, `org` and `revision`. For example: +`{output_dir}/{org}_{model}/{revision}`. The template will be used to create the path for the results +file. Templates that do not use every variable remain valid. ## Pushing Results to the Hugging Face Hub @@ -68,16 +73,17 @@ import glob output_dir = "evals_doc" model_name = "HuggingFaceH4/zephyr-7b-beta" +revision = "main" timestamp = "latest" task = "gsm8k" if timestamp == "latest": - path = f"{output_dir}/details/{model_name}/*/" + path = f"{output_dir}/details/{model_name}/{revision}/*/" timestamps = glob.glob(path) timestamp = sorted(timestamps)[-1].split("/")[-2] print(f"Latest timestamp: {timestamp}") -details_path = f"{output_dir}/details/{model_name}/{timestamp}/details_{task}_{timestamp}.parquet" +details_path = f"{output_dir}/details/{model_name}/{revision}/{timestamp}/details_{task}_{timestamp}.parquet" # Load the details details = load_dataset("parquet", data_files=details_path, split="train") @@ -135,7 +141,7 @@ tracker = EvaluationTracker( ```python tracker = EvaluationTracker( output_dir="./results", - results_path_template="{output_dir}/custom/{org}_{model}", + results_path_template="{output_dir}/custom/{org}_{model}/{revision}", save_details=True, push_to_hub=True, push_to_tensorboard=True, diff --git a/src/lighteval/cli_args.py b/src/lighteval/cli_args.py index 2bd52af31..64fbd0a30 100644 --- a/src/lighteval/cli_args.py +++ b/src/lighteval/cli_args.py @@ -140,7 +140,7 @@ class Arg: type=Annotated[ str | None, Option( - help="Custom template for results file path. Available variables: {output_dir}, {org}, {model}. Example: '{output_dir}/experiments/{org}_{model}' creates results in a subdirectory.", + help="Custom template for results file path. Available variables: {output_dir}, {org}, {model}, {revision}. Example: '{output_dir}/experiments/{org}_{model}/{revision}' creates results in a subdirectory.", rich_help_panel=HELP_PANEL_NAME_2, ), ], diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index d6f6a04a0..79904e7c5 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -111,7 +111,9 @@ class EvaluationTracker: Args: output_dir (str): Local directory to save evaluation results and logs results_path_template (str, optional): Template for results directory structure. - Example: "{output_dir}/results/{org}_{model}" + Supports the {output_dir}, {org}, {model} and {revision} variables; a template + using only a subset of them stays valid. + Example: "{output_dir}/results/{org}_{model}/{revision}" save_details (bool, defaults to True): Whether to save detailed evaluation records push_to_hub (bool, defaults to False): Whether to push results to HF Hub push_to_tensorboard (bool, defaults to False): Whether to push metrics to TensorBoard @@ -303,23 +305,61 @@ def push_to_wandb(self, results_dict: dict, details_datasets: dict) -> None: ) self.wandb_run.finish() + def _get_model_revision(self) -> str: + """Returns the revision of the evaluated model, used to disambiguate output paths. + + Only the backends able to load a specific version of a model define a `revision` on + their config (transformers, VLM transformers, vllm and inference endpoints). For every + other backend, and for a model config logged without a revision, we fall back to "main", + which is the default those backends themselves use. + + Returns: + str: The model revision, or "main" if the model config does not define one. + """ + revision = getattr(self.general_config_logger.model_config, "revision", None) + if not revision: + return "main" + # Stripped like `model_name` is by the callers: a leading or trailing separator would + # otherwise introduce an empty path segment. + return revision.strip("/") or "main" + def save_results(self, date_id: str, results_dict: dict): + revision = self._get_model_revision() if self.results_path_template is not None: org_model_parts = self.general_config_logger.model_name.split("/") org = org_model_parts[0] if len(org_model_parts) >= 2 else "" model = org_model_parts[1] if len(org_model_parts) >= 2 else org_model_parts[0] output_dir = self.output_dir - output_dir_results = Path(self.results_path_template.format(output_dir=output_dir, org=org, model=model)) + output_dir_results = Path( + self.results_path_template.format(output_dir=output_dir, org=org, model=model, revision=revision) + ) else: - output_dir_results = Path(self.output_dir) / "results" / self.general_config_logger.model_name.strip("/") + output_dir_results = ( + Path(self.output_dir) / "results" / self.general_config_logger.model_name.strip("/") / revision + ) self.fs.mkdirs(output_dir_results, exist_ok=True) output_results_file = output_dir_results / f"results_{date_id}.json" logger.info(f"Saving results to {output_results_file}") with self.fs.open(output_results_file, "w") as f: f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False)) - def _get_details_sub_folder(self, date_id: str): + def _get_details_sub_folder(self, date_id: str, use_legacy_layout: bool = False): + """Returns the folder holding the details of a single evaluation run. + + Args: + date_id (str): The run timestamp, or "first"/"last" to resolve it against the + timestamp folders present on disk. + use_legacy_layout (bool): Resolve against the pre-revision layout, + `{output_dir}/details/{model}/`, instead of the revision-scoped one. This is only + ever used to *read* details written by an older lighteval; details are always + written to the revision-scoped path. + + Returns: + Path: The details folder for this run. + """ output_dir_details = Path(self.output_dir) / "details" / self.general_config_logger.model_name.strip("/") + if not use_legacy_layout: + output_dir_details = output_dir_details / self._get_model_revision() if date_id in ["first", "last"]: # Get all folders in output_dir_details if not self.fs.exists(output_dir_details): @@ -335,8 +375,46 @@ def _get_details_sub_folder(self, date_id: str): date_id = max(folders) if date_id == "last" else min(folders) return output_dir_details / date_id + def _find_legacy_details_sub_folder(self, date_id: str) -> Path | None: + """Returns the pre-revision details folder for `date_id`, if it actually holds details. + + Details written before results were keyed on the revision live one level up, directly + under `{output_dir}/details/{model}/`. We require the resolved folder to contain details + files so that a sibling *revision* directory is never mistaken for a timestamp folder. + + Args: + date_id (str): The run timestamp, or "first"/"last". + + Returns: + Path | None: The legacy details folder, or None if it holds no details. + """ + try: + sub_folder = self._get_details_sub_folder(date_id, use_legacy_layout=True) + except FileNotFoundError: + return None + return sub_folder if self.fs.glob(str(sub_folder / "details_*.parquet")) else None + def load_details_datasets(self, date_id: str, task_names: list[str]) -> dict[str, Dataset]: - output_dir_details_sub_folder = self._get_details_sub_folder(date_id) + try: + output_dir_details_sub_folder = self._get_details_sub_folder(date_id) + missing_error = None + except FileNotFoundError as e: + output_dir_details_sub_folder, missing_error = None, e + + if output_dir_details_sub_folder is None or not self.fs.glob( + str(output_dir_details_sub_folder / "details_*.parquet") + ): + legacy_sub_folder = self._find_legacy_details_sub_folder(date_id) + if legacy_sub_folder is not None: + logger.warning( + f"No details found for revision '{self._get_model_revision()}'. Falling back to " + f"the pre-revision layout at {legacy_sub_folder}." + ) + output_dir_details_sub_folder = legacy_sub_folder + elif missing_error is not None: + # Nothing in either layout: surface the original error untouched. + raise missing_error + logger.info(f"Loading details from {output_dir_details_sub_folder}") date_id = output_dir_details_sub_folder.name # Overwrite date_id in case of latest details_datasets = {} diff --git a/tests/unit/logging/test_evaluation_tracker.py b/tests/unit/logging/test_evaluation_tracker.py index 3a76be2a4..065f809ea 100644 --- a/tests/unit/logging/test_evaluation_tracker.py +++ b/tests/unit/logging/test_evaluation_tracker.py @@ -92,7 +92,8 @@ def test_results_logging(self, mock_evaluation_tracker: EvaluationTracker): mock_evaluation_tracker.save() - results_dir = Path(mock_evaluation_tracker.output_dir) / "results" / "test_model" + # DummyModelConfig defines no revision, so the path falls back to "main". + results_dir = Path(mock_evaluation_tracker.output_dir) / "results" / "test_model" / "main" assert results_dir.exists() result_files = list(results_dir.glob("results_*.json")) @@ -139,7 +140,7 @@ def test_details_logging(self, mock_evaluation_tracker, mock_datetime): mock_evaluation_tracker.save() date_id = mock_datetime.isoformat().replace(":", "-") - details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" / date_id + details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" / "main" / date_id assert details_dir.exists() for task in ["task1", "task2"]: @@ -153,7 +154,7 @@ def test_details_logging(self, mock_evaluation_tracker, mock_datetime): def test_no_details_output(self, mock_evaluation_tracker: EvaluationTracker): mock_evaluation_tracker.save() - details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" + details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" / "main" assert not details_dir.exists() @pytest.mark.skip( # skipif @@ -201,6 +202,152 @@ def test_push_to_hub_works( assert len(details_files) == 2 +class TestRevisionInOutputPaths: + """Results and details are keyed on the model revision, so that evaluating several revisions + of the same model does not collapse every run into a single directory. + """ + + @staticmethod + def _make_tracker(output_dir: str, revision: str | None, save_details: bool = False): + """Builds a tracker for `test/model`, logging a model config with or without a revision. + + Args: + output_dir (str): The tracker output directory. + revision (str | None): The revision to evaluate. When None, a `DummyModelConfig` is + logged instead, which does not define a `revision` field at all. + save_details (bool): Whether the tracker should save details. + + Returns: + EvaluationTracker: A tracker with aggregated metrics already populated. + """ + from lighteval.models.dummy.dummy_model import DummyModelConfig + from lighteval.models.transformers.transformers_model import TransformersModelConfig + + tracker = EvaluationTracker(output_dir=output_dir, save_details=save_details) + if revision is None: + model_config = DummyModelConfig(model_name="test/model") + else: + model_config = TransformersModelConfig(model_name="test/model", revision=revision) + tracker.general_config_logger.log_model_info(model_config=model_config) + tracker.metrics_logger.metric_aggregated = {"task1": {"accuracy": 0.8}} + return tracker + + def test_different_revisions_write_results_to_different_directories(self): + with tempfile.TemporaryDirectory() as temp_dir: + for revision in ["v1.0", "v2.0"]: + self._make_tracker(temp_dir, revision).save() + + model_dir = Path(temp_dir) / "results" / "test" / "model" + assert sorted(path.name for path in model_dir.iterdir()) == ["v1.0", "v2.0"] + for revision in ["v1.0", "v2.0"]: + assert len(list((model_dir / revision).glob("results_*.json"))) == 1 + + def test_different_revisions_write_details_to_different_directories(self, mock_datetime): + with tempfile.TemporaryDirectory() as temp_dir: + for revision in ["v1.0", "v2.0"]: + tracker = self._make_tracker(temp_dir, revision, save_details=True) + tracker.details_logger.details = { + "task1": [DetailsLogger.CompiledDetail(hashes=None, truncated=10, padded=5)] + } + tracker.save() + + date_id = mock_datetime.isoformat().replace(":", "-") + model_dir = Path(temp_dir) / "details" / "test" / "model" + assert sorted(path.name for path in model_dir.iterdir()) == ["v1.0", "v2.0"] + for revision in ["v1.0", "v2.0"]: + details_file = model_dir / revision / date_id / f"details_task1_{date_id}.parquet" + assert details_file.is_file() + + def test_missing_revision_falls_back_to_main(self): + """A model config without a `revision` field must still produce a valid path.""" + with tempfile.TemporaryDirectory() as temp_dir: + self._make_tracker(temp_dir, revision=None).save() + + results_dir = Path(temp_dir) / "results" / "test" / "model" / "main" + assert results_dir.is_dir() + assert len(list(results_dir.glob("results_*.json"))) == 1 + + def test_explicit_main_revision_matches_the_fallback(self): + """Evaluating `main` explicitly lands in the same place as not specifying a revision.""" + with tempfile.TemporaryDirectory() as temp_dir: + self._make_tracker(temp_dir, revision="main").save() + + assert (Path(temp_dir) / "results" / "test" / "model" / "main").is_dir() + + def test_results_path_template_supports_revision(self): + with tempfile.TemporaryDirectory() as temp_dir: + tracker = self._make_tracker(temp_dir, "v1.0") + tracker.results_path_template = "{output_dir}/{org}_{model}/{revision}" + tracker.save() + + assert len(list((Path(temp_dir) / "test_model" / "v1.0").glob("results_*.json"))) == 1 + + def test_results_path_template_without_revision_is_unaffected(self): + """Templates written before `{revision}` existed must keep resolving unchanged.""" + with tempfile.TemporaryDirectory() as temp_dir: + tracker = self._make_tracker(temp_dir, "v1.0") + tracker.results_path_template = "{output_dir}/{org}_{model}" + tracker.save() + + assert len(list((Path(temp_dir) / "test_model").glob("results_*.json"))) == 1 + + def test_details_saved_before_this_change_are_still_readable(self): + """Details in the pre-revision layout stay loadable, so upgrading does not strand them.""" + with tempfile.TemporaryDirectory() as temp_dir: + tracker = self._make_tracker(temp_dir, revision="v1.0", save_details=True) + + # Lay details out the way lighteval wrote them before results were keyed on the + # revision: directly under details/{model}/{timestamp}/, with no revision segment. + # The trailing "|0" is the fewshot count, which load_details_datasets strips before + # matching against task_names. + date_id = "2023-01-01T12-00-00.000000" + legacy_dir = Path(temp_dir) / "details" / "test" / "model" / date_id + legacy_dir.mkdir(parents=True) + Dataset.from_dict({"truncated": [10], "padded": [5]}).to_parquet( + str(legacy_dir / f"details_task1|0_{date_id}.parquet") + ) + + loaded = tracker.load_details_datasets(date_id, ["task1"]) + + assert list(loaded.keys()) == ["task1|0"] + assert len(loaded["task1|0"]) == 1 + + def test_legacy_fallback_resolves_the_last_timestamp(self): + """The fallback also covers the "last" alias, not just an explicit timestamp.""" + with tempfile.TemporaryDirectory() as temp_dir: + tracker = self._make_tracker(temp_dir, revision="v1.0", save_details=True) + + model_dir = Path(temp_dir) / "details" / "test" / "model" + for date_id in ["2023-01-01T12-00-00.000000", "2023-06-01T12-00-00.000000"]: + legacy_dir = model_dir / date_id + legacy_dir.mkdir(parents=True) + Dataset.from_dict({"truncated": [10]}).to_parquet( + str(legacy_dir / f"details_task1|0_{date_id}.parquet") + ) + + loaded = tracker.load_details_datasets("last", ["task1"]) + + assert list(loaded.keys()) == ["task1|0"] + + def test_sibling_revision_folder_is_not_mistaken_for_legacy_details(self): + """A revision directory must never be picked up as though it were a timestamp folder. + + `details/{model}/` is the parent of every revision directory, so a naive fallback would + treat `v1.0/` as a timestamp folder when asked for the details of some other revision. + """ + with tempfile.TemporaryDirectory() as temp_dir: + # Only revision-scoped details exist, and none for the revision we ask about. + writer = self._make_tracker(temp_dir, revision="v1.0", save_details=True) + writer.details_logger.details = { + "task1": [DetailsLogger.CompiledDetail(hashes=None, truncated=10, padded=5)] + } + writer.save() + + reader = self._make_tracker(temp_dir, revision="v2.0", save_details=True) + with pytest.raises((FileNotFoundError, ValueError)): + reader.load_details_datasets("last", ["task1"]) + + class TestProperties(unittest.TestCase): def setUp(self): # In setup in case we need to reuse for future tests