diff --git a/src/lighteval/pipeline.py b/src/lighteval/pipeline.py index 1f5da9c14..5549bcc29 100644 --- a/src/lighteval/pipeline.py +++ b/src/lighteval/pipeline.py @@ -415,11 +415,15 @@ def _load_responses_from_details(self): self.pipeline_parameters.load_responses_from_details_date_id, tasks_names ) - for _, dataset in tqdm(details_datasets.items(), desc="Loading responses from details for tasks"): + # Responses of all tasks are concatenated, in the task order used to build `self.sampling_docs`, + # as both are zipped together when computing the metrics + ordered_task_names = sorted(details_datasets.keys(), key=tasks_names.index) + + for task_name in tqdm(ordered_task_names, desc="Loading responses from details for tasks"): for sampling_method in sampling_methods: - model_responses[sampling_method] = [ - ModelResponse(**model_response["model_response"]) for model_response in dataset - ] + model_responses.setdefault(sampling_method, []).extend( + ModelResponse(**model_response["model_response"]) for model_response in details_datasets[task_name] + ) return model_responses diff --git a/tests/unit/pipeline/test_load_responses_from_details.py b/tests/unit/pipeline/test_load_responses_from_details.py new file mode 100644 index 000000000..2265b628c --- /dev/null +++ b/tests/unit/pipeline/test_load_responses_from_details.py @@ -0,0 +1,149 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import tempfile +import unittest +from typing import Optional +from unittest.mock import patch + +from lighteval.logging.evaluation_tracker import EvaluationTracker +from lighteval.metrics.metrics import Metrics +from lighteval.models.dummy.dummy_model import DummyModel, DummyModelConfig +from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters +from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig +from lighteval.tasks.registry import Registry +from lighteval.tasks.requests import Doc, SamplingMethod + + +class TestLoadResponsesFromDetails(unittest.TestCase): + """Test suite for reloading model responses from previously saved details.""" + + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.task_names = ["task_a|0", "task_b|0"] + + self.docs = { + task_name: [ + Doc( + task_name=task_name, + query=f"{task_name} query {i}", + choices=["4"], + gold_index=[0], + instruction="", + sampling_methods=[SamplingMethod.GENERATIVE], + ) + for i in range(2) + ] + for task_name in self.task_names + } + + def _task_config(self, task_name: str) -> LightevalTaskConfig: + return LightevalTaskConfig( + name=task_name.split("|")[0], + prompt_function=lambda x: x, + hf_repo="test_repo", + hf_subset="default", + metrics=[Metrics.exact_match], + hf_avail_splits=["test"], + evaluation_splits=["test"], + few_shots_split=None, + few_shots_select=None, + generation_size=10, + stop_sequence=["\n"], + num_fewshots=0, + ) + + def _build_pipeline(self) -> Pipeline: + docs_per_task = self.docs + + class FakeTask(LightevalTask): + def __init__(self, config, task_name): + super().__init__(config=config) + self._docs = docs_per_task[task_name] + + def get_docs(self, max_samples=None): + return self._docs + + class FakeRegistry(Registry): + def __init__( + self, tasks: Optional[str] = None, load_multilingual: bool = False, custom_tasks: Optional[str] = None + ): + self.tasks_list = list(docs_per_task.keys()) + + def load_tasks(inner_self): + return { + task_name: FakeTask(config=self._task_config(task_name), task_name=task_name) + for task_name in docs_per_task + } + + with patch("lighteval.pipeline.Registry", FakeRegistry), patch.object(LightevalTask, "load_datasets"): + return Pipeline( + tasks="|".join(self.task_names), + pipeline_parameters=PipelineParameters( + launcher_type=ParallelismManager.NONE, + load_responses_from_details_date_id="2024-01-01T00-00-00.000000", + ), + evaluation_tracker=EvaluationTracker(output_dir=self.temp_dir), + model=DummyModel(DummyModelConfig(seed=42)), + ) + + def test_responses_of_all_tasks_are_loaded(self): + """All tasks' responses are reloaded, in the same order as the pipeline documents.""" + pipeline = self._build_pipeline() + + details_datasets = { + task_name: [ + {"model_response": {"text": [f"{task_name} prediction {i}"]}} for i in range(len(self.docs[task_name])) + ] + for task_name in self.task_names + } + + with patch.object(pipeline.evaluation_tracker, "load_details_datasets", return_value=details_datasets): + model_responses = pipeline._load_responses_from_details() + + expected = [ + f"{task_name} prediction {i}" for task_name in self.task_names for i in range(len(self.docs[task_name])) + ] + assert [response.text[0] for response in model_responses[SamplingMethod.GENERATIVE]] == expected + + def test_responses_follow_the_document_order(self): + """Details are realigned on the pipeline task order, whatever order they were loaded in.""" + pipeline = self._build_pipeline() + + details_datasets = { + task_name: [ + {"model_response": {"text": [f"{task_name} prediction {i}"]}} for i in range(len(self.docs[task_name])) + ] + for task_name in reversed(self.task_names) + } + + with patch.object(pipeline.evaluation_tracker, "load_details_datasets", return_value=details_datasets): + model_responses = pipeline._load_responses_from_details() + + loaded_texts = [response.text[0] for response in model_responses[SamplingMethod.GENERATIVE]] + assert loaded_texts == [ + f"{task_name} prediction {i}" for task_name in self.task_names for i in range(len(self.docs[task_name])) + ] + + +if __name__ == "__main__": + unittest.main()