fix(tasks): skip few-shot reuse warning for zero-shot tasks

src/lighteval/tasks/lighteval_task.py

@@ -278,7 +278,8 @@ def get_first_possible_fewshot_splits(self, available_splits: ListLike[str]) ->
         if len(stored_splits) > 0:
             return stored_splits[0]
 
-        logger.warning(f"Careful, the task {self.name} is using evaluation data to build the few shot examples.")
+        if self.config.num_fewshots > 0:
+            logger.warning(f"Careful, the task {self.name} is using evaluation data to build the few shot examples.")
         return None
 
     def _get_docs_from_split(self, splits: list[str], few_shots=False) -> list[Doc]:

tests/unit/tasks/test_lighteval_task.py

@@ -21,6 +21,10 @@
 # SOFTWARE.
 
 
+import logging
+
+import pytest
+
 from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
@@ -29,6 +33,27 @@ def dummy_prompt_function(item, task_name):
     return Doc(query=item["text"], choices=["A", "B"], gold_index=0, task_name=task_name)
 
 
+@pytest.mark.parametrize(("num_fewshots", "warning_expected"), [(0, False), (1, True)])
+def test_evaluation_split_fewshot_warning(caplog, num_fewshots, warning_expected):
+    cfg = LightevalTaskConfig(
+        name="test_only_split",
+        prompt_function=dummy_prompt_function,
+        hf_repo="unused",
+        hf_subset="default",
+        hf_avail_splits=["test"],
+        evaluation_splits=["test"],
+        metrics=[],
+        num_fewshots=num_fewshots,
+    )
+
+    with caplog.at_level(logging.WARNING, logger="lighteval.tasks.lighteval_task"):
+        task = LightevalTask(cfg)
+
+    warning = "Careful, the task test_only_split is using evaluation data to build the few shot examples."
+    assert task.fewshot_split is None
+    assert (warning in caplog.messages) is warning_expected
+
+
 def test_revision_check():
     # Test with a different revision
     cfg_with_revision = LightevalTaskConfig(