diff --git a/src/lighteval/tasks/tasks/evalplus/__init__.py b/src/lighteval/tasks/tasks/evalplus/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/lighteval/tasks/tasks/evalplus/build_data.py b/src/lighteval/tasks/tasks/evalplus/build_data.py new file mode 100644 index 000000000..79ae2da57 --- /dev/null +++ b/src/lighteval/tasks/tasks/evalplus/build_data.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Build/publish the EvalPlus datasets used by the ``humaneval_plus`` / ``mbpp_plus`` tasks. + +Reads the canonical EvalPlus HF datasets (``evalplus/humanevalplus`` and +``evalplus/mbppplus``, split ``test``) and normalizes both into a single uniform +schema so the task's ``record_to_sample`` stays trivial and the Arrow loader never +has to unify heterogeneous fields: + + kind "humaneval" | "mbpp" + task_id original task id + prompt instruct source text (HumanEval: signature+docstring; + MBPP: natural-language description) + entry_point function name the model must define (derived from ``code`` for MBPP) + test EvalPlus ``check(candidate)`` harness (the augmented "+" tests) + example MBPP only: one example assert (conveys the signature); "" otherwise + canonical_solution reference implementation (for local sanity checks / tests) + +With ``--push-to-hub`` it uploads a multi-config HF dataset (config ``humaneval_plus`` +and ``mbpp_plus``, split ``train``). + +Usage: + python build_data.py --collection humaneval_plus --out-dir ./evalplus_data + python build_data.py --collection mbpp_plus --push-to-hub ezosa/evalplus + python build_data.py --collection all --push-to-hub ezosa/evalplus +""" + +import argparse +import json +import os +import re + + +_SOURCES = { + "humaneval_plus": ("evalplus/humanevalplus", "test"), + "mbpp_plus": ("evalplus/mbppplus", "test"), +} + + +def _entry_point_from_code(code: str) -> str: + m = re.findall(r"^\s*def\s+(\w+)\s*\(", code, re.MULTILINE) + return m[-1] if m else "" + + +def build_collection(collection: str) -> list: + from datasets import load_dataset + + repo, split = _SOURCES[collection] + ds = load_dataset(repo, split=split) + rows = [] + if collection == "humaneval_plus": + for r in ds: + rows.append( + { + "kind": "humaneval", + "task_id": r["task_id"], + "prompt": r["prompt"], + "entry_point": r["entry_point"], + "test": r["test"], + "example": "", + "canonical_solution": r["prompt"] + r["canonical_solution"], + } + ) + else: # mbpp_plus + for r in ds: + entry = _entry_point_from_code(r.get("code", "")) + test_list = r.get("test_list") or [] + example = test_list[0] if test_list else "" + rows.append( + { + "kind": "mbpp", + "task_id": str(r["task_id"]), + "prompt": r["prompt"], + "entry_point": entry, + "test": r["test"], + "example": example, + "canonical_solution": r.get("code", ""), + } + ) + return rows + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--collection", choices=[*sorted(_SOURCES), "all"], default="all") + ap.add_argument("--out-dir", default="./evalplus_data") + ap.add_argument("--push-to-hub", metavar="REPO", default=None, help="e.g. ezosa/evalplus") + args = ap.parse_args() + + collections = sorted(_SOURCES) if args.collection == "all" else [args.collection] + os.makedirs(args.out_dir, exist_ok=True) + for collection in collections: + rows = build_collection(collection) + with open(os.path.join(args.out_dir, f"{collection}.jsonl"), "w") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + missing = [r["task_id"] for r in rows if not r["entry_point"]] + print(f"{collection:16s} {len(rows):5d} rows" + (f" WARN missing entry_point: {missing}" if missing else "")) + if args.push_to_hub: + from datasets import Dataset + + Dataset.from_list(rows).push_to_hub(args.push_to_hub, config_name=collection, split="train") + print(f"pushed config={collection} -> {args.push_to_hub}") + + +if __name__ == "__main__": + main() diff --git a/src/lighteval/tasks/tasks/evalplus/execution.py b/src/lighteval/tasks/tasks/evalplus/execution.py new file mode 100644 index 000000000..ffba59b60 --- /dev/null +++ b/src/lighteval/tasks/tasks/evalplus/execution.py @@ -0,0 +1,203 @@ +"""Execution grader for the HumanEval+/MBPP+ (EvalPlus) tasks. + +Grades a model by *running* the code it produced against the benchmark's unit +tests. The public surface deliberately mirrors LiveCodeBench's grader +(``lighteval.tasks.tasks.lcb.codegen_metrics``) so the shared TensorWave network +sandbox (``sandbox_grading.py`` / ``sandboxed_evals.GRADER_PATCHES``) can wrap it +with a single generic wrapper: + + - ``run_test(sample, test=, timeout=...)`` runs one problem's tests + and returns ``list[int]`` (``[1]`` pass, ``[0]`` fail, ``[-1]`` sentinel/timeout). + - ``check_correctness(sample, generation, timeout)`` spawns a + ``multiprocessing.Process`` running ``run_test`` and reads its result via a + ``Manager`` proxy — identical control flow to the sandbox wrapper, so behaviour + is the same whether or not the sandbox monkey-patch is installed. + +The ``sample`` dict carries the unit-test harness and entry point; it also carries +a one-element ``input_output`` (``{"inputs": [""], "outputs": [""]}``) purely so the +generic sandbox wrapper's timeout/sentinel math (which indexes +``sample["input_output"]["inputs"]``) works unchanged. Each EvalPlus problem is a +single ``check(entry_point)`` invocation, hence one "input". + +``reliability_guard`` is defensive hardening (disables destructive os/shutil/… +builtins), NOT a security boundary — real isolation comes from the network/user +namespace applied by the sandbox wrapper. +""" + +import contextlib +import faulthandler +import io +import json +import multiprocessing +import os +import platform +import re +import signal + + +_SINGLE_INPUT_OUTPUT = json.dumps({"inputs": [""], "outputs": [""]}) + + +def extract_code(model_output: str) -> str: + """Return the code from the model's (post-reasoning) final answer. + + Prefers the last fenced ``python`` block, then any last fenced block, then + falls back to the raw text. With ``reasoning_parser=qwen3`` the completion is + already the final answer (reasoning is separated by vLLM), so this runs on the + answer text, not the ```` trace. + """ + if not model_output: + return "" + blocks = re.findall(r"```(?:python|py)?\s*\n(.*?)```", model_output, re.DOTALL) + if blocks: + return blocks[-1].strip() + return model_output.strip() + + +@contextlib.contextmanager +def _time_limit(seconds: float): + def _handler(signum, frame): + raise TimeoutError("timed out") + + signal.setitimer(signal.ITIMER_REAL, seconds) + signal.signal(signal.SIGALRM, _handler) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +@contextlib.contextmanager +def _swallow_io(): + stream = io.StringIO() + with contextlib.redirect_stdout(stream), contextlib.redirect_stderr(stream): + yield + + +def _build_program(sample: dict, generation: str) -> str: + """Assemble the program: model code + test harness + a check(entry_point) call.""" + harness = sample.get("test", "") + entry = sample.get("entry_point", "") + return f"{generation}\n\n{harness}\n\ncheck({entry})\n" + + +def _unsafe_execute(program: str, timeout: float) -> bool: + reliability_guard() + try: + exec_globals: dict = {"__name__": "__main__"} + with _swallow_io(): + with _time_limit(timeout): + exec(compile(program, "", "exec"), exec_globals) + return True + except BaseException: + return False + + +def run_test(sample: dict, test=None, timeout: int = 15) -> list: + """Run one EvalPlus problem. ``test`` is the model-generated code (generation). + + Returns ``[1]`` if the unit tests pass, ``[0]`` otherwise. Signature/return + shape match lcb's ``run_test`` so the shared sandbox wrapper can drive it. + """ + generation = test or "" + program = _build_program(sample, generation) + return [1 if _unsafe_execute(program, timeout) else 0] + + +def check_correctness(sample: dict, generation: str, timeout: int) -> list: + """Run ``run_test`` in a child process; return its ``list[int]`` result. + + Mirrors lcb's ``check_correctness`` (Manager proxy handoff, timeout from + ``sample["input_output"]["inputs"]``, ``[-1]`` sentinel) so the sandbox + monkey-patch is a drop-in. Used verbatim when the sandbox is disabled. + """ + + def _temp_run(sample, generation, result): + result.append(run_test(sample, test=generation, timeout=timeout)) + + manager = multiprocessing.Manager() + result = manager.list() + p = multiprocessing.Process(target=_temp_run, args=(sample, generation, result)) + p.start() + p.join(timeout=(timeout + 1) * len(json.loads(sample["input_output"])["inputs"]) + 5) + if p.is_alive(): + p.kill() + if not result: + in_outs = json.loads(sample["input_output"]) + result = [[-1 for _ in range(len(in_outs["inputs"]))]] + return result[0] + + +def make_sample(test: str, entry_point: str) -> dict: + """Build the ``sample`` dict passed to ``check_correctness`` for one problem.""" + return {"input_output": _SINGLE_INPUT_OUTPUT, "test": test, "entry_point": entry_point} + + +def reliability_guard(maximum_memory_bytes=None): + """Disable destructive builtins/OS calls before exec'ing untrusted code. + + Vendored from OpenAI human-eval. NOT a security sandbox (its own warning) — + the real boundary is the CLONE_NEWUSER|CLONE_NEWNET unshare applied per-sample + by the TensorWave sandbox wrapper. + """ + import builtins + import shutil + import subprocess + + if maximum_memory_bytes is not None: + import resource + + resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) + resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) + if platform.uname().system != "Darwin": + resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) + + faulthandler.disable() + + builtins.exit = None + builtins.quit = None + + os.environ["OMP_NUM_THREADS"] = "1" + + os.kill = None + os.system = None + os.putenv = None + os.remove = None + os.removedirs = None + os.rmdir = None + os.fchdir = None + os.setuid = None + os.fork = None + os.forkpty = None + os.killpg = None + os.rename = None + os.renames = None + os.truncate = None + os.replace = None + os.unlink = None + os.fchmod = None + os.fchown = None + os.chmod = None + os.chown = None + os.chroot = None + os.lchflags = None + os.lchmod = None + os.lchown = None + os.getcwd = None + os.chdir = None + + shutil.rmtree = None + shutil.move = None + shutil.chown = None + + subprocess.Popen = None + + __builtins__["help"] = None + + import sys + + sys.modules["ipdb"] = None + sys.modules["joblib"] = None + sys.modules["resource"] = None + sys.modules["psutil"] = None + sys.modules["tkinter"] = None diff --git a/src/lighteval/tasks/tasks/evalplus/main.py b/src/lighteval/tasks/tasks/evalplus/main.py new file mode 100644 index 000000000..7c853cc6e --- /dev/null +++ b/src/lighteval/tasks/tasks/evalplus/main.py @@ -0,0 +1,130 @@ +""" +name: +EvalPlus (HumanEval+ / MBPP+) + +dataset: +ezosa/evalplus + +abstract: +HumanEval+ and MBPP+ (EvalPlus) code-generation benchmarks evaluated as native +inspect_ai tasks. The model is prompted (instruct-style) to implement a Python +function; the code is extracted from its final answer and graded by *executing* +it against EvalPlus's augmented unit tests. pass@1 is estimated over N samples +via the wrapper's epochs (mean epoch-reducer); no judge model. + +Reasoning is left ON: the checkpoint chat template opens ``, vLLM's +`reasoning_parser=qwen3` separates the reasoning trace, and the scorer extracts +code from the post-reasoning answer. + +Code execution is unsafe and MUST run under the TensorWave sandbox +(`eval_lighteval_sandboxed.sbatch` + `sandbox_grading.py`), which wraps this +task's `execution.check_correctness` with a per-sample network/user namespace. +The task names are registered in `sandboxed_evals.SANDBOXED_TASK_PREFIXES` +(`humaneval`, `mbpp`) and its grader in `GRADER_PATCHES`. + +Exposes: humaneval_plus, mbpp_plus + +languages: +english + +tags: +code-generation, code, humaneval, mbpp, evalplus, sandboxed +""" + +import asyncio +import os + +from inspect_ai.dataset import Sample +from inspect_ai.model import ChatMessageUser +from inspect_ai.scorer import Score, accuracy, scorer, stderr +from inspect_ai.solver import generate + +from lighteval.tasks.lighteval_task import LightevalTaskConfig +from lighteval.tasks.tasks.evalplus import execution + + +# Wall-clock ceiling (seconds) for executing one problem's generated code. +_EXEC_TIMEOUT = 20 + + +def _humaneval_message(record): + return ( + "Complete the following Python function. Return the complete function, " + "including the signature, in a single ```python code block.\n\n" + f"```python\n{record['prompt']}\n```" + ) + + +def _mbpp_message(record): + example = record.get("example") or "" + example_block = f"\n\nYour function must pass this test:\n```python\n{example}\n```" if example else "" + return ( + f"{record['prompt']}\n\n" + f"Write a single Python function named `{record['entry_point']}` that solves this. " + f"Return the complete function in a single ```python code block.{example_block}" + ) + + +def record_to_sample(record): + kind = record["kind"] + message = _humaneval_message(record) if kind == "humaneval" else _mbpp_message(record) + return Sample( + input=[ChatMessageUser(content=message)], + metadata={ + "task_id": record["task_id"], + "kind": kind, + "entry_point": record["entry_point"], + "test": record["test"], + }, + ) + + +@scorer(metrics=[accuracy(), stderr()]) +def code_exec_scorer(): + async def score(state, target): + md = state.metadata + code = execution.extract_code(state.output.completion or "") + if not code: + return Score(value=0.0, explanation="no code block in answer") + sample = execution.make_sample(md["test"], md["entry_point"]) + # Reference the grader via the module attribute so the sandbox monkey-patch + # (sandbox_grading.install) applies when running sandboxed. + result = await asyncio.to_thread(execution.check_correctness, sample, code, _EXEC_TIMEOUT) + passed = bool(result) and all(x == 1 for x in result) + return Score( + value=1.0 if passed else 0.0, + answer=code, + explanation=f"{md['task_id']}: tests {'passed' if passed else 'failed'} ({result})", + ) + + return score + + +# One multi-config HF dataset (split="train"), config per benchmark. Override +# with EVALPLUS_HF_REPO to point at a mirror (e.g. under an org). +_HF_REPO = os.environ.get("EVALPLUS_HF_REPO", "ezosa/evalplus") + +_TASKS = ["humaneval_plus", "mbpp_plus"] + + +def _make_config(name, subset, repo): + return LightevalTaskConfig( + name=name, + prompt_function=None, + hf_repo=repo, + hf_subset=subset, + hf_avail_splits=["train"], + evaluation_splits=["train"], + few_shots_split=None, + few_shots_select=None, + metrics=[], + generation_size=32768, + stop_sequence=[], + version=1, + sample_fields=record_to_sample, + solver=[generate(cache=True)], + scorer=code_exec_scorer(), + ) + + +TASKS_TABLE = [_make_config(name, name, _HF_REPO) for name in _TASKS] diff --git a/tests/unit/tasks/test_evalplus.py b/tests/unit/tasks/test_evalplus.py new file mode 100644 index 000000000..d98e82ef7 --- /dev/null +++ b/tests/unit/tasks/test_evalplus.py @@ -0,0 +1,59 @@ +"""Unit tests for the EvalPlus (HumanEval+/MBPP+) execution grader. + +Pure-stdlib (multiprocessing + signal); no optional deps. Exercises code +extraction and the execute-and-grade path that the sandbox wrapper drives. +""" + +from lighteval.tasks.tasks.evalplus import execution + + +# A tiny HumanEval-style problem: implement add(a, b). +TEST_HARNESS = "def check(candidate):\n assert candidate(2, 3) == 5\n assert candidate(0, 0) == 0\n" +ENTRY_POINT = "add" +CORRECT = "def add(a, b):\n return a + b" +WRONG = "def add(a, b):\n return a - b" + + +def _grade(code: str) -> bool: + sample = execution.make_sample(TEST_HARNESS, ENTRY_POINT) + result = execution.check_correctness(sample, code, timeout=10) + return bool(result) and all(x == 1 for x in result) + + +def test_correct_solution_passes(): + assert _grade(CORRECT) is True + + +def test_wrong_solution_fails(): + assert _grade(WRONG) is False + + +def test_exception_fails(): + assert _grade("def add(a, b):\n raise ValueError('boom')") is False + + +def test_timeout_fails(): + assert _grade("def add(a, b):\n while True:\n pass") is False + + +def test_missing_function_fails(): + assert _grade("x = 1") is False + + +def test_extract_code_prefers_python_fence(): + text = "Here is the solution:\n```python\ndef add(a, b):\n return a + b\n```\nDone." + assert execution.extract_code(text) == "def add(a, b):\n return a + b" + + +def test_extract_code_post_reasoning_answer(): + # With reasoning_parser=qwen3 the completion is the final answer only. + text = "The function adds the two inputs.\n```python\ndef add(a, b):\n return a + b\n```" + assert _grade(execution.extract_code(text)) is True + + +def test_extract_code_falls_back_to_raw_text(): + assert execution.extract_code("def add(a, b):\n return a + b").startswith("def add") + + +def test_extract_code_empty(): + assert execution.extract_code("") == ""