src/eval/test_runner.py - master - Watashicuvu/agentic-tools

src/eval/test_runner.py
872 строки34 KB
Your Name
added come roles
28 май 2026, 15:16
28 май 2026, 15:16745363b
"""Eval test runner — orchestrator for eval sessions.

Loads tasks from tasks.json, executes each through the LLM agent,
evaluates results (Lua + LLM), and collects EvalResult instances.
"""

from __future__ import annotations

import json
import logging
import re
import tempfile
import time
import traceback
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional

from src.eval.lua_evaluator import LuaCodeEvaluator, LuaEvaluator
from src.eval.code_evaluator import CodeEvalResult
from src.eval.models import EvalResult, EvalTask


@dataclass(frozen=True)
class ExecutionResult:
    """Result of executing a single task.

    Attributes:
        code: Generated code from the LLM.
        tools_called: List of tool names that were invoked.
        tool_call_count: Total number of tool calls.
        execution_time_ms: Time spent executing the task.
        error: Optional error message if execution failed.
        extra: Optional dict for detailed debugging info (tool_calls log, etc.).
    """

    code: str = ""
    tools_called: list[str] = field(default_factory=list)
    tool_call_count: int = 0
    execution_time_ms: float = 0.0
    error: Optional[str] = None
    extra: Optional[dict] = None


logger = logging.getLogger(__name__)

# Паттерны проектных ошибок (баги Python-инфраструктуры)
_PROJECT_ERROR_PATTERNS = [
    re.compile(r"ImportError"),
    re.compile(r"ModuleNotFoundError"),
    re.compile(r"KeyError"),
    re.compile(r"FileNotFoundError"),
    re.compile(r"TypeError"),
    re.compile(r"NameError"),
    re.compile(r"cannot import name"),
    re.compile(r"No module named"),
    re.compile(r"has no attribute"),
]


def _is_project_error(error_text: str) -> bool:
    """Проверяет, является ли ошибка проектной (баг инфраструктуры)."""
    for pattern in _PROJECT_ERROR_PATTERNS:
        if pattern.search(error_text):
            return True
    return False


class EvalTestRunner:
    """Orchestrates eval task execution and evaluation.

    Supports two modes:
    1. Simple mode: chat_completion without tools (no MCP URL)
    2. Full agent mode: EvalAgent with ToolOrchestrator + LLMStreamingClient (MCP URL)

    Args:
        llm_client: AsyncSmartOpenAI for code generation.
        llm_model: Model for code generation (default: "gpt-4o").
        llm_eval_client: Optional AsyncSmartOpenAI for quality scoring.
        llm_eval_model: Model for evaluation (default: "gpt-4o").
        use_llm_eval: Whether to run LLM-based quality scoring.
        lua_timeout: Timeout for Lua execution in seconds.
        mcp_url: Optional MCP Server URL for full agent pipeline.
        iron_user_llm_client: Optional AsyncSmartOpenAI for IronUser (answering agent questions).
        iron_user_model: Model for IronUser responses (default: "gpt-4o").
    """

    def __init__(
        self,
        llm_client: Any,
        llm_model: str = "gpt-4o",
        llm_eval_client: Optional[Any] = None,
        llm_eval_model: str = "gpt-4o",
        use_llm_eval: bool = True,
        lua_timeout: int = 30,
        mcp_url: Optional[str] = None,
        iron_user_llm_client: Optional[Any] = None,
        iron_user_model: str = "gpt-4o",
        use_swarm: bool = False,
        use_langgraph: bool = False,
        project_root: Optional[str] = None,
        eval_dir: Optional[str] = None,
    ) -> None:
        self.llm_client = llm_client
        self.llm_model = llm_model
        self.llm_eval_client = llm_eval_client
        self.llm_eval_model = llm_eval_model
        self.use_llm_eval = use_llm_eval
        self.lua_timeout = lua_timeout
        self.mcp_url = mcp_url
        self.iron_user_llm_client = iron_user_llm_client
        self.iron_user_model = iron_user_model
        self.use_swarm = use_swarm
        self.use_langgraph = use_langgraph
        self.project_root = project_root
        self.eval_dir = eval_dir

        # Глобальный счётчик LLM вызовов (cross-task) — для защиты от зацикливания
        self._global_llm_calls = 0
        self._global_tool_calls = 0
        self.max_global_llm_calls = 50  # Лимит на всю сессию

    @staticmethod
    def load_tasks(tasks_path: str) -> list[EvalTask]:
        """Load tasks from a JSON file.

        Supports two formats:
        1. New format: description + expected_behavior + reference_code
        2. Real tasks.json format: title + description + expected_lua + context

        Args:
            tasks_path: Path to tasks.json file.

        Returns:
            List of EvalTask instances.

        Raises:
            FileNotFoundError: If tasks file doesn't exist.
            json.JSONDecodeError: If tasks file is invalid.
        """
        path = Path(tasks_path)
        if not path.exists():
            raise FileNotFoundError(f"Tasks file not found: {tasks_path}")

        with open(path, encoding="utf-8") as f:
            raw_tasks = json.load(f)

        tasks = []
        for raw in raw_tasks:
            # Detect format and use appropriate parser
            if "title" in raw or "expected_lua" in raw or "context" in raw:
                # Real tasks.json format
                task = EvalTask.from_tasks_json_entry(raw)
            else:
                # New format
                task = EvalTask(
                    id=raw["id"],
                    description=raw["description"],
                    expected_behavior=raw["expected_behavior"],
                    reference_code=raw.get("reference_code"),
                    lint_rules=raw.get("lint_rules", ["luacheck"]),
                    mode=raw.get("mode", "batch"),
                )
            tasks.append(task)

        return tasks

    async def run_tasks(
        self,
        tasks: list[EvalTask],
        progress_callback: Optional[callable] = None,
    ) -> list[EvalResult]:
        """Execute all tasks and collect results.

        Args:
            tasks: List of EvalTask instances.
            progress_callback: Optional callback(task_index, total, result).

        Returns:
            List of EvalResult instances.
        """
        results = []
        total = len(tasks)

        for idx, task in enumerate(tasks):
            from src.services.telemetry import telemetry, EventType

            telemetry.emit(
                EventType.STEP_INFO,
                f"Running task {idx + 1}/{total}",
                {"task_id": task.id, "description": task.description[:50]},
            )

            result = await self._run_single_task(task)
            results.append(result)

            if progress_callback:
                progress_callback(idx, total, result)

        return results

    async def _run_single_task(self, task: EvalTask) -> EvalResult:
        """Execute and evaluate a single task with retry-loop.

        При syntax/lint/run ошибке перегенерирует код с feedback.

        Args:
            task: EvalTask to execute.

        Returns:
            EvalResult with all evaluation metrics.
        """
        import time as time_module
        from src.eval.iron_user import IronUser
        from src.eval.models import ClarificationEvent
        import dataclasses

        # Максимум retry-попыток при syntax/lint/run ошибках
        MAX_RETRIES = 4

        start = time_module.monotonic()
        last_error_log = None
        last_execution: Optional[ExecutionResult] = None  # Для fallback при лимите

        # Проверка глобального лимита
        if self._global_llm_calls >= self.max_global_llm_calls:
            logger.warning(f"[GLOBAL_LIMIT] LLM calls limit reached ({self._global_llm_calls}), marking task as fail")
            return EvalResult(
                task_id=task.id,
                role=task.role,
                status="failed",
                execution_time_ms=0,
                tools_called=[],
                tool_call_count=0,
                error=f"Global LLM calls limit reached ({self.max_global_llm_calls})",
            )

        for attempt in range(MAX_RETRIES + 1):
            # Если retry, добавляем feedback в task
            if attempt > 0 and last_error_log:
                feedback = f"\n\n⚠️ ПОПЫТКА {attempt + 1}/{MAX_RETRIES + 1}: Предыдущая попытка вернула ошибку:\n{last_error_log}\n\nИСПРАВЬ КОД!"
                task = dataclasses.replace(task, error_feedback=feedback)
                logger.info(f"[RETRY] Attempt {attempt + 1}/{MAX_RETRIES + 1} for task {task.id}")

            # Create IronUser if task requires clarification
            iron_user: Optional[IronUser] = None
            clarifications: list[ClarificationEvent] = []
            questions_asked = 0
            questions_answered = 0
            ambiguity_detected = False

            if task.requires_clarification and self.iron_user_llm_client:
                iron_user = IronUser(
                    llm_client=self.iron_user_llm_client,
                    model=self.iron_user_model,
                    task_context=task.context or {},
                    expected_lua=task.expected_lua or task.reference_code or "",
                    task_description=task.description,
                )

            # Step 1: Generate code via LLM
            self._global_llm_calls += 1
            if self._global_llm_calls >= self.max_global_llm_calls:
                logger.warning(f"[GLOBAL_LIMIT] Limit reached at task {task.id} attempt {attempt}, using last code")
                # Используем последний сохранённый код (если есть) или пустой
                code = last_execution.code if last_execution else ""
                elapsed = (time_module.monotonic() - start) * 1000
                return EvalResult(
                    task_id=task.id,
                    role=task.role,
                    status="failed",
                    execution_time_ms=round(elapsed, 2),
                    tools_called=last_execution.tools_called if last_execution else [],
                    tool_call_count=last_execution.tool_call_count if last_execution else 0,
                    error=f"Global LLM calls limit reached ({self.max_global_llm_calls}) during task {task.id}",
                )

            execution = await self._generate_code(task, iron_user=iron_user)
            last_execution = execution  # Сохраняем для fallback

            # === CHECK FOR QUESTION (для LangGraph и EvalAgent) ===
            # LangGraph НЕ поддерживает clarification node — проверяем ответ на QUESTION
            # Проверяем code И full_response (LLM может положить вопрос в full_response)
            question_text = None
            if execution.code and execution.code.strip().startswith("QUESTION:"):
                question_text = execution.code[len("QUESTION:"):].strip()
            elif execution.extra and "full_response" in execution.extra:
                full_resp = execution.extra["full_response"]
                if full_resp and "QUESTION:" in full_resp:
                    idx = full_resp.find("QUESTION:")
                    question_text = full_resp[idx + len("QUESTION:"):].strip().split("\n")[0]

            if question_text is not None:
                if iron_user:
                    logger.info(f"[QUESTION] LLM asked: {question_text[:100]}")
                    iron_response = await iron_user.answer(question_text)
                    questions_asked += 1
                    questions_answered += 1
                    ambiguity_detected = True
                    clarifications.append(ClarificationEvent(
                        question=question_text,
                        answer=iron_response.answer,
                        round_num=questions_asked,
                    ))
                    # Перегенерируем код с ответом IronUser
                    feedback = f"\n\nIronUser ответил на ваш вопрос: {iron_response.answer}\n\nТеперь сгенерируйте ФИНАЛЬНЫЙ КОД (без вопросов)!"
                    task = dataclasses.replace(task, error_feedback=feedback)
                    self._global_llm_calls += 1  # Инкремент при регенерации
                    execution = await self._generate_code(task, iron_user=None)  # Второй вызов без iron_user
                    last_execution = execution  # Обновляем fallback
                    logger.info(f"[REGENERATE] Code regenerated after IronUser answer")
                else:
                    logger.warning(f"[QUESTION] LLM asked but no IronUser available: {question_text[:100]}")
                    # Извлекаем код после QUESTION (если есть)
                    execution = ExecutionResult(
                        code="",
                        tools_called=execution.tools_called,
                        tool_call_count=execution.tool_call_count,
                        execution_time_ms=execution.execution_time_ms,
                        error=f"LLM asked question but no IronUser: {question_text[:200]}",
                        extra=execution.extra,
                    )
            # ============================================================

            # Track clarification metrics
            if iron_user:
                questions_asked = iron_user.question_count
                questions_answered = questions_asked
                ambiguity_detected = questions_asked > 0

                conv_history = iron_user._conversation_history
                for i in range(0, len(conv_history) - 1, 2):
                    if i + 1 < len(conv_history):
                        clarifications.append(ClarificationEvent(
                            question=conv_history[i]["content"],
                            answer=conv_history[i + 1]["content"],
                            round_num=i // 2 + 1,
                        ))

            # Если ошибка генерации — не retry (это не syntax/lint/run error)
            if execution.error:
                elapsed = (time_module.monotonic() - start) * 1000

                if _is_project_error(execution.error):
                    logger.error(
                        "PROJECT ERROR — инфраструктурный баг, пайплайн остановлен:\n%s",
                        execution.error,
                    )
                    raise RuntimeError(
                        f"Проектная ошибка eval-инфраструктуры:\n{execution.error}"
                    )

                status = "partial" if (task.requires_clarification and questions_asked > 0) else "failed"

                error_analysis = None
                if self.llm_client:
                    from src.eval.error_analyzer import analyze_task_error
                    extra = execution.extra or {}
                    error_analysis_obj = await analyze_task_error(
                        smart_client=self.llm_client,
                        model=self.llm_model,
                        task_description=task.description,
                        error=execution.error,
                        generated_code=extra.get("generated_code", execution.code),
                        reference_code=task.reference_code or "",
                        tool_calls_log=extra.get("tool_calls"),
                        task_role=task.role,
                    )
                    if error_analysis_obj:
                        error_analysis = error_analysis_obj.model_dump()

                return EvalResult(
                    task_id=task.id,
                    role=task.role,
                    status=status,
                    execution_time_ms=round(elapsed, 2),
                    tools_called=execution.tools_called,
                    tool_call_count=execution.tool_call_count,
                    questions_asked=questions_asked,
                    questions_answered=questions_answered,
                    ambiguity_detected=ambiguity_detected,
                    clarifications=clarifications,
                    error=execution.error,
                    error_analysis=error_analysis,
                )

            # Step 2: Lua evaluation (syntax + lint + run)
            with tempfile.TemporaryDirectory() as temp_dir:
                lua_evaluator = LuaCodeEvaluator(timeout=self.lua_timeout)
                lua_result = await lua_evaluator.evaluate(
                    code=execution.code,
                    work_dir=temp_dir,
                    run_code=True,
                )

                # Проверяем нужно ли retry
                needs_retry = False
                error_detail = None

                # Проверка 0: код пустой или содержит escaped newlines
                if not execution.code.strip() or '\\n' in execution.code:
                    needs_retry = True
                    error_detail = "Code is empty or contains escaped newlines (\\n instead of real newlines)"
                elif not lua_result.code_syntax_ok:
                    needs_retry = True
                    error_detail = f"Syntax error:\n{lua_result.code_syntax_output}"
                elif not lua_result.code_lint_ok:
                    # Lint errors — обычно не retry, но если errors серьёзные
                    lint_errors = "\n".join(lua_result.code_lint_errors[:5])
                    if "error" in lint_errors.lower() and "warning" not in lint_errors.lower():
                        needs_retry = True
                        error_detail = f"Lint errors:\n{lint_errors}"
                elif not lua_result.code_run_ok:
                    needs_retry = True
                    error_detail = f"Runtime error:\n{lua_result.code_run_output}"

                if needs_retry and attempt < MAX_RETRIES:
                    # Сохраняем лог ошибок для feedback
                    last_error_log = error_detail
                    logger.warning(f"[RETRY] {error_detail[:200]}... Retrying...")
                    continue  # Переходим к следующей попытке

                # Успех или максимум retry — формируем результат
                elapsed = (time_module.monotonic() - start) * 1000

                # LLM quality scoring
                llm_score: Optional[float] = None
                llm_feedback: Optional[str] = None
                if self.use_llm_eval and self.llm_eval_client:
                    llm_result = await self._score_code(
                        code=execution.code,
                        task=task,
                        lua_result=lua_result,
                    )
                    llm_score = llm_result.score
                    llm_feedback = llm_result.feedback

                status = self._determine_status(lua_result, llm_score)

                if task.requires_clarification and questions_asked > 0 and status == "passed":
                    status = "partial"

                return EvalResult(
                    task_id=task.id,
                    role=task.role,
                    status=status,
                    execution_time_ms=round(elapsed, 2),
                    tools_called=execution.tools_called,
                    tool_call_count=execution.tool_call_count,
                    code_syntax_ok=lua_result.code_syntax_ok,
                    code_lint_ok=lua_result.code_lint_ok,
                    code_lint_errors=lua_result.code_lint_errors,
                    code_run_ok=lua_result.code_run_ok,
                    code_run_output=lua_result.code_run_output,
                    llm_score=llm_score,
                    llm_feedback=llm_feedback,
                    questions_asked=questions_asked,
                    questions_answered=questions_answered,
                    ambiguity_detected=ambiguity_detected,
                    clarifications=clarifications,
                    error=None,
                )

    async def _generate_code(
        self,
        task: EvalTask,
        iron_user: Optional[Any] = None,
    ) -> ExecutionResult:
        """Generate code using either simple chat_completion or full agent pipeline.

        If mcp_url is set, uses EvalAgent with ToolOrchestrator + LLMStreamingClient.
        Otherwise, falls back to simple chat_completion.

        Args:
            task: EvalTask describing the requirement.
            iron_user: Optional IronUser for answering agent questions.

        Returns:
            ExecutionResult with generated code and metadata.
        """
        import time as time_module

        if self.mcp_url:
            # Full agent pipeline with MCP tools
            return await self._generate_code_with_agent(task, iron_user=iron_user)
        else:
            # Simple chat_completion
            return await self._generate_code_simple(task, iron_user=iron_user)

    async def _generate_code_with_agent(
        self,
        task: EvalTask,
        iron_user: Optional[Any] = None,
    ) -> ExecutionResult:
        """Generate code using EvalAgent (full MCP pipeline).

        Если use_langgraph=True — запускает LangGraph-пайплайн.
        Если use_swarm=True — запускает swarm-пайплайн.
        Иначе — legacy pipeline с ToolOrchestrator + LLMStreamingClient.

        Args:
            task: EvalTask.
            iron_user: Optional IronUser for answering agent questions.

        Returns:
            ExecutionResult with generated code and tool call tracking.
        """
        import time as time_module

        start = time_module.monotonic()

        try:
            # LangGraph пайплайн
            if self.use_langgraph:
                import sys
                print(f"[TEST_RUNNER] use_langgraph=True, calling langgraph for task {task.id}", file=sys.stderr)
                sys.stderr.flush()
                return await self._generate_code_with_langgraph(task, iron_user)

            # Swarm пайплайн
            from src.eval.eval_agent import EvalAgent

            async with EvalAgent(
                smart_client=self.llm_client,
                model=self.llm_model,
                mcp_url=self.mcp_url,
                use_swarm=self.use_swarm,
                project_root=self.project_root,
                eval_dir=self.eval_dir,
            ) as agent:
                agent_result = await agent.generate_code(
                    task_description=task.description,
                    expected_behavior=task.expected_behavior,
                    reference_code=task.reference_code,
                    context=task.context,
                    iron_user=iron_user,
                    role=task.role,
                    requires_clarification=task.requires_clarification,
                )

            elapsed = (time_module.monotonic() - start) * 1000

            if agent_result.error:
                return ExecutionResult(
                    code=agent_result.code,
                    tools_called=[tc.tool_name for tc in agent_result.tool_calls],
                    tool_call_count=agent_result.tool_call_count,
                    execution_time_ms=round(elapsed, 2),
                    error=agent_result.error,
                    # Сохраняем детальную информацию для error analyzer
                    extra={
                        "generated_code": agent_result.code,
                        "full_response": agent_result.full_response,
                        "tool_calls": [
                            {
                                "tool_name": tc.tool_name,
                                "args": tc.args,
                                "success": tc.success,
                                "round_num": tc.round_num,
                            }
                            for tc in agent_result.tool_calls
                        ],
                    },
                )

            return ExecutionResult(
                code=agent_result.code,
                tools_called=[tc.tool_name for tc in agent_result.tool_calls],
                tool_call_count=agent_result.tool_call_count,
                execution_time_ms=round(elapsed, 2),
            )

        except Exception as e:
            elapsed = (time_module.monotonic() - start) * 1000
            import traceback
            full_traceback = traceback.format_exc()
            return ExecutionResult(
                code="",
                tools_called=[],
                tool_call_count=0,
                execution_time_ms=round(elapsed, 2),
                error=f"Agent generation failed: {e}\n\n{full_traceback}",
            )

    async def _generate_code_with_langgraph(
        self,
        task: EvalTask,
        iron_user: Optional[Any] = None,
    ) -> ExecutionResult:
        """Generate code using LangGraph pipeline с LangChain LLM."""
        print(f"[LANGGRAPH_FN] Starting langgraph generation for task {task.id}")
        import time as time_module
        from src.eval.langgraph_eval_shim import EvalLangGraphShim
        from src.swarm.langgraph.langchain_llm_client import LangChainLLMClient

        start = time_module.monotonic()

        try:
            # Создаём LangChain LLM клиент вместо smart_client
            langchain_client = LangChainLLMClient(
                model=self.llm_model,
            )

            shim = EvalLangGraphShim(
                smart_client=langchain_client,  # LangChain вместо smart_client
                model=self.llm_model,
                mcp_url=self.mcp_url,
                project_root=self.project_root,
                eval_dir=self.eval_dir,
            )
            result = await shim.run_task(task)

            elapsed = (time_module.monotonic() - start) * 1000

            if result.error:
                return ExecutionResult(
                    code=result.code,
                    tools_called=[tc.tool_name for tc in result.tool_calls],
                    tool_call_count=result.tool_call_count,
                    execution_time_ms=round(elapsed, 2),
                    error=result.error,
                    extra={
                        "generated_code": result.code,
                        "full_response": result.full_response,
                        "tool_calls": [
                            {
                                "tool_name": tc.tool_name,
                                "args": tc.args,
                                "success": tc.success,
                                "round_num": tc.round_num,
                            }
                            for tc in result.tool_calls
                        ],
                    },
                )

            return ExecutionResult(
                code=result.code,
                tools_called=[tc.tool_name for tc in result.tool_calls],
                tool_call_count=result.tool_call_count,
                execution_time_ms=round(elapsed, 2),
                extra={
                    "generated_code": result.code,
                    "full_response": result.full_response,
                    "tool_calls": [
                        {
                            "tool_name": tc.tool_name,
                            "args": tc.args,
                            "success": tc.success,
                            "round_num": tc.round_num,
                        }
                        for tc in result.tool_calls
                    ],
                },
            )

        except Exception as e:
            elapsed = (time_module.monotonic() - start) * 1000
            import traceback
            full_traceback = traceback.format_exc()
            return ExecutionResult(
                code="",
                tools_called=[],
                tool_call_count=0,
                execution_time_ms=round(elapsed, 2),
                error=f"LangGraph generation failed: {e}\n\n{full_traceback}",
            )

    async def _generate_code_simple(
        self,
        task: EvalTask,
        iron_user: Optional[Any] = None,
    ) -> ExecutionResult:
        """Generate code using simple chat_completion (no MCP tools).

        If iron_user is provided and task requires clarification, the agent
        may ask questions before generating code.

        Args:
            task: EvalTask.
            iron_user: Optional IronUser for answering agent questions.

        Returns:
            ExecutionResult with generated code.
        """
        import time as time_module
        import re as re_module

        start = time_module.monotonic()

        system_prompt = (
            "You are an expert Lua programmer. Write clean, efficient Lua code "
            "that fulfills the given task. Output ONLY the code, no explanations. "
            "Use proper error handling and follow Lua best practices.\n\n"
            "IMPORTANT: If the task is ambiguous and you need clarification, you may "
            "ask a question by starting your response with 'QUESTION: ' followed by "
            "your question. Otherwise, output only the Lua code."
        )

        user_prompt = (
            f"Task: {task.description}\n\n"
            f"Expected behavior: {task.expected_behavior}"
        )

        if task.reference_code:
            user_prompt += (
                f"\n\nReference implementation (for inspiration only):\n"
                f"```lua\n{task.reference_code}\n```"
            )

        if task.context:
            import json as _json
            user_prompt += (
                f"\n\nContext (available data):\n"
                f"```json\n{_json.dumps(task.context, indent=2, ensure_ascii=False)}\n```\n"
            )

        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt},
        ]

        max_qa_rounds = 3  # Maximum clarification rounds
        qa_round = 0
        code = ""
        assistant_content = ""

        try:
            while qa_round < max_qa_rounds:
                response = await self.llm_client.chat_completion(
                    messages=messages,
                    model=self.llm_model,
                    temperature=0.2,
                    max_tokens=2500,
                )

                assistant_content = response.choices[0].message.content.strip()
                messages.append({"role": "assistant", "content": assistant_content})

                # Check if LLM is asking a question
                question_match = re_module.match(r"^QUESTION:\s*(.+)", assistant_content.strip(), re_module.DOTALL)

                if question_match and iron_user:
                    # Agent asked a question — get answer from IronUser
                    question = question_match.group(1).strip()
                    iron_response = await iron_user.answer(question)

                    # Add IronUser answer to messages
                    messages.append({
                        "role": "user",
                        "content": f"IronUser answers: {iron_response.answer}",
                    })
                    qa_round += 1
                    # Continue the loop to get the final code
                else:
                    # LLM provided code (or asked question but no IronUser)
                    code = self._strip_code_fences(assistant_content)
                    break
            else:
                # Max QA rounds reached — try to extract code from last response
                code = self._strip_code_fences(assistant_content)

            elapsed = (time_module.monotonic() - start) * 1000

            return ExecutionResult(
                code=code,
                tools_called=["llm_chat_completion"],
                tool_call_count=1,
                execution_time_ms=round(elapsed, 2),
            )

        except Exception as e:
            elapsed = (time_module.monotonic() - start) * 1000
            return ExecutionResult(
                code="",
                tools_called=["llm_chat_completion"],
                tool_call_count=1,
                execution_time_ms=round(elapsed, 2),
                error=f"Code generation failed: {e}",
            )

    async def _score_code(
        self,
        code: str,
        task: EvalTask,
        lua_result: CodeEvalResult,
    ) -> Any:
        """Score code quality using LLM.

        Args:
            code: Generated code.
            task: Original task.
            lua_result: Lua evaluation results.

        Returns:
            LLMQualityScore from llm_evaluator.
        """
        from src.eval.llm_evaluator import LLMEvaluator

        evaluator = LLMEvaluator(
            client=self.llm_eval_client,
            model=self.llm_eval_model,
        )

        lint_output = ""
        if not lua_result.code_lint_ok:
            lint_output = "\n".join(lua_result.code_lint_errors)

        return await evaluator.score(
            code=code,
            task_description=task.description,
            expected_behavior=task.expected_behavior,
            reference_code=task.reference_code,
            lint_output=lint_output,
            run_output=lua_result.code_run_output,
        )

    @staticmethod
    def _determine_status(
        lua_result: CodeEvalResult,
        llm_score: Optional[float],
    ) -> str:
        """Determine overall task status.

        Args:
            lua_result: Lua evaluation results.
            llm_score: Optional LLM quality score.

        Returns:
            Status string: "passed", "failed", or "partial".
        """
        if not lua_result.code_syntax_ok:
            return "failed"

        if not lua_result.code_run_ok:
            return "failed"

        # Syntax and run passed
        if not lua_result.code_lint_ok:
            return "partial"

        # All Lua checks passed — check LLM score if available
        if llm_score is not None:
            if llm_score >= 7.0:
                return "passed"
            elif llm_score >= 4.0:
                return "partial"
            else:
                return "failed"

        return "passed"

    @staticmethod
    def _strip_code_fences(code: str) -> str:
        """Remove markdown code fences from code.

        Args:
            code: Code potentially wrapped in ```lua ... ```.

        Returns:
            Clean code without fences.
        """
        code = code.strip()
        if code.startswith("```"):
            lines = code.splitlines()
            # Remove first line (```lua or ```)
            lines = lines[1:]
            # Remove last line if it's ```
            if lines and lines[-1].strip() == "```":
                lines = lines[:-1]
            code = "\n".join(lines)
        return code