Tool-Selection Evals: Verifying the Agent Picks the Right Tool

An agent with 20 tools is only useful if it picks the right one. Tool-selection evals measure whether your agent reliably maps user intent to the correct function, avoiding both false positives (calling unnecessary tools) and false negatives (missing tools that should be called).

Why Tool Selection Matters

Tool-selection errors are among the most expensive agent failures:

• Wrong tool called: Wastes API calls, returns irrelevant data, confuses the user
• No tool called when needed: Agent hallucinates instead of using available data
• Wrong arguments: Tool is correct but parameters are incorrect
• Unnecessary tool calls: Adds latency and cost without value

Building a Tool-Selection Eval Suite

Define the Ground Truth

For each eval case, specify which tool should be called and with what arguments:

eval_cases = [
    {
        "id": "ts-001",
        "user_input": "What's the current stock price of AAPL?",
        "expected_tool": "get_stock_price",
        "expected_args": {"symbol": "AAPL"},
        "should_not_call": ["search_web", "get_news"],
        "description": "Direct stock price query"
    },
    {
        "id": "ts-002",
        "user_input": "Summarize the latest earnings report for Apple",
        "expected_tool": "search_web",
        "expected_args": {"query": "Apple latest earnings report"},
        "should_not_call": ["get_stock_price"],
        "description": "Qualitative query requiring web search"
    },
    {
        "id": "ts-003",
        "user_input": "Compare Tesla and Ford's revenue this quarter",
        "expected_tools": ["get_financial_data", "get_financial_data"],
        "expected_args_list": [
            {"symbol": "TSLA", "metric": "revenue"},
            {"symbol": "F", "metric": "revenue"}
        ],
        "description": "Multi-tool comparison task"
    }
]

The Eval Runner

from dataclasses import dataclass
from typing import Optional
import json

@dataclass
class ToolSelectionResult:
    case_id: str
    predicted_tool: str
    expected_tool: str
    correct_tool: bool
    args_correct: bool
    extra_tools_called: list[str]
    missing_tools: list[str]
    score: float

def evaluate_tool_selection(agent, eval_cases: list[dict]) -> list[ToolSelectionResult]:
    results = []
    
    for case in eval_cases:
        # Run the agent
        response = agent.run(case["user_input"])
        
        # Extract tool calls from response
        tool_calls = extract_tool_calls(response)
        
        # Score tool selection
        predicted_tool = tool_calls[0]["name"] if tool_calls else None
        correct_tool = predicted_tool == case["expected_tool"]
        
        # Score arguments
        args_correct = False
        if correct_tool and tool_calls:
            args_correct = validate_args(
                tool_calls[0]["args"],
                case["expected_args"]
            )
        
        # Check for unnecessary tool calls
        called_tools = [tc["name"] for tc in tool_calls]
        extra = [t for t in called_tools if t not in case.get("expected_tools", [case["expected_tool"]])]
        missing = [t for t in case.get("expected_tools", [case["expected_tool"]]) if t not in called_tools]
        
        # Compute score
        score = compute_score(correct_tool, args_correct, extra, missing)
        
        results.append(ToolSelectionResult(
            case_id=case["id"],
            predicted_tool=predicted_tool,
            expected_tool=case["expected_tool"],
            correct_tool=correct_tool,
            args_correct=args_correct,
            extra_tools_called=extra,
            missing_tools=missing,
            score=score
        ))
    
    return results

def compute_score(correct_tool, args_correct, extra, missing):
    if not correct_tool:
        return 0.0
    base = 0.5
    if args_correct:
        base += 0.3
    penalty = len(extra) * 0.1 + len(missing) * 0.2
    return max(0, base - penalty)

Tool Confusion Matrix

Visualize which tools get confused with each other:

import numpy as np
from collections import defaultdict

def build_confusion_matrix(results: list[ToolSelectionResult], tool_names: list[str]):
    matrix = np.zeros((len(tool_names), len(tool_names)))
    tool_to_idx = {name: i for i, name in enumerate(tool_names)}
    
    for r in results:
        if r.predicted_tool in tool_to_idx and r.expected_tool in tool_to_idx:
            matrix[tool_to_idx[r.expected_tool]][tool_to_idx[r.predicted_tool]] += 1
    
    return matrix

def print_confusion_matrix(matrix, tool_names):
    print(f"{'':>20}", end="")
    for name in tool_names:
        print(f"{name[:8]:>10}", end="")
    print()
    
    for i, name in enumerate(tool_names):
        print(f"{name[:18]:>20}", end="")
        for j in range(len(tool_names)):
            val = int(matrix[i][j])
            marker = "*" if i == j and val > 0 else " "
            print(f"{val:>9}{marker}", end="")
        print()

# Output might look like:
#                 get_stock search_web get_news  get_financial
#           get_stock       45*        3        1             1
#          search_web        2       38*        4             0
#             get_news        0        1       42*             0
#       get_financial        1        0        0            44*

Prompt Engineering for Better Tool Selection

Tool Description Quality

The single biggest factor in tool-selection accuracy is how well you describe your tools:

# Bad: Vague description
tools = [{
    "name": "search",
    "description": "Searches for information"
}]

# Good: Specific, with use-case guidance
tools = [{
    "name": "web_search",
    "description": "Search the web for current information. Use this when: the user asks about recent events, current prices, news, or anything that changes frequently. Do NOT use for: mathematical calculations, unit conversions, or definitions of established concepts.",
    "parameters": {
        "query": {
            "type": "string",
            "description": "The search query. Be specific and include relevant context."
        }
    }
}]

System Prompt Tool Guidance

system_prompt = """
You have access to the following tools. Choose the MINIMUM number of tools needed:

1. `get_stock_price(symbol)` - Current price and daily change. Use ONLY for real-time price checks.
2. `get_historical_data(symbol, start, end)` - Historical OHLCV data. Use for trends, charts, analysis.
3. `search_news(query)` - Recent news articles. Use for sentiment, events, announcements.
4. `get_financials(symbol)` - Quarterly/annual financial statements. Use for revenue, earnings, margins.

RULES:
- If the user asks about "current price" or "what's X trading at", use get_stock_price.
- If the user asks about "performance" or "trend", use get_historical_data.
- If the user asks about "earnings" or "revenue", use get_financials.
- If the user asks about "news" or "what happened", use search_news.
- Never call a tool "just in case". Only call tools you need.
"""

Running Evals in CI

import pytest

@pytest.fixture
def tool_eval_suite():
    return load_eval_cases("tool_selection_cases.json")

@pytest.mark.parametrize("case", tool_eval_suite(), ids=lambda c: c["id"])
def test_tool_selection(agent, case):
    response = agent.run(case["user_input"])
    tool_calls = extract_tool_calls(response)
    
    called_tools = [tc["name"] for tc in tool_calls]
    
    # Must call the expected tool
    assert case["expected_tool"] in called_tools, \
        f"Expected {case['expected_tool']}, got {called_tools}"
    
    # Must NOT call excluded tools
    for excluded in case.get("should_not_call", []):
        assert excluded not in called_tools, \
            f"Should not have called {excluded}"
    
    # Arguments must match
    if tool_calls:
        matching = [tc for tc in tool_calls if tc["name"] == case["expected_tool"]]
        if matching:
            assert validate_args(matching[0]["args"], case["expected_args"]), \
                f"Args mismatch: {matching[0]['args']} vs {case['expected_args']}"

Tracking Tool-Selection Drift

Monitor your tool-selection metrics over time:

# Weekly eval run
results = evaluate_tool_selection(agent, eval_cases)

metrics = {
    "tool_accuracy": sum(r.correct_tool for r in results) / len(results),
    "args_accuracy": sum(r.args_correct for r in results) / len(results),
    "avg_extra_calls": sum(len(r.extra_tools_called) for r in results) / len(results),
    "confusion_pairs": identify_confusion_pairs(results)
}

# Alert on regression
if metrics["tool_accuracy"] < 0.90:
    send_alert(f"Tool selection accuracy dropped to {metrics['tool_accuracy']:.1%}")

Tool-selection evals should be part of every agent’s CI pipeline. A confusion matrix reveals not just how often your agent fails, but which tools it confuses—guiding both prompt improvements and tool-description refinements.