Agno — API Tests, Evals & CI¶

# tests/test_api.py
import pytest
from agno.os import AgentOS
from fastapi.testclient import TestClient

from app.support_agent import build_support_agent
from tests.fakes import scripted


@pytest.fixture
def client(sqlite_db):
    agent = build_support_agent(scripted("Hello from the API"), db=sqlite_db)
    app = AgentOS(agents=[agent], db=sqlite_db, telemetry=False).get_app()
    with TestClient(app) as test_client:
        yield test_client


def test_health(client):
    assert client.get("/health").json()["status"] == "ok"


def test_run_endpoint_returns_run_output(client):
    response = client.post("/agents/support-agent/runs",
                           data={"message": "hi", "stream": "false", "session_id": "api-1"})

    assert response.status_code == 200
    body = response.json()
    assert body["content"] == "Hello from the API"
    assert body["status"] == "COMPLETED" and body["session_id"] == "api-1"

import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer


class FakeOpenAI:
    """Minimal OpenAI-compatible /v1/chat/completions server (non-streaming)."""

    def __init__(self, replies: list[dict]):
        self.replies = list(replies)      # assistant messages: {"content": ...} or {"tool_calls": [...]}
        self.requests: list[dict] = []    # JSON bodies the client sent
        fake = self

        class Handler(BaseHTTPRequestHandler):
            def do_POST(self):
                body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
                fake.requests.append(body)
                message = {"role": "assistant", "content": None, **fake.replies.pop(0)}
                payload = {
                    "id": f"chatcmpl-{len(fake.requests)}", "object": "chat.completion", "created": 0,
                    "model": body["model"],
                    "choices": [{"index": 0, "message": message,
                                 "finish_reason": "tool_calls" if message.get("tool_calls") else "stop"}],
                    "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
                }
                data = json.dumps(payload).encode()
                self.send_response(200)
                self.send_header("Content-Type", "application/json")
                self.send_header("Content-Length", str(len(data)))
                self.end_headers()
                self.wfile.write(data)

            def log_message(self, *args):
                pass

        self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
        self.base_url = f"http://127.0.0.1:{self.server.server_port}/v1"
        threading.Thread(target=self.server.serve_forever, daemon=True).start()

    def close(self):
        self.server.shutdown()
        self.server.server_close()

# tests/test_openai_contract.py
import json

import pytest
from agno.models.openai import OpenAIChat

from app.support_agent import build_support_agent
from tests.fake_openai import FakeOpenAI


@pytest.fixture
def fake_openai():
    server = FakeOpenAI([
        {"tool_calls": [{"id": "call_1", "type": "function",
                         "function": {"name": "get_order_status", "arguments": json.dumps({"order_id": "ord-42"})}}]},
        {"content": "Order ord-42 has shipped."},
    ])
    yield server
    server.close()


def test_real_openai_adapter_against_fake_server(fake_openai):
    model = OpenAIChat(id="gpt-test", api_key="test-key", base_url=fake_openai.base_url)

    run = build_support_agent(model).run("Where is ord-42?")

    assert run.content == "Order ord-42 has shipped."
    first, second = fake_openai.requests
    assert first["model"] == "gpt-test"
    assert {t["function"]["name"] for t in first["tools"]} == {"get_order_status", "issue_refund"}
    assert second["messages"][-1] == {"role": "tool", "content": "ord-42: shipped", "tool_call_id": "call_1"}
    assert run.metrics.total_tokens == 30                   # usage from both responses

# tests/test_tracing.py
import pytest
from openinference.instrumentation.agno import AgnoInstrumentor
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter

from app.support_agent import build_support_agent
from tests.fakes import scripted, tool_call


@pytest.fixture
def spans():
    exporter = InMemorySpanExporter()
    provider = TracerProvider()
    provider.add_span_processor(SimpleSpanProcessor(exporter))
    AgnoInstrumentor().instrument(tracer_provider=provider)
    yield exporter
    AgnoInstrumentor().uninstrument()


def test_run_produces_agent_and_tool_spans(spans):
    model = scripted(tool_call("get_order_status", {"order_id": "ord-1"}), "Shipped.")
    build_support_agent(model).run("Where is ord-1?")

    kinds = {(s.name, s.attributes.get("openinference.span.kind")) for s in spans.get_finished_spans()}
    assert ("Support_Agent.run", "AGENT") in kinds          # spaces in the name become "_"
    assert ("get_order_status", "TOOL") in kinds

# tests/test_evals.py
from agno.eval.accuracy import AccuracyEval
from agno.eval.performance import PerformanceEval
from agno.eval.reliability import ReliabilityEval

from app.support_agent import build_support_agent, get_order_status
from tests.fakes import scripted, tool_call


def test_reliability_eval_checks_tool_calls_and_arguments():
    model = scripted(tool_call("get_order_status", {"order_id": "ord-42"}), "Shipped.")
    run = build_support_agent(model).run("Where is ord-42?")

    result = ReliabilityEval(
        agent_response=run,
        expected_tool_calls=["get_order_status"],
        expected_tool_call_arguments={"get_order_status": {"order_id": "ord-42"}},
    ).run()

    result.assert_passed()                                  # eval_status == "PASSED"
    assert result.missing_tool_calls == [] and result.failed_argument_checks == []


def test_reliability_eval_reports_missing_tools():
    run = build_support_agent(scripted("It has shipped, I think.")).run("Where is ord-42?")

    result = ReliabilityEval(agent_response=run, expected_tool_calls=["get_order_status"]).run()

    assert result.eval_status == "FAILED"
    assert result.missing_tool_calls == ["get_order_status"]


def test_accuracy_eval_with_scripted_judge():
    agent = build_support_agent(scripted(tool_call("get_order_status", {"order_id": "ord-42"}),
                                         "Order ord-42 has shipped."))
    judge = scripted('{"accuracy_score": 9, "accuracy_reason": "Matches the expected status."}')

    result = AccuracyEval(agent=agent, model=judge, input="Where is ord-42?",
                          expected_output="ord-42 has shipped", num_iterations=1).run(
        print_summary=False, print_results=False)

    assert result.avg_score >= 8


def test_tool_latency_budget():
    result = PerformanceEval(func=lambda: get_order_status("ord-1"), num_iterations=50,
                             warmup_runs=5, measure_memory=False).run()

    assert result.p95_run_time < 0.01                       # seconds

# tests/test_deepeval.py
import os

import pytest
from deepeval import assert_test
from deepeval.metrics import ToolCorrectnessMetric
from deepeval.test_case import LLMTestCase, ToolCall, ToolCallParams

from app.support_agent import build_support_agent
from tests.fakes import scripted, tool_call


def to_test_case(question: str, run, expected_tools: list[ToolCall]) -> LLMTestCase:
    return LLMTestCase(
        input=question,
        actual_output=str(run.content),
        tools_called=[ToolCall(name=t.tool_name, input_parameters=t.tool_args, output=t.result)
                      for t in run.tools or []],
        expected_tools=expected_tools,
    )


def test_run_output_maps_to_deepeval_test_case():
    run = build_support_agent(scripted(tool_call("get_order_status", {"order_id": "ord-42"}), "Shipped.")).run(
        "Where is ord-42?")

    case = to_test_case("Where is ord-42?", run, [ToolCall(name="get_order_status")])

    assert [t.name for t in case.tools_called] == ["get_order_status"]
    assert case.tools_called[0].input_parameters == {"order_id": "ord-42"}


@pytest.mark.llm
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="DeepEval metrics need a judge model")
def test_tool_correctness_with_deepeval():
    question = "Where is ord-42?"
    run = build_support_agent("openai:gpt-5-mini").run(question)
    expected = [ToolCall(name="get_order_status", input_parameters={"order_id": "ord-42"})]

    metric = ToolCorrectnessMetric(evaluation_params=[ToolCallParams.INPUT_PARAMETERS], threshold=1.0)
    assert_test(to_test_case(question, run, expected), [metric])

# tests/test_real_llm.py
import os

import pytest
from agno.run.base import RunStatus

from app.support_agent import build_support_agent

pytestmark = [
    pytest.mark.llm,                                              # run with: pytest -m llm
    pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="no API key"),
]


@pytest.mark.parametrize("attempt", range(3))                     # repeat: behaviour must be stable
def test_real_model_uses_the_order_tool(attempt):
    agent = build_support_agent("openai:gpt-5-mini")

    run = agent.run("Where is order ord-42?")

    assert run.status == RunStatus.completed
    assert [(t.tool_name, t.tool_args) for t in run.tools] == [("get_order_status", {"order_id": "ord-42"})]
    assert "shipped" in run.content.lower()

# .github/workflows/agents.yml
name: agents
on: [push, pull_request]
env:
  AGNO_TELEMETRY: "false"
jobs:
  offline:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v6
      - run: uv sync --locked
      - run: uv run pytest -m "not llm" -n auto        # pytest-xdist; every test has its own db
  llm:
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    needs: offline
    runs-on: ubuntu-latest
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v6
      - run: uv sync --locked
      - run: uv run pytest -m llm --junitxml=reports/llm.xml