promptfoo — Test Cases & Assertions¶

tests:
  - description: refund window          # shown in the table; --filter-pattern matches it
    vars:
      question: How long do refunds take?
      locale: en
    assert:
      - type: contains
        value: 14 days
    metadata:
      area: billing                     # --filter-metadata area=billing
    threshold: 0.8                      # minimum weighted score of all assertions
    options:
      repeat: 3                         # run this test three times
    providers: [gateway]                # only these providers (labels, ids, openai:*)
    prompts: [v2-polite]                # only these prompts (labels or ids)

question,__expected1,__expected2,__description,__metadata:area
What is the capital of France?,icontains: paris,not-contains: berlin,capital,geo
How long do refunds take?,contains: 14 days,regex: \d+ days,refund window,billing

assert:
  - type: is-json
    value:
      type: object
      required: [priority, category]
      properties:
        priority: { type: string, enum: [P1, P2, P3, P4] }
        category: { type: string }
  - type: equals                       # pick a field, then compare
    value: P1
    transform: JSON.parse(output).priority
  - type: not-regex
    value: '\b\d{16}\b'                # no card-like numbers
  - type: word-count
    value: { max: 60 }

assert:
  - type: similar
    value: ['Refunds take 14 days', 'You get your money back within two weeks']
    threshold: 0.8                     # cosine similarity, default 0.75
    provider: openai:embedding:text-embedding-3-large

defaultTest:
  options:
    provider:                           # grader for every model-graded assertion
      text:
        id: openai:gpt-6-sol
        config: { reasoning: { effort: none }, temperature: 0 }
      embedding:
        id: openai:embedding:text-embedding-3-large
    rubricPrompt: file://prompts/rubric.txt     # optional: {{output}} and {{rubric}}
tests:
  - vars: { question: 'What is the capital of France?' }
    assert:
      - type: llm-rubric
        value: States that the capital of France is Paris
      - type: factuality
        value: The capital of France is Paris
        provider: anthropic:messages:claude-sonnet-5-5   # per-assertion override

assert:
  - type: javascript
    value: output.length < 200                         # expression: boolean or score
  - type: javascript
    value: file://assertions/graded.js
    config: { maxWords: 10 }
  - type: python
    value: file://assertions/check_json.py:get_assert  # default function name is get_assert
    metric: category_correct

// assertions/graded.js
module.exports = (output, context) => {
  // context: prompt, vars, test, config, provider, providerResponse, metadata, trace
  const words = output.split(/\s+/).length;
  return {
    pass: words <= context.config.maxWords,
    score: Math.max(0, 1 - words / 20),
    reason: `${words} words`,
    componentResults: [
      { pass: output.includes('Paris'), score: 1, reason: 'mentions Paris', namedScores: { mentions_paris: 1 } },
    ],
  };
};

# assertions/check_json.py
import json

def get_assert(output, context):
    data = json.loads(output)
    expected = context["config"]["expected"]
    ok = data.get("category") == expected
    return {
        "pass": ok,
        "score": 1.0 if ok else 0.0,
        "reason": f"category={data.get('category')!r}, expected={expected!r}",
        "named_scores": {"category_correct": 1.0 if ok else 0.0},   # snake_case is mapped
    }

defaultTest:
  assert:
    - type: latency
      threshold: 3000
      metric: latency_ok
      weight: 0                          # record, do not gate
tests:
  - vars: { question: 'What is the capital of France?' }
    threshold: 0.6                       # weighted average of every assertion, defaults included
    assert:
      - type: contains
        value: Paris
        weight: 3
        metric: accuracy
      - type: assert-set                 # 1 of 2 must pass
        threshold: 0.5
        metric: style
        assert:
          - type: icontains
            value: capital
          - type: contains
            value: '!'
derivedMetrics:                          # after the run, per prompt / provider
  - name: quality
    value: accuracy * 0.7 + style * 0.3

promptfoo eval --var locale=de                 # override a var for the whole run
promptfoo eval --repeat 5 --no-cache           # flakiness check: 5 samples per test
promptfoo eval --filter-metadata area=billing  # run one slice

# echo returns the prompt as the output: the vars are the "model answer"
prompts: ['{{text}}']
providers: [echo]
tests:
  - vars: { text: 'Here is the result: {"priority": "P1", "category": "billing"} Thanks!' }
    assert:
      - type: contains-json
        value: { type: object, required: [priority] }
      - type: not-is-json

promptfoo eval --assertions asserts.yaml --model-outputs outputs.json   # score saved outputs, no provider calls