An eval suite only protects a release if something acts on its result. A regression gate compares the eval run of a change with the run of the code on main and fails the pull request when a test that used to pass now fails. You will build gate.py for the issue-triage suite from Build an eval harness with pytest, give it a written policy, and run it in GitHub Actions on replayed recordings, so the job needs no model and no API key.
When you need this. A teammate opens a pull request that tweaks the triage prompt, or switches to a cheaper local model to cut cost. Reading a prompt diff doesn’t tell you what it does to the answers. The gate runs on every pull request and blocks the merge when a test that passes on main starts to fail, and it lists the tests.
The lab records four candidate changes against the baseline on main: a new prompt, a smaller model, a bigger dataset, and the same prompt change measured on the bigger dataset. Step 4 compares the saved prompt and model changes.
python gate.py baseline.json candidate.json · no model call
Step 1: Make each component a switch
Download the complete lab, extract it and open a terminal in build-regression-gates-for-prompts-models-data-and-indexes/. The README lists its runtime and recorded examples.
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r requirements-lock.txtRecord what each run used so you can identify which components changed between reports. In this lab the prompt, the model and the dataset are environment variables, and the recording key stores all three.
"""Issue triage. TRIAGE_MODEL and TRIAGE_PROMPT select the components under test:
a Gemini model through the API, or any other name (gemma4:e2b) through a local Ollama server."""
import os
from functools import cache
from typing import Literal
from pydantic import BaseModel, Field
MODEL = os.environ.get("TRIAGE_MODEL", "gemini-3.8-flash")
PROMPT_VERSION = os.environ.get("TRIAGE_PROMPT", "v1")
V1 = """You triage GitHub issues for `tracker`, an open-source task-tracker CLI.
Choose one label:
- bug: the program behaves wrongly, crashes, loses data or leaks secrets.
- feature: a request for new behavior.
- docs: the documentation is wrong or missing.
- question: the reporter asks how something works.
Set priority to p0 for data loss or a leaked secret, p1 for other bugs and p2 otherwise.
Write a summary of at most 20 words that keeps the command or file the issue names."""
V2 = V1 + """
Tasks that are dropped, deleted or lost without the user asking count as data loss.
The summary also keeps the reporter's goal when the issue states one."""
PROMPTS = {"v1": V1, "v2": V2}
class Triage(BaseModel):
label: Literal["bug", "feature", "docs", "question"]
priority: Literal["p0", "p1", "p2"]
summary: str = Field(min_length=1)
@cache
def gemini():
from google import genai
return genai.Client() # reads GOOGLE_API_KEY from the environment
def triage(issue: str, model: str = MODEL, prompt: str = PROMPT_VERSION) -> str:
"""Return the model's raw JSON reply; the caller validates it against Triage."""
if model.startswith("gemini"):
from google.genai import types
config = types.GenerateContentConfig(
system_instruction=PROMPTS[prompt],
response_mime_type="application/json",
response_schema=Triage,
thinking_config=types.ThinkingConfig(thinking_level="low"),
)
return gemini().models.generate_content(model=model, contents=issue, config=config).text
from ollama import chat
response = chat(
model=model,
messages=[{"role": "system", "content": PROMPTS[prompt]}, {"role": "user", "content": issue}],
format=Triage.model_json_schema(),
options={"temperature": 0, "seed": 7},
think=False,
)
return response.message.content
if __name__ == "__main__":
import sys
print(triage(sys.argv[1]))Prompt v2 adds two sentences to v1: it says that dropped or lost tasks count as data loss, and that the summary keeps the reporter’s goal. Both target errors the harness lesson found on Gemma.
"""Fixtures for the triage eval: the dataset, the cassette and the judge model."""
import hashlib
import os
from functools import cache
from pathlib import Path
import pytest
os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "1")
from deepeval.models import GeminiModel, OllamaModel # noqa: E402
from harness import Cassette, load_dataset # noqa: E402
from triage import MODEL, PROMPT_VERSION, PROMPTS, Triage, triage # noqa: E402
DATASET = load_dataset(Path(__file__).with_name(os.environ.get("TRIAGE_DATASET", "dataset.json")))
JUDGE_MODEL = "gemini-3.8-flash" # the judge stays fixed while the model under test changes
KEY = {
"dataset": f"{DATASET.name}@{DATASET.version}",
"dataset_sha": DATASET.sha,
"model": MODEL,
"prompt_version": PROMPT_VERSION,
"prompt_sha": hashlib.sha256(PROMPTS[PROMPT_VERSION].encode()).hexdigest()[:12],
"judge_model": JUDGE_MODEL,
}
def pytest_addoption(parser):
parser.addoption("--live", action="store_true", help="call the model and rewrite the recording")
@pytest.fixture(scope="session")
def cassette(request):
live = request.config.getoption("--live")
name = f"{MODEL.replace(':', '-')}-{PROMPT_VERSION}-v{DATASET.version}.json"
tape = Cassette(Path(__file__).with_name("recordings") / name, KEY, live)
yield tape
if live:
tape.save()
@pytest.fixture
def raw_output(case, cassette):
return cassette.get(f"triage/{case.id}", lambda: triage(case.issue))
@pytest.fixture
def output(raw_output):
return Triage.model_validate_json(raw_output)
def make_judge(name: str):
"""DeepEval's judge wrapper for the model name: Gemini through the API, anything else through Ollama."""
if name.startswith("gemini"):
return GeminiModel(model=name)
return OllamaModel(model=name, base_url="http://localhost:11434", temperature=0)
@pytest.fixture(scope="session")
def judge():
"""Builds the judge on first use, so a replay needs no API key."""
return cache(lambda: make_judge(JUDGE_MODEL))
@pytest.hookimpl(optionalhook=True)
def pytest_json_modifyreport(json_report):
# Keep the run record about the eval, not about the machine that ran it.
for field in ("environment", "root"):
json_report.pop(field, None)
json_report["eval"] = KEYThe judge is pinned to gemini-3.8-flash and doesn’t follow TRIAGE_MODEL. If the judge changed with the model under test, a lower faithfulness score could mean a worse model or a stricter judge, and the gate could not tell which. Each combination of model, prompt and dataset version gets its own recording file, so these recorded configurations use separate paths:
TRIAGE_PROMPT=v2 pytest -q --live test_triage.py --json-report --json-report-file=runs/prompt-v2.json
TRIAGE_MODEL=gemma4:e2b pytest -q --live test_triage.py --json-report --json-report-file=runs/gemma4-e2b.json
TRIAGE_PROMPT=v2 TRIAGE_DATASET=dataset-v2.json pytest -q --live test_triage.py --json-report --json-report-file=runs/prompt-v2-dataset-v2.json
TRIAGE_DATASET=dataset-v2.json pytest -q --live test_triage.py --json-report --json-report-file=runs/baseline-dataset-v2.jsonThe baseline, runs/baseline.json, is the harness lesson’s recorded run of prompt v1 on gemini-3.8-flash, replayed here: 51 of 51 tests passed.
Step 2: Write the policy
{
"max_regressions": 0,
"min_pass_rate": 0.9,
"critical": ["test_priority", "test_reply_matches_schema"]
}The policy has three rules, each answering a different question. max_regressions: 0 says no test that passes on main may fail on the candidate. critical names tests that must pass whatever the baseline did: the policy blocks a wrong priority on a leaked token or a malformed reply even when the baseline has the same failure. min_pass_rate: 0.9 is a floor on the candidate alone.
The gate compares outcomes per test, not one aggregate score, on purpose. A change that fixes one case and breaks another leaves the pass rate unchanged, and an average judge score can rise while one important case gets worse. Per-test outcomes make both visible. Keep policy.json in the repository, so changing a limit is itself a reviewed change.
Critical test fails on both runs. The policy returns block even without a regression; fix the critical case before accepting the candidate.
Pass rate below the floor. The policy returns block and names the rate; review failing cases rather than lowering the limit inside CI.
Step 3: Write the gate
Now we write the decision itself.
"""Regression gate: compare a candidate eval run with the baseline run and fail the job on a regression."""
import argparse
import json
import sys
from pathlib import Path
EXIT = {"pass": 0, "block": 1, "incomparable": 2}
COMPONENTS = ("model", "prompt_version", "prompt_sha")
def outcomes(report: dict) -> dict[str, str]:
return {t["nodeid"]: t["outcome"] for t in report["tests"]}
def gate(baseline: dict, candidate: dict, policy: dict) -> dict:
b, c = baseline["eval"], candidate["eval"]
if b["dataset_sha"] != c["dataset_sha"]:
return {"decision": "incomparable", "reasons": [
f"baseline ran {b['dataset']} ({b['dataset_sha']}), candidate ran {c['dataset']} ({c['dataset_sha']}); "
"rerun the baseline on the candidate's dataset"]}
if b["judge_model"] != c["judge_model"]:
return {"decision": "incomparable", "reasons": [
f"judge changed from {b['judge_model']} to {c['judge_model']}; judged scores are not comparable"]}
before, after = outcomes(baseline), outcomes(candidate)
regressed = sorted(n for n, o in before.items() if o == "passed" and after.get(n) != "passed")
fixed = sorted(n for n, o in before.items() if o != "passed" and after.get(n) == "passed")
is_critical = lambda n: n.split("::")[-1].startswith(tuple(policy["critical"])) # noqa: E731
critical = sorted(n for n, o in after.items() if o != "passed" and is_critical(n))
missing_critical = sorted(n for n in before if is_critical(n) and n not in after)
pass_rate = sum(o == "passed" for o in after.values()) / len(after)
reasons = []
if len(regressed) > policy["max_regressions"]:
reasons.append(f"{len(regressed)} regression(s); policy allows {policy['max_regressions']}")
if critical:
reasons.append(f"{len(critical)} critical test(s) fail")
if missing_critical:
reasons.append(f"{len(missing_critical)} critical test(s) from the baseline are missing from the candidate")
if pass_rate < policy["min_pass_rate"]:
reasons.append(f"pass rate {pass_rate:.3f} is below {policy['min_pass_rate']}")
return {"decision": "block" if reasons else "pass", "reasons": reasons,
"changed": {k: [b[k], c[k]] for k in COMPONENTS if b[k] != c[k]},
"pass_rate": round(pass_rate, 3), "regressed": regressed, "fixed": fixed, "critical_failures": critical,
"missing_critical": missing_critical}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("baseline", type=Path)
parser.add_argument("candidate", type=Path)
parser.add_argument("--policy", type=Path, default=Path(__file__).with_name("policy.json"))
parser.add_argument("--out", type=Path, help="write the decision as JSON")
args = parser.parse_args()
load = lambda path: json.loads(path.read_text()) # noqa: E731
result = gate(load(args.baseline), load(args.candidate), load(args.policy))
if args.out:
args.out.write_text(json.dumps(result, indent=2) + "\n")
print(f"decision: {result['decision'].upper()}")
for key in ("reasons", "changed", "regressed", "fixed", "critical_failures", "missing_critical"):
if result.get(key):
print(f"{key}: {json.dumps(result[key], indent=2)}")
return EXIT[result["decision"]]
if __name__ == "__main__":
sys.exit(main())gate() first checks that the two runs can be compared at all. A different dataset hash means different tests, and a different judge means different scoring, so the gate returns incomparable instead of a decision. Then it computes five things from the per-test outcomes: regressed tests, fixed tests, failing critical tests, critical tests missing from the candidate and the candidate’s pass rate. Every broken rule becomes a reason, and any reason blocks. changed names the components that differ between the two eval keys, which is what a reviewer needs first.
A test that passes on main and is missing from the candidate counts as a regression. A critical test that is in the baseline but missing from the candidate blocks with its own reason, whether it passed or failed on main, so deleting a failing critical test does not pass the gate. The gate does not block the deletion of a non-critical test that already fails on main; that deletion raises the pass rate, so review test and dataset deletions in the pull request. The exit code is the decision, 0 for pass, 1 for block and 2 for incomparable, so CI needs no parsing.
One test regresses. block, the test is listed and changed names the component.
Test missing from the candidate. Counted as a regression when the test passed on the baseline.
Failing critical test deleted from the candidate. block, the test is listed in missing_critical.
Step 4: Gate the recorded candidates
python gate.py runs/baseline.json runs/prompt-v2.json --out runs/gate-prompt-v2.jsondecision: PASS
changed: {
"prompt_version": [
"v1",
"v2"
],
"prompt_sha": [
"01859a9d3df1",
"573cd471224c"
]
}
exit code: 0The prompt change passes: no test that passed on v1 fails on v2. On a model that already passed every test, the gate shows only that v2 broke nothing. It can’t show that v2 helps, because the baseline left nothing to fix.
python gate.py runs/baseline.json runs/gemma4-e2b.jsondecision: BLOCK
reasons: [
"2 regression(s); policy allows 0",
"1 critical test(s) fail"
]
changed: {
"model": [
"gemini-3.8-flash",
"gemma4:e2b"
]
}
regressed: [
"test_triage.py::test_priority[import-drops-tasks]",
"test_triage.py::test_summary_is_short_and_specific[where-is-data]"
]
critical_failures: [
"test_triage.py::test_priority[import-drops-tasks]"
]
exit code: 1The model change is blocked. Its 2 regressed tests are tests that gemini-3.8-flash passes and gemma4:e2b fails; the live run shows the replies:
output = Triage(label='bug', priority='p1', summary='tracker import tasks.csv fails to import all 500 rows, dropping tasks silently.')
E AssertionError: assert 'p1' == 'p0'
output = Triage(label='question', priority='p2', summary='Where does tracker store its database on disk?')
E AssertionError: summary lacks ['backup']: 'Where does tracker store its database on disk?'
FAILED test_triage.py::test_priority[import-drops-tasks] - AssertionError: as...
FAILED test_triage.py::test_summary_is_short_and_specific[where-is-data] - As...
2 failed, 49 passed in 176.65s (0:02:56)Full output: runs/gemma4-e2b.txt.
Candidate ran on another dataset. The gate returns incomparable; rerun the baseline on the candidate’s dataset before comparing.
Candidate used another judge. The gate returns incomparable; score both runs with the same judge configuration.
A fix and no regression. pass, the fixed test is listed.
Recorded decisions. Recomputing each gate from its runs gives the saved decision.
A smaller model that regresses. The blocked gate above is a real regression: the model swap breaks tests that pass on main. The decision lists each one:
| Regressed test | Critical |
|---|---|
test_priority[import-drops-tasks] | yes |
test_summary_is_short_and_specific[where-is-data] | no |
To ship a smaller model anyway, change the dataset or the policy in a reviewed commit and say why, rather than lowering a limit inside the job.
A changed dataset. Dataset v2 adds 2 cases, tags-lost-on-edit and week-start-question. The prompt v2 candidate on it can’t be compared with the baseline on v1:
decision: INCOMPARABLE
reasons: [
"baseline ran issue-triage@1 (e12a5e739a1e), candidate ran issue-triage@2 (ae3ec987b16f); rerun the baseline on the candidate's dataset"
]
exit code: 2Comparing pass rates across the two datasets would credit or blame the prompt for the new cases. The fix is the one the reason names: record the baseline prompt on the new dataset, then gate again. Both runs now grade the same 14 cases:
decision: PASS
changed: {
"prompt_version": [
"v1",
"v2"
],
"prompt_sha": [
"01859a9d3df1",
"573cd471224c"
]
}
exit code: 0Offline gate checks. test_gate.py has 10 offline tests:
pytest -q test_gate.pyInspect test_gate.py
"""Failure cases for the regression gate. No model call; runs offline."""
import json
import subprocess
import sys
from pathlib import Path
import pytest
from gate import gate
HERE = Path(__file__).parent
POLICY = json.loads((HERE / "policy.json").read_text())
KEY = {"dataset": "issue-triage@1", "dataset_sha": "aaa", "model": "m", "prompt_version": "v1",
"prompt_sha": "p1", "judge_model": "j"}
def run(outcomes: dict, **key) -> dict:
tests = [{"nodeid": f"test_triage.py::{name}", "outcome": o} for name, o in outcomes.items()]
return {"eval": {**KEY, **key}, "tests": tests}
BASE = {f"test_label[c{i}]": "passed" for i in range(20)} | {"test_priority[c0]": "passed"}
def test_changed_dataset_is_incomparable():
result = gate(run(BASE), run(BASE, dataset_sha="bbb"), POLICY)
assert result["decision"] == "incomparable" and "rerun the baseline" in result["reasons"][0]
def test_changed_judge_is_incomparable():
assert gate(run(BASE), run(BASE, judge_model="other"), POLICY)["decision"] == "incomparable"
def test_one_regression_blocks():
result = gate(run(BASE), run(BASE | {"test_label[c3]": "failed"}, prompt_version="v2"), POLICY)
assert result["decision"] == "block"
assert result["regressed"] == ["test_triage.py::test_label[c3]"]
assert result["changed"] == {"prompt_version": ["v1", "v2"]}
def test_critical_failure_blocks_even_if_the_baseline_failed_it_too():
both = BASE | {"test_priority[c0]": "failed"}
result = gate(run(both), run(both), POLICY)
assert result["decision"] == "block" and result["regressed"] == []
assert result["critical_failures"] == ["test_triage.py::test_priority[c0]"]
def test_low_pass_rate_blocks_without_a_regression():
weak = BASE | {f"test_label[c{i}]": "failed" for i in range(3)}
result = gate(run(weak), run(weak), POLICY)
assert result["decision"] == "block" and result["reasons"] == ["pass rate 0.857 is below 0.9"]
def test_test_missing_from_the_candidate_is_a_regression():
fewer = {k: v for k, v in BASE.items() if k != "test_label[c5]"}
assert gate(run(BASE), run(fewer), POLICY)["regressed"] == ["test_triage.py::test_label[c5]"]
def test_deleting_a_failing_critical_test_blocks():
failing = BASE | {"test_priority[c0]": "failed"}
deleted = {k: v for k, v in failing.items() if k != "test_priority[c0]"}
result = gate(run(failing), run(deleted), POLICY)
assert result["decision"] == "block" and result["regressed"] == []
assert result["missing_critical"] == ["test_triage.py::test_priority[c0]"]
assert result["reasons"] == ["1 critical test(s) from the baseline are missing from the candidate"]
def test_fix_without_regression_passes():
result = gate(run(BASE | {"test_label[c1]": "failed"}), run(BASE), POLICY)
assert result["decision"] == "pass" and result["fixed"] == ["test_triage.py::test_label[c1]"]
@pytest.mark.parametrize("candidate, code", [(BASE, 0), (BASE | {"test_label[c3]": "failed"}, 1), (None, 2)])
def test_exit_code_is_the_decision(tmp_path, candidate, code):
(tmp_path / "b.json").write_text(json.dumps(run(BASE)))
report = run(candidate) if candidate else run(BASE, dataset_sha="bbb")
(tmp_path / "c.json").write_text(json.dumps(report))
done = subprocess.run([sys.executable, str(HERE / "gate.py"), str(tmp_path / "b.json"), str(tmp_path / "c.json")],
capture_output=True, text=True)
assert done.returncode == code, done.stdout
@pytest.mark.parametrize("baseline, candidate, decision", [
("baseline", "prompt-v2", "prompt-v2"),
("baseline", "gemma4-e2b", "gemma4-e2b"),
("baseline", "prompt-v2-dataset-v2", "prompt-v2-dataset-v2"),
("baseline-dataset-v2", "prompt-v2-dataset-v2", "prompt-v2-on-dataset-v2"),
])
def test_recorded_decisions_reproduce(baseline, candidate, decision):
load = lambda name: json.loads((HERE / "runs" / f"{name}.json").read_text()) # noqa: E731
assert gate(load(baseline), load(candidate), POLICY) == load(f"gate-{decision}")Step 5: Run the gate in GitHub Actions
# Copy to .github/workflows/eval-gate.yml in a repository whose root holds this lab.
# CI has no model server: it replays the recording committed with the change,
# then gates that run against the baseline report on main.
name: eval-gate
on: pull_request
jobs:
gate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- uses: actions/setup-python@v7
with:
python-version: "3.12"
- run: pip install -r requirements-lock.txt
- name: Replay the candidate's recording
# pytest exits 1 when tests fail; the gate decides what that means. Any other code is an error.
run: pytest -q test_triage.py --json-report --json-report-file=candidate.json || [ $? -eq 1 ]
- name: Gate against the baseline on main
run: |
git show origin/main:runs/baseline.json > baseline.json
python gate.py baseline.json candidate.json --out gate.jsonThe job replays the recording committed with the pull request and gates that run against runs/baseline.json from main. pytest exits 1 when any test fails, which is not an error here, so || [ $? -eq 1 ] lets the gate decide; any other pytest exit code, such as 2 for a collection error, still fails the step. fetch-depth: 0 fetches main so git show can read its baseline.
The design choice is where the model runs. Replaying keeps model calls, cost and the API key out of pull-request jobs, and the pure gate computes the same decision from the same recorded outcomes, eval keys and policy. This does not establish repeatability of a fresh model or judge call. The cost is trust. The gate grades whatever recording the pull request contains, and a recording is a JSON file the author can edit. For your own team, review the recording like code. For changes from outside contributors, record again in a job that only maintainers can start, with the key available only to that job.
Exit code. The command returns 0, 1 or 2 for pass, block or incomparable. Let CI fail on either nonzero decision; an incomparable result needs a corrected baseline, not a forced pass.
A pull request adds two cases to the dataset and changes the prompt, and the gate returns INCOMPARABLE. Decide what should happen next before you answer.
Read the explanation
A regression is a test that passed on the baseline and fails on the candidate. That comparison needs the same cases on both sides, which is why the gate checks the dataset hash before anything else.
Adapt the separate documentation stage report
The documentation stage-report lesson produces metric rows, while this gate accepts named outcomes. stage_report_adapter.py maps applicable tools, page-ID retrieval, faithfulness, contextual recall and answer-relevancy checks into that input shape. These are mapped metric outcomes, not a new pytest or judge run. Contextual precision stays in the saved report; it is not added to the stage-failure policy.
"""Normalize saved docs-stage metrics for gate.py; this does not run pytest or a judge."""
import argparse,hashlib,json,math
from pathlib import Path
from gate import gate,EXIT
METRICS=('faithfulness','contextual_recall','answer_relevancy')
POLICY={'max_regressions':0,'min_pass_rate':0.9,'critical':['test_tools','test_faithfulness']}
def digest(value):return hashlib.sha256(json.dumps(value,sort_keys=True,separators=(',',':')).encode()).hexdigest()
def score(value):
if type(value) not in (int,float) or not math.isfinite(value) or not 0<=value<=1:raise ValueError('invalid_score')
return value
def adapt(report):
key=report['key'];threshold=score(report['threshold'])
required=('cases','cases_sha','docs_sha','prompt_sha','model','embedder','judge')
if any(not isinstance(key.get(k),str) or not key[k] for k in required):raise ValueError('missing_report_key')
if type(key.get('strict_faithfulness')) is not bool or type(report['k']) is not int or report['k']<1 or key['k']!=report['k']:raise ValueError('invalid_contract')
rows=report['rows']
if not isinstance(rows,list) or not rows:raise ValueError('empty_cases')
tests=[];applicability={};seen=set()
for row in rows:
case=row['id']
if not isinstance(case,str) or not case or '::' in case or case in seen:raise ValueError('invalid_or_duplicate_case')
seen.add(case);checks={};tools=row['tools'];s=score(tools['score'])
if type(tools['pass']) is not bool or not isinstance(tools['unexpected'],list) or tools['pass']!=(s>=threshold and not tools['unexpected']):raise ValueError('inconsistent_tools')
checks['tools']=tools['pass'];retrieval=row['retrieval']
if retrieval is not None:
if type(retrieval['hit']) is not bool:raise ValueError('invalid_retrieval')
score(retrieval['reciprocal_rank']);checks['retrieval']=score(retrieval['recall'])>=threshold
judged=row['judged']
if not isinstance(judged,dict) or 'answer_relevancy' not in judged:raise ValueError('missing_answer_metric')
for name,metric in judged.items():
if name not in (*METRICS,'contextual_precision') or not isinstance(metric,dict) or not isinstance(metric.get('reason'),str):raise ValueError('invalid_judged_metric')
score(metric['score'])
# Source contract: no source context on the thanks case. Every other case has source metrics.
expected={'answer_relevancy'} if case=='thanks' else {*METRICS,'contextual_precision'}
if set(judged)!=expected:raise ValueError('missing_or_unexpected_metric')
for name in METRICS:
if name in judged:checks[name]=judged[name]['score']>=threshold
first=next((name for name in ('tools','retrieval',*METRICS) if name in checks and not checks[name]),None)
if first!=row['first_failure']:raise ValueError('inconsistent_first_failure')
applicability[case]=sorted(checks)
for name,passed in checks.items():tests.append({'nodeid':f'docs-stages::{case}::test_{name}','outcome':'passed' if passed else 'failed'})
contract={k:key[k] for k in ('cases','cases_sha','docs_sha','embedder','k','strict_faithfulness')}
contract.update({'threshold':threshold,'protocol':'docs-stage-scores-v1','applicability':applicability})
return {'eval':{'dataset':key['cases'],'dataset_sha':digest(contract),'judge_model':key['judge'],'model':key['model'],'prompt_version':'sha:'+key['prompt_sha'],'prompt_sha':key['prompt_sha']},'tests':tests,'contract':contract,'provenance':{'source_report_sha256':digest(report),'execution_mode':'offline-metric-to-outcome-adapter','prompt_version_is_derived':True,'source_metrics':rows}}
def compare(baseline,candidate):
return gate(adapt(baseline),adapt(candidate),POLICY)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('baseline',type=Path);p.add_argument('candidate',type=Path);p.add_argument('--out',type=Path,required=True);a=p.parse_args()
if a.out.exists():raise SystemExit('Refusing to overwrite a result')
b=json.loads(a.baseline.read_text());c=json.loads(a.candidate.read_text());result={'execution_mode':'offline-saved-stage-report-adaptation','policy':POLICY,'baseline':adapt(b),'candidate':adapt(c),'decision':compare(b,c)}
a.out.parent.mkdir(parents=True,exist_ok=True);a.out.write_text(json.dumps(result,indent=2)+'\n');print(json.dumps(result['decision']));raise SystemExit(EXIT[result['decision']['decision']])The adapter retains scores and reasons, derives a prompt identifier from the saved prompt hash, and compares the corpus/case hashes, embedder, retrieval count, threshold, adapter mapping and applicable checks. Missing or duplicate cases change the comparison contract. Missing metrics, nonfinite scores and inconsistent first-failure fields are rejected. Not-applicable retrieval is omitted, not marked passed. Its explicit documentation policy makes tools and faithfulness critical; it does not replace the issue-triage policy above. The saved reports omit the full judge prompt and package version, so matching judge IDs cannot verify those details.
python stage_report_adapter.py fixtures/stage-reports/gemini-3.8-flash.json fixtures/stage-reports/gemma4-e2b.json --out runs/my-stage-adapter/result.json{
"decision": "incomparable",
"reasons": [
"baseline ran docs-assistant@1 (bd96d29ac620e83bdee7cebba2a7aa0d03fbc383c4b1ee5e049093a6d384be73), candidate ran docs-assistant@1 (5b11d6883e7ec7c8d025d237d38a734d6800b7f02da437bf56243c7859652d4e); rerun the baseline on the candidate's dataset"
]
}The saved Gemini and Gemma reports use different embedders, so the adapter refuses a model-only comparison. It returns exit code 2 for incomparable reports and 1 for a blocking decision. Comparing the Gemini report with itself also blocks: its existing pipe-colors faithfulness failure violates the critical policy. Reusing a known failure as a baseline does not waive it. The tests introduce a controlled tool regression under a matching contract and verify that it blocks. They also reject missing metrics and changed judges.
This adapter connects formats, not applications or datasets. It cannot compare these documentation scores with the issue-triage outcomes or clear the separate Python retrieval-task gate.
Next, separate development and held-out cases and check for duplicates or derived cases crossing the split before comparing releases.
Inspect the saved enhancement result
{
"execution_mode": "offline-saved-stage-report-adaptation",
"policy": {
"max_regressions": 0,
"min_pass_rate": 0.9,
"critical": [
"test_tools",
"test_faithfulness"
]
},
"baseline": {
"eval": {
"dataset": "docs-assistant@1",
"dataset_sha": "bd96d29ac620e83bdee7cebba2a7aa0d03fbc383c4b1ee5e049093a6d384be73",
"judge_model": "gemini-3.8-flash",
"model": "gemini-3.8-flash",
"prompt_version": "sha:a7990e4dc62c",
"prompt_sha": "a7990e4dc62c"
},
"tests": [
{
"nodeid": "docs-stages::backup::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::backup::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::backup::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::backup::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::backup::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::pipe-colors::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::pipe-colors::test_retrieval",
"outcome": "failed"
},
{
"nodeid": "docs-stages::pipe-colors::test_faithfulness",
"outcome": "failed"
},
{
"nodeid": "docs-stages::pipe-colors::test_contextual_recall",
"outcome": "failed"
},
{
"nodeid": "docs-stages::pipe-colors::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-212::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-212::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-212::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-212::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-231::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-231::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-231::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-231::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::token-leak::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::token-leak::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::token-leak::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::token-leak::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::token-leak::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::thanks::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::thanks::test_answer_relevancy",
"outcome": "passed"
}
],
"contract": {
"cases": "docs-assistant@1",
"cases_sha": "98b0e5bca6c8",
"docs_sha": "ea13ab95ac63",
"embedder": "gemini-embedding-2",
"k": 3,
"strict_faithfulness": true,
"threshold": 0.7,
"protocol": "docs-stage-scores-v1",
"applicability": {
"backup": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"two-laptops": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"restore": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"config-keys": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"pipe-colors": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"dry-run-import": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"issue-212": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"tools"
],
"issue-231": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"tools"
],
"token-leak": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"thanks": [
"answer_relevancy",
"tools"
]
}
},
"provenance": {
"source_report_sha256": "3102b06ea14a460f880e6e1524511b4525389eebe495d0a77e0d305f0beee601",
"execution_mode": "offline-metric-to-outcome-adapter",
"prompt_version_is_derived": true,
"source_metrics": [
{
"id": "backup",
"answer": "Tracker stores its data in a SQLite file named `tasks.db` inside the list folder, which defaults to `~/.tracker`. To back up every task, tag, and comment, back up that `tasks.db` file. Make sure to close running tracker commands before copying it.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses where Tracker stores its data and what to back up, without any irrelevant information. Great job!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the output is completely aligned with the context without any contradictions, demonstrating perfect faithfulness!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the most relevant information was perfectly prioritized at the very top! The first node in the retrieval contexts directly answers the question by stating that \"Each task list is one SQLite file named tasks.db inside the list folder, ~/.tracker by default\" and instructing to \"Back up that file to back up every task, tag and comment,\" while the irrelevant nodes were correctly ranked lower."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is completely and accurately supported by node 1 in retrieval context, making the retrieval perfectly spot on!"
}
},
"first_failure": null
},
{
"id": "two-laptops",
"answer": "Yes, running sync from two machines at once is safe. `tracker sync` takes a lock on the remote, so a second machine attempting to sync concurrently will wait up to 30 seconds and then exit with \"remote is locked.\"",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses the question with completely relevant information. Great job!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the output is completely faithful to the retrieval context, with no contradictions found! Great job!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the most relevant information is perfectly ranked at the top! The first node in the retrieval contexts directly answers the query, confirming that 'Running sync from two machines at once is safe' and clarifying that 'Sync takes a lock on the remote, so a second machine that syncs at the same time waits up to 30 seconds and then exits with \\'remote is locked\\'', while the irrelevant nodes concerning SQLite file storage and API tokens were correctly placed lower."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because both sentence 1 and sentence 2 are perfectly supported by node 1 in retrieval context, successfully covering all the expected details!"
}
},
"first_failure": null
},
{
"id": "restore",
"answer": "You can bring back an archived task using the command `tracker restore ID`.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses how to restore an archived task with completely relevant and helpful information. Nicely done!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the actual output aligns perfectly with the retrieval context with no contradictions detected. Great job!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly prioritized at the top, directly answering the question by stating that '`tracker restore ID` brings it back' ahead of irrelevant nodes."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is fully supported by node 1 in retrieval context, providing a perfectly accurate retrieval!"
}
},
"first_failure": null
},
{
"id": "config-keys",
"answer": "You can include `default_project`, `week_start` (set to `monday` or `sunday`), `color` (set to `auto`, `always`, or `never`), and `editor` in `config.toml`.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses which keys can be included in config.toml with completely relevant and helpful information."
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because there are no contradictions, and the actual output is completely faithful to the retrieval context!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly placed right at the top, directly answering the query by noting that 'Allowed keys are default_project, week_start (monday or sunday), color (auto, always or never) and editor.', cleanly outranking the irrelevant nodes."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is completely and accurately supported by node 1 in retrieval context. Excellent retrieval quality!"
}
},
"first_failure": null
},
{
"id": "pipe-colors",
"answer": "Color output follows the `color` setting. Set `color = auto` so that tracker automatically turns color off when standard output is not a terminal (such as when piped to `grep`). Alternatively, you can use `tracker list --format csv` for comma-separated output without table formatting.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 0.5,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly and accurately addresses how to disable color codes when piping tracker list, without any irrelevant information."
},
"faithfulness": {
"score": 0.6666666666666666,
"reason": "The score is 0.67 because while the retrieval context confirms color is disabled when standard output is not a terminal under auto settings, it does not specifically mention piping to 'grep' as an example."
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the retrieval contexts are ranked perfectly! The first node in the retrieval contexts is directly relevant, explaining that 'with color = auto, tracker turns color off when standard output is not a terminal', while the irrelevant nodes discussing installation instructions and 'tasks.db' were correctly placed after it."
},
"contextual_recall": {
"score": 0.0,
"reason": "The score is 0.00 because sentence 1 is not fully supported; while node 1 in retrieval context mentions turning color off when output is not a terminal under color = auto, the context completely lacks information on setting color = never in config.toml to turn it off always."
}
},
"first_failure": "retrieval"
},
{
"id": "dry-run-import",
"answer": "You can use the `--dry-run` flag when running the import command, such as `tracker import FILE.csv --dry-run`. This will display the import summary without writing any tasks.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses the question with zero irrelevant information, providing a perfectly focused and helpful answer!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the actual output aligns perfectly with the retrieval context with no contradictions found. Great job!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the retrieval contexts are perfectly ordered! The first node is directly relevant as it explains that you can use `--dry-run` to 'see the summary without writing tasks', rightfully placing it ahead of the irrelevant second and third nodes that focus on 'output formats for `tracker list`' and 'debug logging'."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is completely and accurately supported by node 1 in retrieval context, capturing the required command and flag perfectly!"
}
},
"first_failure": null
},
{
"id": "issue-212",
"answer": "Yes, issue 212 (\"`tracker sync` crashes with KeyError: 'due'\") is closed and fixed. The fix is included in version 2.4.1.",
"calls": [
"get_issue"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": null,
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses both parts of the question clearly and contains no irrelevant information. Nicely done!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the output is completely faithful to the retrieval context with zero contradictions detected. Great job!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the retrieval context performed brilliantly! The first node in the retrieval contexts is perfectly relevant, directly answering the query by showing that issue number 212 has \"state\": \"closed\" and was \"fixed_in\": \"2.4.1\"."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is fully and accurately substantiated by node 1 in retrieval context, capturing all the key details flawlessly!"
}
},
"first_failure": null
},
{
"id": "issue-231",
"answer": "Issue 231, titled \"Import drops rows and reports success\", is currently open.",
"calls": [
"get_issue"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": null,
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly and accurately addresses the status of issue 231 without any irrelevant information. Great job!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the actual output aligns perfectly with the retrieval context, with no contradictions found! Great job!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly relevant, directly specifying that issue number 231 has the title 'Import drops rows and reports success' and its state is 'open'."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is fully supported by node 1 in retrieval context, capturing the required information perfectly!"
}
},
"first_failure": null
},
{
"id": "token-leak",
"answer": "Issue 218 is fixed in version 2.4.1. When using `--debug`, request details are written to `debug.log` in the list folder.",
"calls": [
"get_issue",
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly and accurately addresses both questions with no irrelevant information included. Great job!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the actual output perfectly aligns with the retrieval context with no contradictions whatsoever. Fantastic job!"
},
"contextual_precision": {
"score": 0.75,
"reason": "The score is 0.75 because the first node in retrieval contexts is correctly placed at the top, directly answering the query by stating '`--debug` writes request details to debug.log in the list folder' and that 'Version 2.4.1 redacts API tokens in debug output'. However, the score is not higher because the relevant fourth node, which confirms issue 218 was closed with '\"fixed_in\": \"2.4.1\"', is ranked below irrelevant nodes: the second node merely discusses 'output formats and color settings for `tracker list`', and the third node only details 'how `tracker login` stores tokens and when they expire, without addressing debug output or version fixes for issue 218'."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 in the expected output is fully supported by node 1 and node 4 in retrieval context, accurately capturing all the required details!"
}
},
"first_failure": null
},
{
"id": "thanks",
"answer": "You're very welcome! I'm glad to hear that tracker has been working well for your team. Let me know if you have any questions or need help with anything!",
"calls": [],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": null,
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response is completely relevant and spot-on with no irrelevant statements included. Great job!"
}
},
"first_failure": null
}
]
}
},
"candidate": {
"eval": {
"dataset": "docs-assistant@1",
"dataset_sha": "5b11d6883e7ec7c8d025d237d38a734d6800b7f02da437bf56243c7859652d4e",
"judge_model": "gemini-3.8-flash",
"model": "gemma4:e2b",
"prompt_version": "sha:a7990e4dc62c",
"prompt_sha": "a7990e4dc62c"
},
"tests": [
{
"nodeid": "docs-stages::backup::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::backup::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::backup::test_faithfulness",
"outcome": "failed"
},
{
"nodeid": "docs-stages::backup::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::backup::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::two-laptops::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::restore::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::config-keys::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::pipe-colors::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::pipe-colors::test_retrieval",
"outcome": "failed"
},
{
"nodeid": "docs-stages::pipe-colors::test_faithfulness",
"outcome": "failed"
},
{
"nodeid": "docs-stages::pipe-colors::test_contextual_recall",
"outcome": "failed"
},
{
"nodeid": "docs-stages::pipe-colors::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_retrieval",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::dry-run-import::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-212::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-212::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-212::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-212::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-231::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-231::test_faithfulness",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-231::test_contextual_recall",
"outcome": "passed"
},
{
"nodeid": "docs-stages::issue-231::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::token-leak::test_tools",
"outcome": "failed"
},
{
"nodeid": "docs-stages::token-leak::test_retrieval",
"outcome": "failed"
},
{
"nodeid": "docs-stages::token-leak::test_faithfulness",
"outcome": "failed"
},
{
"nodeid": "docs-stages::token-leak::test_contextual_recall",
"outcome": "failed"
},
{
"nodeid": "docs-stages::token-leak::test_answer_relevancy",
"outcome": "passed"
},
{
"nodeid": "docs-stages::thanks::test_tools",
"outcome": "passed"
},
{
"nodeid": "docs-stages::thanks::test_answer_relevancy",
"outcome": "passed"
}
],
"contract": {
"cases": "docs-assistant@1",
"cases_sha": "98b0e5bca6c8",
"docs_sha": "ea13ab95ac63",
"embedder": "qwen3-embedding:0.6b",
"k": 3,
"strict_faithfulness": true,
"threshold": 0.7,
"protocol": "docs-stage-scores-v1",
"applicability": {
"backup": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"two-laptops": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"restore": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"config-keys": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"pipe-colors": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"dry-run-import": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"issue-212": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"tools"
],
"issue-231": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"tools"
],
"token-leak": [
"answer_relevancy",
"contextual_recall",
"faithfulness",
"retrieval",
"tools"
],
"thanks": [
"answer_relevancy",
"tools"
]
}
},
"provenance": {
"source_report_sha256": "e7b6bea91fc1de5b9f72ad0d484ee6a56af260a58e0c59db5fba6178ff5a9390",
"execution_mode": "offline-metric-to-outcome-adapter",
"prompt_version_is_derived": true,
"source_metrics": [
{
"id": "backup",
"answer": "Tracker stores each task list in an SQLite file named `tasks.db` within the list folder, typically located at `~/.tracker` by default. You should back up the `tasks.db` file for every task, tag, and comment.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly and completely answers both where tracker stores its data and what to back up, without any irrelevant details. Excellent work!"
},
"faithfulness": {
"score": 0.6666666666666666,
"reason": "The score is 0.67 because the output incorrectly prescribes backing up tasks.db for every task, tag, and comment, whereas the context only notes that backing it up preserves these items."
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the retrieval context perfectly prioritized the most relevant information! The first node in the retrieval contexts directly answers the user's question by stating that each task list is 'named tasks.db inside the list folder, ~/.tracker by default' and explicitly instructing to 'Back up that file', successfully placing it above the less relevant nodes that focus on archiving or CSV importing."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is completely and accurately supported by node 1 in retrieval context, achieving perfect recall!"
}
},
"first_failure": "faithfulness"
},
{
"id": "two-laptops",
"answer": "When running `tracker sync` from two machines simultaneously, it is safe. The `tracker sync` command takes a lock on the remote, so a second machine attempting to sync will wait up to 30 seconds before exiting with a \"remote is locked\" message.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly and accurately addresses whether it is safe to sync from two laptops simultaneously with zero irrelevant information. Spot on!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because there are no contradictions, and the output aligns perfectly with the provided context."
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly ranked at the very top! It directly answers the query by clarifying that 'Running sync from two machines at once is safe', while the irrelevant nodes correctly appear after it."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because both sentence 1 and sentence 2 are completely and accurately backed by node 1 in retrieval context, confirming that sync is safe and detailing the exact lock-and-wait behavior!"
}
},
"first_failure": null
},
{
"id": "restore",
"answer": "To bring back a task you archived, you can use the `tracker restore ID` command. This command will restore the task that has the specified ID.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses how to bring back an archived task without any irrelevant information. Nicely done!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because there are no contradictions, and the actual output aligns perfectly with the retrieval context!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the retrieval results are ranked perfectly! The first node in the retrieval contexts is relevant as it directly answers the question by stating that '`tracker restore ID` brings it back', while irrelevant nodes regarding commands like `tracker init --dir ~/tasks` and `tracker import FILE.csv` are correctly placed lower."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because the single sentence in the expected output is completely and accurately backed by node 1 in retrieval context, perfectly capturing the necessary command!"
}
},
"first_failure": null
},
{
"id": "config-keys",
"answer": "The allowed keys in `config.toml` are `default_project`, `week_start` (which can be `monday` or `sunday`), `color` (which can be `auto`, `always`, or `never`), and `editor`.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses which keys can be included in config.toml with completely relevant and helpful information!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because there are no contradictions, and the actual output is completely faithful and perfectly aligned with the retrieval context!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the retrieval contexts are perfectly ordered! The first node in retrieval contexts is the only relevant one, directly answering the question by explicitly stating that 'Allowed keys are default_project, week_start (monday or sunday), color (auto, always or never) and editor.' in config.toml, while irrelevant nodes discussing importing from a CSV file using `tracker import FILE.csv` and installation via `pipx install tracker-cli` are correctly ranked lower at ranks 2 and 3."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 of the expected output is completely supported by node 1 in retrieval context, capturing all the required configuration keys perfectly!"
}
},
"first_failure": null
},
{
"id": "pipe-colors",
"answer": "The color codes are part of the output when using `tracker list`. You can stop the color from printing by using the `--format csv` option with the command.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 0.5,
"reciprocal_rank": 0.5
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly and concisely addresses how to stop color codes from printing into the pipe, without any irrelevant information!"
},
"faithfulness": {
"score": 0.0,
"reason": "The score is 0.00 because the output relies on unconfirmed details, claiming that color codes are included in the 'tracker list' output and that the '--format csv' option prevents color from printing, neither of which is substantiated by the retrieval context."
},
"contextual_precision": {
"score": 0.5,
"reason": "The score is 0.50 because the first node in the retrieval contexts is irrelevant as it \"discusses importing tasks from a CSV file using `tracker import FILE.csv` and has no information about terminal color settings or piping output\", yet it was ranked ahead of the relevant second node. The score remains at 0.50 because the second node is relevant, noting that \"Color follows the color setting; with color = auto, tracker turns color off when standard output is not a terminal\", directly addressing the query."
},
"contextual_recall": {
"score": 0.0,
"reason": "The score is 0.00 because, although node 2 in retrieval context supports tracker turning color off when output is not a terminal under color = auto, sentence 1 is not fully supported as the context fails to mention setting color = never in config.toml to turn it off completely."
}
},
"first_failure": "retrieval"
},
{
"id": "dry-run-import",
"answer": "The `tracker import FILE.csv` command reads columns like title, due, and tags from a CSV file. You can use the `--dry-run` option with this command to see the summary without actually writing any tasks.",
"calls": [
"search_docs"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": {
"hit": true,
"recall": 1.0,
"reciprocal_rank": 1.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses how to check a CSV import without tasks, without any irrelevant information. Great job!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the output is completely faithful to the retrieval context with zero contradictions found. Excellent work!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the relevant information is ranked perfectly at the very top! The first node directly answers the input by showing how to 'Use `--dry-run` to see the summary without writing tasks', while the irrelevant nodes discussing SQLite files and task initialization are correctly placed lower down."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is fully supported by node 1 in retrieval context, perfectly capturing the dry-run command!"
}
},
"first_failure": null
},
{
"id": "issue-212",
"answer": "Issue 212 is closed, and the fix is in version 2.4.1.",
"calls": [
"get_issue"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": null,
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses the status of issue 212 and the relevant version without including any irrelevant information. Great job!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the actual output perfectly aligns with the retrieval context with no contradictions found. Excellent work!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly relevant, directly stating that for issue number 212, the 'state' is 'closed' and it is 'fixed_in': '2.4.1', with no irrelevant nodes retrieved."
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is fully supported by node 1 in retrieval context, capturing all the details about issue 212 perfectly!"
}
},
"first_failure": null
},
{
"id": "issue-231",
"answer": "The status of issue 231 is open. It has no version that fixed it.",
"calls": [
"get_issue"
],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": null,
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly addresses the status of issue 231 without including any irrelevant statements. Excellent work!"
},
"faithfulness": {
"score": 1.0,
"reason": "The score is 1.00 because the output is completely faithful to the context with no contradictions detected, reflecting stellar accuracy!"
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the first node in retrieval contexts is perfectly relevant, directly providing the status and title for issue 231 by stating '\"number\": 231', '\"title\": \"Import drops rows and reports success\"', and '\"state\": \"open\"', seamlessly addressing the query right at the top rank!"
},
"contextual_recall": {
"score": 1.0,
"reason": "The score is 1.00 because sentence 1 is completely verified by node 1 in retrieval context, accurately confirming the open status of issue 231!"
}
},
"first_failure": null
},
{
"id": "token-leak",
"answer": "The issue with `--debug` leaking the API token is fixed in version 2.4.1. The documentation does not specify where debug output goes.",
"calls": [
"get_issue"
],
"tools": {
"score": 0.5,
"unexpected": [],
"pass": false
},
"retrieval": {
"hit": false,
"recall": 0.0,
"reciprocal_rank": 0.0
},
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response directly and accurately addresses both questions with zero irrelevant information. Fantastic job!"
},
"faithfulness": {
"score": 0.5,
"reason": "The score is 0.50 because the output refers to documentation and details where debug output goes, neither of which is supported by the retrieval context."
},
"contextual_precision": {
"score": 1.0,
"reason": "The score is 1.00 because the first node in the retrieval contexts is completely relevant, directly answering the question by stating '\"fixed_in\": \"2.4.1\"' for issue number 218, achieving an ideal ranking with no irrelevant nodes."
},
"contextual_recall": {
"score": 0.0,
"reason": "The score is 0.00 because sentence 1 cannot be adequately recalled; while the 1st node in retrieval context notes that the issue was fixed in version 2.4.1, it lacks information regarding redacting tokens and sending debug output to debug.log in the list folder."
}
},
"first_failure": "tools"
},
{
"id": "thanks",
"answer": "I'm glad to hear that you are enjoying tracker! Is there anything specific you'd like to know about tracker, such as how it works or a particular feature?",
"calls": [],
"tools": {
"score": 1.0,
"unexpected": [],
"pass": true
},
"retrieval": null,
"judged": {
"answer_relevancy": {
"score": 1.0,
"reason": "The score is 1.00 because the response is completely relevant to the input with zero irrelevant statements. Great job!"
}
},
"first_failure": null
}
]
}
},
"decision": {
"decision": "incomparable",
"reasons": [
"baseline ran docs-assistant@1 (bd96d29ac620e83bdee7cebba2a7aa0d03fbc383c4b1ee5e049093a6d384be73), candidate ran docs-assistant@1 (5b11d6883e7ec7c8d025d237d38a734d6800b7f02da437bf56243c7859652d4e); rerun the baseline on the candidate's dataset"
]
}
}References
- GitHub Actions: workflow syntax and actions/checkout. Checked October 21, 2026.
- pytest: exit codes. Checked October 21, 2026.
- pytest-json-report. Checked October 21, 2026.
- DeepEval: G-Eval. Checked October 21, 2026.