{ "execution_mode": "offline-saved-stage-report-adaptation", "policy": { "max_regressions": 0, "min_pass_rate": 0.9, "critical": [ "test_tools", "test_faithfulness" ] }, "baseline": { "eval": { "dataset": "docs-assistant@1", "dataset_sha": "bd96d29ac620e83bdee7cebba2a7aa0d03fbc383c4b1ee5e049093a6d384be73", "judge_model": "gemini-3.8-flash", "model": "gemini-3.8-flash", "prompt_version": "sha:a7990e4dc62c", "prompt_sha": "a7990e4dc62c" }, "tests": [ { "nodeid": "docs-stages::backup::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::backup::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::backup::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::backup::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::backup::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::pipe-colors::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::pipe-colors::test_retrieval", "outcome": "failed" }, { "nodeid": "docs-stages::pipe-colors::test_faithfulness", "outcome": "failed" }, { "nodeid": "docs-stages::pipe-colors::test_contextual_recall", "outcome": "failed" }, { "nodeid": "docs-stages::pipe-colors::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::issue-212::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::issue-212::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::issue-212::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::issue-212::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::issue-231::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::issue-231::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::issue-231::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::issue-231::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::token-leak::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::token-leak::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::token-leak::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::token-leak::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::token-leak::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::thanks::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::thanks::test_answer_relevancy", "outcome": "passed" } ], "contract": { "cases": "docs-assistant@1", "cases_sha": "98b0e5bca6c8", "docs_sha": "ea13ab95ac63", "embedder": "gemini-embedding-2", "k": 3, "strict_faithfulness": true, "threshold": 0.7, "protocol": "docs-stage-scores-v1", "applicability": { "backup": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "two-laptops": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "restore": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "config-keys": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "pipe-colors": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "dry-run-import": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "issue-212": [ "answer_relevancy", "contextual_recall", "faithfulness", "tools" ], "issue-231": [ "answer_relevancy", "contextual_recall", "faithfulness", "tools" ], "token-leak": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "thanks": [ "answer_relevancy", "tools" ] } }, "provenance": { "source_report_sha256": "3102b06ea14a460f880e6e1524511b4525389eebe495d0a77e0d305f0beee601", "execution_mode": "offline-metric-to-outcome-adapter", "prompt_version_is_derived": true, "source_metrics": [ { "id": "backup", "answer": "Tracker stores its data in a SQLite file named `tasks.db` inside the list folder, which defaults to `~/.tracker`. To back up every task, tag, and comment, back up that `tasks.db` file. Make sure to close running tracker commands before copying it.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses where Tracker stores its data and what to back up, without any irrelevant information. Great job!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the output is completely aligned with the context without any contradictions, demonstrating perfect faithfulness!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the most relevant information was perfectly prioritized at the very top! The first node in the retrieval contexts directly answers the question by stating that \"Each task list is one SQLite file named tasks.db inside the list folder, ~/.tracker by default\" and instructing to \"Back up that file to back up every task, tag and comment,\" while the irrelevant nodes were correctly ranked lower." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is completely and accurately supported by node 1 in retrieval context, making the retrieval perfectly spot on!" } }, "first_failure": null }, { "id": "two-laptops", "answer": "Yes, running sync from two machines at once is safe. `tracker sync` takes a lock on the remote, so a second machine attempting to sync concurrently will wait up to 30 seconds and then exit with \"remote is locked.\"", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses the question with completely relevant information. Great job!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the output is completely faithful to the retrieval context, with no contradictions found! Great job!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the most relevant information is perfectly ranked at the top! The first node in the retrieval contexts directly answers the query, confirming that 'Running sync from two machines at once is safe' and clarifying that 'Sync takes a lock on the remote, so a second machine that syncs at the same time waits up to 30 seconds and then exits with \\'remote is locked\\'', while the irrelevant nodes concerning SQLite file storage and API tokens were correctly placed lower." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because both sentence 1 and sentence 2 are perfectly supported by node 1 in retrieval context, successfully covering all the expected details!" } }, "first_failure": null }, { "id": "restore", "answer": "You can bring back an archived task using the command `tracker restore ID`.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses how to restore an archived task with completely relevant and helpful information. Nicely done!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the actual output aligns perfectly with the retrieval context with no contradictions detected. Great job!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly prioritized at the top, directly answering the question by stating that '`tracker restore ID` brings it back' ahead of irrelevant nodes." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is fully supported by node 1 in retrieval context, providing a perfectly accurate retrieval!" } }, "first_failure": null }, { "id": "config-keys", "answer": "You can include `default_project`, `week_start` (set to `monday` or `sunday`), `color` (set to `auto`, `always`, or `never`), and `editor` in `config.toml`.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses which keys can be included in config.toml with completely relevant and helpful information." }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because there are no contradictions, and the actual output is completely faithful to the retrieval context!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly placed right at the top, directly answering the query by noting that 'Allowed keys are default_project, week_start (monday or sunday), color (auto, always or never) and editor.', cleanly outranking the irrelevant nodes." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is completely and accurately supported by node 1 in retrieval context. Excellent retrieval quality!" } }, "first_failure": null }, { "id": "pipe-colors", "answer": "Color output follows the `color` setting. Set `color = auto` so that tracker automatically turns color off when standard output is not a terminal (such as when piped to `grep`). Alternatively, you can use `tracker list --format csv` for comma-separated output without table formatting.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 0.5, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly and accurately addresses how to disable color codes when piping tracker list, without any irrelevant information." }, "faithfulness": { "score": 0.6666666666666666, "reason": "The score is 0.67 because while the retrieval context confirms color is disabled when standard output is not a terminal under auto settings, it does not specifically mention piping to 'grep' as an example." }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the retrieval contexts are ranked perfectly! The first node in the retrieval contexts is directly relevant, explaining that 'with color = auto, tracker turns color off when standard output is not a terminal', while the irrelevant nodes discussing installation instructions and 'tasks.db' were correctly placed after it." }, "contextual_recall": { "score": 0.0, "reason": "The score is 0.00 because sentence 1 is not fully supported; while node 1 in retrieval context mentions turning color off when output is not a terminal under color = auto, the context completely lacks information on setting color = never in config.toml to turn it off always." } }, "first_failure": "retrieval" }, { "id": "dry-run-import", "answer": "You can use the `--dry-run` flag when running the import command, such as `tracker import FILE.csv --dry-run`. This will display the import summary without writing any tasks.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses the question with zero irrelevant information, providing a perfectly focused and helpful answer!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the actual output aligns perfectly with the retrieval context with no contradictions found. Great job!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the retrieval contexts are perfectly ordered! The first node is directly relevant as it explains that you can use `--dry-run` to 'see the summary without writing tasks', rightfully placing it ahead of the irrelevant second and third nodes that focus on 'output formats for `tracker list`' and 'debug logging'." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is completely and accurately supported by node 1 in retrieval context, capturing the required command and flag perfectly!" } }, "first_failure": null }, { "id": "issue-212", "answer": "Yes, issue 212 (\"`tracker sync` crashes with KeyError: 'due'\") is closed and fixed. The fix is included in version 2.4.1.", "calls": [ "get_issue" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": null, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses both parts of the question clearly and contains no irrelevant information. Nicely done!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the output is completely faithful to the retrieval context with zero contradictions detected. Great job!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the retrieval context performed brilliantly! The first node in the retrieval contexts is perfectly relevant, directly answering the query by showing that issue number 212 has \"state\": \"closed\" and was \"fixed_in\": \"2.4.1\"." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is fully and accurately substantiated by node 1 in retrieval context, capturing all the key details flawlessly!" } }, "first_failure": null }, { "id": "issue-231", "answer": "Issue 231, titled \"Import drops rows and reports success\", is currently open.", "calls": [ "get_issue" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": null, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly and accurately addresses the status of issue 231 without any irrelevant information. Great job!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the actual output aligns perfectly with the retrieval context, with no contradictions found! Great job!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly relevant, directly specifying that issue number 231 has the title 'Import drops rows and reports success' and its state is 'open'." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is fully supported by node 1 in retrieval context, capturing the required information perfectly!" } }, "first_failure": null }, { "id": "token-leak", "answer": "Issue 218 is fixed in version 2.4.1. When using `--debug`, request details are written to `debug.log` in the list folder.", "calls": [ "get_issue", "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly and accurately addresses both questions with no irrelevant information included. Great job!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the actual output perfectly aligns with the retrieval context with no contradictions whatsoever. Fantastic job!" }, "contextual_precision": { "score": 0.75, "reason": "The score is 0.75 because the first node in retrieval contexts is correctly placed at the top, directly answering the query by stating '`--debug` writes request details to debug.log in the list folder' and that 'Version 2.4.1 redacts API tokens in debug output'. However, the score is not higher because the relevant fourth node, which confirms issue 218 was closed with '\"fixed_in\": \"2.4.1\"', is ranked below irrelevant nodes: the second node merely discusses 'output formats and color settings for `tracker list`', and the third node only details 'how `tracker login` stores tokens and when they expire, without addressing debug output or version fixes for issue 218'." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 in the expected output is fully supported by node 1 and node 4 in retrieval context, accurately capturing all the required details!" } }, "first_failure": null }, { "id": "thanks", "answer": "You're very welcome! I'm glad to hear that tracker has been working well for your team. Let me know if you have any questions or need help with anything!", "calls": [], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": null, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response is completely relevant and spot-on with no irrelevant statements included. Great job!" } }, "first_failure": null } ] } }, "candidate": { "eval": { "dataset": "docs-assistant@1", "dataset_sha": "5b11d6883e7ec7c8d025d237d38a734d6800b7f02da437bf56243c7859652d4e", "judge_model": "gemini-3.8-flash", "model": "gemma4:e2b", "prompt_version": "sha:a7990e4dc62c", "prompt_sha": "a7990e4dc62c" }, "tests": [ { "nodeid": "docs-stages::backup::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::backup::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::backup::test_faithfulness", "outcome": "failed" }, { "nodeid": "docs-stages::backup::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::backup::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::two-laptops::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::restore::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::config-keys::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::pipe-colors::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::pipe-colors::test_retrieval", "outcome": "failed" }, { "nodeid": "docs-stages::pipe-colors::test_faithfulness", "outcome": "failed" }, { "nodeid": "docs-stages::pipe-colors::test_contextual_recall", "outcome": "failed" }, { "nodeid": "docs-stages::pipe-colors::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_retrieval", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::dry-run-import::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::issue-212::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::issue-212::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::issue-212::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::issue-212::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::issue-231::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::issue-231::test_faithfulness", "outcome": "passed" }, { "nodeid": "docs-stages::issue-231::test_contextual_recall", "outcome": "passed" }, { "nodeid": "docs-stages::issue-231::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::token-leak::test_tools", "outcome": "failed" }, { "nodeid": "docs-stages::token-leak::test_retrieval", "outcome": "failed" }, { "nodeid": "docs-stages::token-leak::test_faithfulness", "outcome": "failed" }, { "nodeid": "docs-stages::token-leak::test_contextual_recall", "outcome": "failed" }, { "nodeid": "docs-stages::token-leak::test_answer_relevancy", "outcome": "passed" }, { "nodeid": "docs-stages::thanks::test_tools", "outcome": "passed" }, { "nodeid": "docs-stages::thanks::test_answer_relevancy", "outcome": "passed" } ], "contract": { "cases": "docs-assistant@1", "cases_sha": "98b0e5bca6c8", "docs_sha": "ea13ab95ac63", "embedder": "qwen3-embedding:0.6b", "k": 3, "strict_faithfulness": true, "threshold": 0.7, "protocol": "docs-stage-scores-v1", "applicability": { "backup": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "two-laptops": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "restore": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "config-keys": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "pipe-colors": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "dry-run-import": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "issue-212": [ "answer_relevancy", "contextual_recall", "faithfulness", "tools" ], "issue-231": [ "answer_relevancy", "contextual_recall", "faithfulness", "tools" ], "token-leak": [ "answer_relevancy", "contextual_recall", "faithfulness", "retrieval", "tools" ], "thanks": [ "answer_relevancy", "tools" ] } }, "provenance": { "source_report_sha256": "e7b6bea91fc1de5b9f72ad0d484ee6a56af260a58e0c59db5fba6178ff5a9390", "execution_mode": "offline-metric-to-outcome-adapter", "prompt_version_is_derived": true, "source_metrics": [ { "id": "backup", "answer": "Tracker stores each task list in an SQLite file named `tasks.db` within the list folder, typically located at `~/.tracker` by default. You should back up the `tasks.db` file for every task, tag, and comment.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly and completely answers both where tracker stores its data and what to back up, without any irrelevant details. Excellent work!" }, "faithfulness": { "score": 0.6666666666666666, "reason": "The score is 0.67 because the output incorrectly prescribes backing up tasks.db for every task, tag, and comment, whereas the context only notes that backing it up preserves these items." }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the retrieval context perfectly prioritized the most relevant information! The first node in the retrieval contexts directly answers the user's question by stating that each task list is 'named tasks.db inside the list folder, ~/.tracker by default' and explicitly instructing to 'Back up that file', successfully placing it above the less relevant nodes that focus on archiving or CSV importing." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is completely and accurately supported by node 1 in retrieval context, achieving perfect recall!" } }, "first_failure": "faithfulness" }, { "id": "two-laptops", "answer": "When running `tracker sync` from two machines simultaneously, it is safe. The `tracker sync` command takes a lock on the remote, so a second machine attempting to sync will wait up to 30 seconds before exiting with a \"remote is locked\" message.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly and accurately addresses whether it is safe to sync from two laptops simultaneously with zero irrelevant information. Spot on!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because there are no contradictions, and the output aligns perfectly with the provided context." }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly ranked at the very top! It directly answers the query by clarifying that 'Running sync from two machines at once is safe', while the irrelevant nodes correctly appear after it." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because both sentence 1 and sentence 2 are completely and accurately backed by node 1 in retrieval context, confirming that sync is safe and detailing the exact lock-and-wait behavior!" } }, "first_failure": null }, { "id": "restore", "answer": "To bring back a task you archived, you can use the `tracker restore ID` command. This command will restore the task that has the specified ID.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses how to bring back an archived task without any irrelevant information. Nicely done!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because there are no contradictions, and the actual output aligns perfectly with the retrieval context!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the retrieval results are ranked perfectly! The first node in the retrieval contexts is relevant as it directly answers the question by stating that '`tracker restore ID` brings it back', while irrelevant nodes regarding commands like `tracker init --dir ~/tasks` and `tracker import FILE.csv` are correctly placed lower." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because the single sentence in the expected output is completely and accurately backed by node 1 in retrieval context, perfectly capturing the necessary command!" } }, "first_failure": null }, { "id": "config-keys", "answer": "The allowed keys in `config.toml` are `default_project`, `week_start` (which can be `monday` or `sunday`), `color` (which can be `auto`, `always`, or `never`), and `editor`.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses which keys can be included in config.toml with completely relevant and helpful information!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because there are no contradictions, and the actual output is completely faithful and perfectly aligned with the retrieval context!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the retrieval contexts are perfectly ordered! The first node in retrieval contexts is the only relevant one, directly answering the question by explicitly stating that 'Allowed keys are default_project, week_start (monday or sunday), color (auto, always or never) and editor.' in config.toml, while irrelevant nodes discussing importing from a CSV file using `tracker import FILE.csv` and installation via `pipx install tracker-cli` are correctly ranked lower at ranks 2 and 3." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 of the expected output is completely supported by node 1 in retrieval context, capturing all the required configuration keys perfectly!" } }, "first_failure": null }, { "id": "pipe-colors", "answer": "The color codes are part of the output when using `tracker list`. You can stop the color from printing by using the `--format csv` option with the command.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 0.5, "reciprocal_rank": 0.5 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly and concisely addresses how to stop color codes from printing into the pipe, without any irrelevant information!" }, "faithfulness": { "score": 0.0, "reason": "The score is 0.00 because the output relies on unconfirmed details, claiming that color codes are included in the 'tracker list' output and that the '--format csv' option prevents color from printing, neither of which is substantiated by the retrieval context." }, "contextual_precision": { "score": 0.5, "reason": "The score is 0.50 because the first node in the retrieval contexts is irrelevant as it \"discusses importing tasks from a CSV file using `tracker import FILE.csv` and has no information about terminal color settings or piping output\", yet it was ranked ahead of the relevant second node. The score remains at 0.50 because the second node is relevant, noting that \"Color follows the color setting; with color = auto, tracker turns color off when standard output is not a terminal\", directly addressing the query." }, "contextual_recall": { "score": 0.0, "reason": "The score is 0.00 because, although node 2 in retrieval context supports tracker turning color off when output is not a terminal under color = auto, sentence 1 is not fully supported as the context fails to mention setting color = never in config.toml to turn it off completely." } }, "first_failure": "retrieval" }, { "id": "dry-run-import", "answer": "The `tracker import FILE.csv` command reads columns like title, due, and tags from a CSV file. You can use the `--dry-run` option with this command to see the summary without actually writing any tasks.", "calls": [ "search_docs" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": { "hit": true, "recall": 1.0, "reciprocal_rank": 1.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses how to check a CSV import without tasks, without any irrelevant information. Great job!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the output is completely faithful to the retrieval context with zero contradictions found. Excellent work!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the relevant information is ranked perfectly at the very top! The first node directly answers the input by showing how to 'Use `--dry-run` to see the summary without writing tasks', while the irrelevant nodes discussing SQLite files and task initialization are correctly placed lower down." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is fully supported by node 1 in retrieval context, perfectly capturing the dry-run command!" } }, "first_failure": null }, { "id": "issue-212", "answer": "Issue 212 is closed, and the fix is in version 2.4.1.", "calls": [ "get_issue" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": null, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses the status of issue 212 and the relevant version without including any irrelevant information. Great job!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the actual output perfectly aligns with the retrieval context with no contradictions found. Excellent work!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the first node in the retrieval contexts is perfectly relevant, directly stating that for issue number 212, the 'state' is 'closed' and it is 'fixed_in': '2.4.1', with no irrelevant nodes retrieved." }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is fully supported by node 1 in retrieval context, capturing all the details about issue 212 perfectly!" } }, "first_failure": null }, { "id": "issue-231", "answer": "The status of issue 231 is open. It has no version that fixed it.", "calls": [ "get_issue" ], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": null, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly addresses the status of issue 231 without including any irrelevant statements. Excellent work!" }, "faithfulness": { "score": 1.0, "reason": "The score is 1.00 because the output is completely faithful to the context with no contradictions detected, reflecting stellar accuracy!" }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the first node in retrieval contexts is perfectly relevant, directly providing the status and title for issue 231 by stating '\"number\": 231', '\"title\": \"Import drops rows and reports success\"', and '\"state\": \"open\"', seamlessly addressing the query right at the top rank!" }, "contextual_recall": { "score": 1.0, "reason": "The score is 1.00 because sentence 1 is completely verified by node 1 in retrieval context, accurately confirming the open status of issue 231!" } }, "first_failure": null }, { "id": "token-leak", "answer": "The issue with `--debug` leaking the API token is fixed in version 2.4.1. The documentation does not specify where debug output goes.", "calls": [ "get_issue" ], "tools": { "score": 0.5, "unexpected": [], "pass": false }, "retrieval": { "hit": false, "recall": 0.0, "reciprocal_rank": 0.0 }, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response directly and accurately addresses both questions with zero irrelevant information. Fantastic job!" }, "faithfulness": { "score": 0.5, "reason": "The score is 0.50 because the output refers to documentation and details where debug output goes, neither of which is supported by the retrieval context." }, "contextual_precision": { "score": 1.0, "reason": "The score is 1.00 because the first node in the retrieval contexts is completely relevant, directly answering the question by stating '\"fixed_in\": \"2.4.1\"' for issue number 218, achieving an ideal ranking with no irrelevant nodes." }, "contextual_recall": { "score": 0.0, "reason": "The score is 0.00 because sentence 1 cannot be adequately recalled; while the 1st node in retrieval context notes that the issue was fixed in version 2.4.1, it lacks information regarding redacting tokens and sending debug output to debug.log in the list folder." } }, "first_failure": "tools" }, { "id": "thanks", "answer": "I'm glad to hear that you are enjoying tracker! Is there anything specific you'd like to know about tracker, such as how it works or a particular feature?", "calls": [], "tools": { "score": 1.0, "unexpected": [], "pass": true }, "retrieval": null, "judged": { "answer_relevancy": { "score": 1.0, "reason": "The score is 1.00 because the response is completely relevant to the input with zero irrelevant statements. Great job!" } }, "first_failure": null } ] } }, "decision": { "decision": "incomparable", "reasons": [ "baseline ran docs-assistant@1 (bd96d29ac620e83bdee7cebba2a7aa0d03fbc383c4b1ee5e049093a6d384be73), candidate ran docs-assistant@1 (5b11d6883e7ec7c8d025d237d38a734d6800b7f02da437bf56243c7859652d4e); rerun the baseline on the candidate's dataset" ] } }