{"benchmarkVersion":"2026.10.2","retrieval":"lexical","datasetSnapshot":"2026.10.17","evaluationDefinition":"Mean scenario recall and precision against maintainer-specified relevant and distractor incident IDs, evaluated over curated real-incident leads only (illustrative scenarios and live advisories excluded).","totalScenarios":10,"meanRecallAt3":0.4666666666666667,"meanRecallAt5":0.5,"meanPrecisionAt3":0.4666666666666667,"meanPrecisionAt5":0.36,"distractorInclusionRate":0.02,"scenarioResults":[{"scenarioId":"BENCH-001","category":"AI Financial Refunds","query":"Build a multi-tenant SaaS application where an AI support agent reads customer emails and may automatically issue refunds through an external payment API.","retrievedIncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-SEC-014","SEA-AI-005","SEA-AI-003","SEA-SEC-007"],"top3IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-SEC-014"],"top5IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-SEC-014","SEA-AI-005","SEA-AI-003"],"recallAt3":0.3333333333333333,"recallAt5":0.25,"precisionAt3":0.3333333333333333,"precisionAt5":0.2,"distractorHits":0},{"scenarioId":"BENCH-002","category":"Autonomous Cloud Deletion","query":"Build an AI agent that scans cloud infrastructure accounts, detects unused databases and VMs, and automatically deletes them to reduce costs.","retrievedIncidentIds":["SEA-POST-009","SEA-SEC-012","SEA-POST-004","SEA-AI-001","SEA-POST-010","SEA-SEC-004"],"top3IncidentIds":["SEA-POST-009","SEA-SEC-012","SEA-POST-004"],"top5IncidentIds":["SEA-POST-009","SEA-SEC-012","SEA-POST-004","SEA-AI-001","SEA-POST-010"],"recallAt3":0.6666666666666666,"recallAt5":1,"precisionAt3":0.6666666666666666,"precisionAt5":0.6,"distractorHits":0},{"scenarioId":"BENCH-003","category":"Multi-Tenant Session Token Cache API","query":"Build a high-performance multi-tenant REST API with an asynchronous Redis caching layer that caches user session tokens and permission objects.","retrievedIncidentIds":["SEA-SEC-001","SEA-REG-002","SEA-AI-009","SEA-POST-008","SEA-SEC-021","SEA-SEC-024"],"top3IncidentIds":["SEA-SEC-001","SEA-REG-002","SEA-AI-009"],"top5IncidentIds":["SEA-SEC-001","SEA-REG-002","SEA-AI-009","SEA-POST-008","SEA-SEC-021"],"recallAt3":0,"recallAt5":0,"precisionAt3":0,"precisionAt5":0,"distractorHits":0},{"scenarioId":"BENCH-004","category":"Autonomous Coding Shell Agent","query":"Create an AI coding assistant that reads GitHub pull request files, runs bash build scripts in terminal, and commits automated fixes.","retrievedIncidentIds":["SEA-SEC-028","SEA-AI-007","SEA-AI-008","SEA-AI-001","SEA-SEC-026","SEA-AI-003"],"top3IncidentIds":["SEA-SEC-028","SEA-AI-007","SEA-AI-008"],"top5IncidentIds":["SEA-SEC-028","SEA-AI-007","SEA-AI-008","SEA-AI-001","SEA-SEC-026"],"recallAt3":0,"recallAt5":0,"precisionAt3":0,"precisionAt5":0,"distractorHits":0},{"scenarioId":"BENCH-005","category":"AI Tool Credential Security","query":"Build an AI agent with search tools that queries external servers with basic authentication credentials.","retrievedIncidentIds":["SEA-AI-003","SEA-SEC-027","SEA-SEC-028","SEA-AI-004","SEA-SEC-017","SEA-SEC-023"],"top3IncidentIds":["SEA-AI-003","SEA-SEC-027","SEA-SEC-028"],"top5IncidentIds":["SEA-AI-003","SEA-SEC-027","SEA-SEC-028","SEA-AI-004","SEA-SEC-017"],"recallAt3":0,"recallAt5":0.25,"precisionAt3":0,"precisionAt5":0.2,"distractorHits":0},{"scenarioId":"BENCH-006","category":"Employee Expense Reimbursement","query":"Build an automated expense reimbursement system where employees upload receipts and the AI agent approves eligible claims and sends payouts via Stripe.","retrievedIncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-001","SEA-AI-005","SEA-AI-003","SEA-AI-007"],"top3IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-001"],"top5IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-001","SEA-AI-005","SEA-AI-003"],"recallAt3":0.6666666666666666,"recallAt5":0.5,"precisionAt3":0.6666666666666666,"precisionAt5":0.4,"distractorHits":0},{"scenarioId":"BENCH-007","category":"Global Edge Content Routing","query":"Build a global reverse proxy CDN with dynamic configuration compiler and customer traffic routing filters.","retrievedIncidentIds":["SEA-POST-006","SEA-POST-005","SEA-SEC-003","SEA-SEC-010","SEA-SEC-013","SEA-POST-024"],"top3IncidentIds":["SEA-POST-006","SEA-POST-005","SEA-SEC-003"],"top5IncidentIds":["SEA-POST-006","SEA-POST-005","SEA-SEC-003","SEA-SEC-010","SEA-SEC-013"],"recallAt3":1,"recallAt5":1,"precisionAt3":1,"precisionAt5":0.6,"distractorHits":0},{"scenarioId":"BENCH-008","category":"Algorithmic Pricing & Value Allocation","query":"Build an automated algorithmic valuation and asset acquisition system that automatically approves purchase transactions based on ML price predictions.","retrievedIncidentIds":["SEA-SEC-011","SEA-REG-004","SEA-SEC-007","SEA-REG-003","SEA-AI-001","SEA-POST-001"],"top3IncidentIds":["SEA-SEC-011","SEA-REG-004","SEA-SEC-007"],"top5IncidentIds":["SEA-SEC-011","SEA-REG-004","SEA-SEC-007","SEA-REG-003","SEA-AI-001"],"recallAt3":1,"recallAt5":1,"precisionAt3":1,"precisionAt5":0.8,"distractorHits":0},{"scenarioId":"BENCH-009","category":"Support Tooling & Customer Diagnostics","query":"Build a customer support portal where agents can view customer-uploaded log files and reset account credentials on behalf of users.","retrievedIncidentIds":["SEA-SEC-027","SEA-SEC-006","SEA-AI-003","SEA-SEC-008","SEA-SEC-022","SEA-REG-002"],"top3IncidentIds":["SEA-SEC-027","SEA-SEC-006","SEA-AI-003"],"top5IncidentIds":["SEA-SEC-027","SEA-SEC-006","SEA-AI-003","SEA-SEC-008","SEA-SEC-022"],"recallAt3":0.3333333333333333,"recallAt5":0.25,"precisionAt3":0.3333333333333333,"precisionAt5":0.2,"distractorHits":0},{"scenarioId":"BENCH-010","category":"Ransomware-Resilient Order Platform","query":"Build an order and billing platform on virtualized servers that must keep taking orders if the main database or hypervisors are encrypted by ransomware.","retrievedIncidentIds":["SEA-SEC-005","SEA-SEC-014","SEA-POST-001","SEA-SEC-002","SEA-SEC-007","SEA-REG-001"],"top3IncidentIds":["SEA-SEC-005","SEA-SEC-014","SEA-POST-001"],"top5IncidentIds":["SEA-SEC-005","SEA-SEC-014","SEA-POST-001","SEA-SEC-002","SEA-SEC-007"],"recallAt3":0.6666666666666666,"recallAt5":0.75,"precisionAt3":0.6666666666666666,"precisionAt5":0.6,"distractorHits":1}],"hybrid":{"benchmarkVersion":"2026.10.2","retrieval":"hybrid","datasetSnapshot":"2026.10.17","evaluationDefinition":"Mean scenario recall and precision against maintainer-specified relevant and distractor incident IDs, evaluated over curated real-incident leads only (illustrative scenarios and live advisories excluded).","totalScenarios":10,"meanRecallAt3":0.4333333333333334,"meanRecallAt5":0.5,"meanPrecisionAt3":0.4333333333333334,"meanPrecisionAt5":0.4,"distractorInclusionRate":0.02040816326530612,"scenarioResults":[{"scenarioId":"BENCH-001","category":"AI Financial Refunds","query":"Build a multi-tenant SaaS application where an AI support agent reads customer emails and may automatically issue refunds through an external payment API.","retrievedIncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-SEC-014","SEA-AI-004","SEA-AI-003","SEA-AI-005"],"top3IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-SEC-014"],"top5IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-SEC-014","SEA-AI-004","SEA-AI-003"],"recallAt3":0.3333333333333333,"recallAt5":0.25,"precisionAt3":0.3333333333333333,"precisionAt5":0.2,"distractorHits":0},{"scenarioId":"BENCH-002","category":"Autonomous Cloud Deletion","query":"Build an AI agent that scans cloud infrastructure accounts, detects unused databases and VMs, and automatically deletes them to reduce costs.","retrievedIncidentIds":["SEA-AI-001","SEA-POST-004","SEA-POST-009","SEA-SEC-012","SEA-POST-016","SEA-SEC-004"],"top3IncidentIds":["SEA-AI-001","SEA-POST-004","SEA-POST-009"],"top5IncidentIds":["SEA-AI-001","SEA-POST-004","SEA-POST-009","SEA-SEC-012","SEA-POST-016"],"recallAt3":0.6666666666666666,"recallAt5":0.6666666666666666,"precisionAt3":0.6666666666666666,"precisionAt5":0.4,"distractorHits":0},{"scenarioId":"BENCH-003","category":"Multi-Tenant Session Token Cache API","query":"Build a high-performance multi-tenant REST API with an asynchronous Redis caching layer that caches user session tokens and permission objects.","retrievedIncidentIds":["SEA-SEC-001","SEA-AI-004","SEA-SEC-006","SEA-POST-011","SEA-POST-007","SEA-SEC-022"],"top3IncidentIds":["SEA-SEC-001","SEA-AI-004","SEA-SEC-006"],"top5IncidentIds":["SEA-SEC-001","SEA-AI-004","SEA-SEC-006","SEA-POST-011","SEA-POST-007"],"recallAt3":0.3333333333333333,"recallAt5":0.75,"precisionAt3":0.3333333333333333,"precisionAt5":0.6,"distractorHits":0},{"scenarioId":"BENCH-004","category":"Autonomous Coding Shell Agent","query":"Create an AI coding assistant that reads GitHub pull request files, runs bash build scripts in terminal, and commits automated fixes.","retrievedIncidentIds":["SEA-SEC-028","SEA-AI-007","SEA-AI-008","SEA-AI-001","SEA-AI-003","SEA-AI-004"],"top3IncidentIds":["SEA-SEC-028","SEA-AI-007","SEA-AI-008"],"top5IncidentIds":["SEA-SEC-028","SEA-AI-007","SEA-AI-008","SEA-AI-001","SEA-AI-003"],"recallAt3":0,"recallAt5":0,"precisionAt3":0,"precisionAt5":0,"distractorHits":0},{"scenarioId":"BENCH-005","category":"AI Tool Credential Security","query":"Build an AI agent with search tools that queries external servers with basic authentication credentials.","retrievedIncidentIds":["SEA-AI-003","SEA-SEC-028","SEA-AI-004","SEA-SEC-017","SEA-AI-008","SEA-POST-011"],"top3IncidentIds":["SEA-AI-003","SEA-SEC-028","SEA-AI-004"],"top5IncidentIds":["SEA-AI-003","SEA-SEC-028","SEA-AI-004","SEA-SEC-017","SEA-AI-008"],"recallAt3":0,"recallAt5":0.25,"precisionAt3":0,"precisionAt5":0.2,"distractorHits":0},{"scenarioId":"BENCH-006","category":"Employee Expense Reimbursement","query":"Build an automated expense reimbursement system where employees upload receipts and the AI agent approves eligible claims and sends payouts via Stripe.","retrievedIncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-001","SEA-POST-001","SEA-AI-006","SEA-AI-004"],"top3IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-001"],"top5IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-001","SEA-POST-001","SEA-AI-006"],"recallAt3":0.6666666666666666,"recallAt5":0.75,"precisionAt3":0.6666666666666666,"precisionAt5":0.6,"distractorHits":0},{"scenarioId":"BENCH-007","category":"Global Edge Content Routing","query":"Build a global reverse proxy CDN with dynamic configuration compiler and customer traffic routing filters.","retrievedIncidentIds":["SEA-POST-006","SEA-SEC-010","SEA-POST-024","SEA-SEC-007","SEA-SEC-017"],"top3IncidentIds":["SEA-POST-006","SEA-SEC-010","SEA-POST-024"],"top5IncidentIds":["SEA-POST-006","SEA-SEC-010","SEA-POST-024","SEA-SEC-007","SEA-SEC-017"],"recallAt3":0.3333333333333333,"recallAt5":0.3333333333333333,"precisionAt3":0.3333333333333333,"precisionAt5":0.2,"distractorHits":0},{"scenarioId":"BENCH-008","category":"Algorithmic Pricing & Value Allocation","query":"Build an automated algorithmic valuation and asset acquisition system that automatically approves purchase transactions based on ML price predictions.","retrievedIncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-004","SEA-SEC-007"],"top3IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-004"],"top5IncidentIds":["SEA-SEC-011","SEA-REG-003","SEA-REG-004","SEA-SEC-007"],"recallAt3":1,"recallAt5":1,"precisionAt3":1,"precisionAt5":1,"distractorHits":0},{"scenarioId":"BENCH-009","category":"Support Tooling & Customer Diagnostics","query":"Build a customer support portal where agents can view customer-uploaded log files and reset account credentials on behalf of users.","retrievedIncidentIds":["SEA-SEC-006","SEA-SEC-027","SEA-AI-004","SEA-SEC-008","SEA-AI-003","SEA-SEC-020"],"top3IncidentIds":["SEA-SEC-006","SEA-SEC-027","SEA-AI-004"],"top5IncidentIds":["SEA-SEC-006","SEA-SEC-027","SEA-AI-004","SEA-SEC-008","SEA-AI-003"],"recallAt3":0.3333333333333333,"recallAt5":0.25,"precisionAt3":0.3333333333333333,"precisionAt5":0.2,"distractorHits":0},{"scenarioId":"BENCH-010","category":"Ransomware-Resilient Order Platform","query":"Build an order and billing platform on virtualized servers that must keep taking orders if the main database or hypervisors are encrypted by ransomware.","retrievedIncidentIds":["SEA-SEC-005","SEA-SEC-014","SEA-POST-001","SEA-SEC-002","SEA-SEC-007","SEA-REG-004"],"top3IncidentIds":["SEA-SEC-005","SEA-SEC-014","SEA-POST-001"],"top5IncidentIds":["SEA-SEC-005","SEA-SEC-014","SEA-POST-001","SEA-SEC-002","SEA-SEC-007"],"recallAt3":0.6666666666666666,"recallAt5":0.75,"precisionAt3":0.6666666666666666,"precisionAt5":0.6,"distractorHits":1}]},"known_item":{"corpus":"Bundled corpus 2026.09.30: documented incidents and illustrative scenarios (live advisories excluded)","lexical":{"benchmarkVersion":"KI-2026.10.1","retrieval":"lexical","datasetSnapshot":"2026.09.30","evaluationDefinition":"One query per documented incident, describing the affected system before its failure without naming the organization, a CVE, a date or the failure itself (underlying technologies may be named). hitAt5 is the share of queries whose own incident ranks in the top five of the evaluated corpus; patternHitAt5 accepts any top-five match that shares the incident's primary failure pattern.","totalScenarios":65,"hitAt1":0.23076923076923078,"hitAt5":0.4461538461538462,"hitAt5Interval":[0.331693339691393,0.5666239568269913],"meanReciprocalRank":0.3282844932844932,"patternHitAt5":0.6923076923076923,"results":[{"scenarioId":"KI-SEC-001","incidentId":"SEA-SEC-001","rank":1,"top5IncidentIds":["SEA-SEC-001","SEA-SEC-021","SEA-SEC-024","SEA-REG-002","SEA-POST-008"],"patternHitAt5":true},{"scenarioId":"KI-SEC-002","incidentId":"SEA-SEC-002","rank":2,"top5IncidentIds":["SEA-POST-001","SEA-SEC-002","SEA-SEC-019","SEA-REG-001","SEA-SEC-005"],"patternHitAt5":true},{"scenarioId":"KI-SEC-003","incidentId":"SEA-SEC-003","rank":null,"top5IncidentIds":["SEA-AI-007"],"patternHitAt5":false},{"scenarioId":"KI-SEC-004","incidentId":"SEA-SEC-004","rank":null,"top5IncidentIds":["SEA-REG-001","SEA-SEC-002","SEA-POST-001","SEA-REG-003","SEA-SEC-001"],"patternHitAt5":true},{"scenarioId":"KI-SEC-005","incidentId":"SEA-SEC-005","rank":1,"top5IncidentIds":["SEA-SEC-005","SEA-POST-001","SEA-SEC-014","SEA-PUB-044","SEA-SEC-002"],"patternHitAt5":true},{"scenarioId":"KI-SEC-006","incidentId":"SEA-SEC-006","rank":1,"top5IncidentIds":["SEA-SEC-006","SEA-PUB-058"],"patternHitAt5":true},{"scenarioId":"KI-SEC-007","incidentId":"SEA-SEC-007","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-SEC-008","incidentId":"SEA-SEC-008","rank":1,"top5IncidentIds":["SEA-SEC-008","SEA-SEC-027","SEA-SEC-022","SEA-SEC-006","SEA-REG-002"],"patternHitAt5":true},{"scenarioId":"KI-SEC-009","incidentId":"SEA-SEC-009","rank":null,"top5IncidentIds":["SEA-REG-001","SEA-SEC-005","SEA-REG-003","SEA-POST-001","SEA-REG-005"],"patternHitAt5":false},{"scenarioId":"KI-SEC-010","incidentId":"SEA-SEC-010","rank":5,"top5IncidentIds":["SEA-SEC-013","SEA-SEC-003","SEA-POST-006","SEA-POST-005","SEA-SEC-010"],"patternHitAt5":true},{"scenarioId":"KI-SEC-011","incidentId":"SEA-SEC-011","rank":6,"top5IncidentIds":["SEA-SEC-005","SEA-REG-001","SEA-POST-001","SEA-REG-003","SEA-REG-005"],"patternHitAt5":true},{"scenarioId":"KI-SEC-012","incidentId":"SEA-SEC-012","rank":null,"top5IncidentIds":["SEA-POST-001","SEA-SEC-005","SEA-SEC-002","SEA-SEC-019","SEA-SEC-014"],"patternHitAt5":true},{"scenarioId":"KI-SEC-013","incidentId":"SEA-SEC-013","rank":1,"top5IncidentIds":["SEA-SEC-013","SEA-SEC-015","SEA-AI-009"],"patternHitAt5":true},{"scenarioId":"KI-SEC-014","incidentId":"SEA-SEC-014","rank":null,"top5IncidentIds":["SEA-SEC-009","SEA-POST-007","SEA-AI-009","SEA-SEC-001","SEA-POST-004"],"patternHitAt5":false},{"scenarioId":"KI-SEC-015","incidentId":"SEA-SEC-015","rank":null,"top5IncidentIds":["SEA-POST-001","SEA-SEC-005","SEA-SEC-014","SEA-REG-001","SEA-REG-003"],"patternHitAt5":true},{"scenarioId":"KI-SEC-016","incidentId":"SEA-SEC-016","rank":null,"top5IncidentIds":["SEA-REG-002","SEA-AI-009","SEA-PUB-058","SEA-SEC-020","SEA-SEC-001"],"patternHitAt5":true},{"scenarioId":"KI-SEC-017","incidentId":"SEA-SEC-017","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-SEC-018","incidentId":"SEA-SEC-018","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-SEC-019","incidentId":"SEA-SEC-019","rank":2,"top5IncidentIds":["SEA-POST-001","SEA-SEC-019","SEA-SEC-002","SEA-SEC-005","SEA-SEC-014"],"patternHitAt5":true},{"scenarioId":"KI-SEC-020","incidentId":"SEA-SEC-020","rank":null,"top5IncidentIds":["SEA-AI-010"],"patternHitAt5":false},{"scenarioId":"KI-SEC-021","incidentId":"SEA-SEC-021","rank":1,"top5IncidentIds":["SEA-SEC-021","SEA-SEC-020","SEA-SEC-001","SEA-SEC-019","SEA-SEC-022"],"patternHitAt5":true},{"scenarioId":"KI-SEC-022","incidentId":"SEA-SEC-022","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-SEC-023","incidentId":"SEA-SEC-023","rank":null,"top5IncidentIds":["SEA-SEC-001","SEA-SEC-027","SEA-REG-002","SEA-SEC-021","SEA-AI-004"],"patternHitAt5":true},{"scenarioId":"KI-SEC-024","incidentId":"SEA-SEC-024","rank":9,"top5IncidentIds":["SEA-SEC-022","SEA-SEC-027","SEA-SEC-006","SEA-SEC-008","SEA-SEC-010"],"patternHitAt5":true},{"scenarioId":"KI-SEC-025","incidentId":"SEA-SEC-025","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-SEC-026","incidentId":"SEA-SEC-026","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-REG-001","incidentId":"SEA-REG-001","rank":1,"top5IncidentIds":["SEA-REG-001","SEA-SEC-005","SEA-POST-001","SEA-SEC-014","SEA-REG-003"],"patternHitAt5":true},{"scenarioId":"KI-REG-002","incidentId":"SEA-REG-002","rank":7,"top5IncidentIds":["SEA-SEC-027","SEA-SEC-022","SEA-SEC-008","SEA-SEC-006","SEA-SEC-017"],"patternHitAt5":true},{"scenarioId":"KI-REG-003","incidentId":"SEA-REG-003","rank":4,"top5IncidentIds":["SEA-POST-001","SEA-SEC-005","SEA-SEC-014","SEA-REG-003","SEA-SEC-007"],"patternHitAt5":true},{"scenarioId":"KI-REG-004","incidentId":"SEA-REG-004","rank":7,"top5IncidentIds":["SEA-SEC-005","SEA-REG-003","SEA-REG-001","SEA-REG-005","SEA-POST-001"],"patternHitAt5":true},{"scenarioId":"KI-POST-001","incidentId":"SEA-POST-001","rank":1,"top5IncidentIds":["SEA-POST-001","SEA-SEC-002","SEA-REG-003","SEA-PUB-044","SEA-SEC-019"],"patternHitAt5":true},{"scenarioId":"KI-POST-002","incidentId":"SEA-POST-002","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-POST-004","incidentId":"SEA-POST-004","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-POST-005","incidentId":"SEA-POST-005","rank":2,"top5IncidentIds":["SEA-POST-006","SEA-POST-005","SEA-POST-021","SEA-POST-018","SEA-POST-019"],"patternHitAt5":true},{"scenarioId":"KI-POST-006","incidentId":"SEA-POST-006","rank":2,"top5IncidentIds":["SEA-SEC-022","SEA-POST-006","SEA-POST-021","SEA-POST-026","SEA-POST-024"],"patternHitAt5":true},{"scenarioId":"KI-POST-007","incidentId":"SEA-POST-007","rank":4,"top5IncidentIds":["SEA-POST-009","SEA-SEC-022","SEA-PUB-043","SEA-POST-007"],"patternHitAt5":true},{"scenarioId":"KI-POST-008","incidentId":"SEA-POST-008","rank":6,"top5IncidentIds":["SEA-SEC-028","SEA-SEC-029","SEA-AI-003","SEA-AI-004","SEA-AI-008"],"patternHitAt5":true},{"scenarioId":"KI-POST-009","incidentId":"SEA-POST-009","rank":null,"top5IncidentIds":["SEA-SEC-003","SEA-POST-006","SEA-POST-005","SEA-POST-007","SEA-AI-001"],"patternHitAt5":true},{"scenarioId":"KI-POST-010","incidentId":"SEA-POST-010","rank":1,"top5IncidentIds":["SEA-POST-010","SEA-POST-004","SEA-POST-009","SEA-SEC-009","SEA-SEC-018"],"patternHitAt5":true},{"scenarioId":"KI-POST-011","incidentId":"SEA-POST-011","rank":null,"top5IncidentIds":["SEA-SEC-027","SEA-SEC-006","SEA-AI-010"],"patternHitAt5":true},{"scenarioId":"KI-POST-016","incidentId":"SEA-POST-016","rank":5,"top5IncidentIds":["SEA-POST-004","SEA-POST-009","SEA-POST-010","SEA-SEC-012","SEA-POST-016"],"patternHitAt5":true},{"scenarioId":"KI-POST-017","incidentId":"SEA-POST-017","rank":2,"top5IncidentIds":["SEA-POST-024","SEA-POST-017","SEA-POST-019","SEA-PUB-053","SEA-POST-023"],"patternHitAt5":true},{"scenarioId":"KI-POST-018","incidentId":"SEA-POST-018","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-POST-019","incidentId":"SEA-POST-019","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-POST-020","incidentId":"SEA-POST-020","rank":1,"top5IncidentIds":["SEA-POST-020","SEA-POST-013"],"patternHitAt5":true},{"scenarioId":"KI-POST-021","incidentId":"SEA-POST-021","rank":null,"top5IncidentIds":["SEA-SEC-017","SEA-SEC-023","SEA-SEC-027","SEA-SEC-022","SEA-SEC-010"],"patternHitAt5":false},{"scenarioId":"KI-POST-022","incidentId":"SEA-POST-022","rank":null,"top5IncidentIds":["SEA-POST-018","SEA-POST-021","SEA-POST-006","SEA-POST-024","SEA-POST-026"],"patternHitAt5":true},{"scenarioId":"KI-POST-023","incidentId":"SEA-POST-023","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-AI-001","incidentId":"SEA-AI-001","rank":8,"top5IncidentIds":["SEA-POST-005","SEA-POST-006","SEA-SEC-003","SEA-SEC-010","SEA-SEC-013"],"patternHitAt5":false},{"scenarioId":"KI-AI-002","incidentId":"SEA-AI-002","rank":null,"top5IncidentIds":["SEA-AI-001","SEA-SEC-028","SEA-AI-008","SEA-AI-007","SEA-SEC-010"],"patternHitAt5":true},{"scenarioId":"KI-AI-003","incidentId":"SEA-AI-003","rank":1,"top5IncidentIds":["SEA-AI-003","SEA-AI-004","SEA-PUB-040","SEA-AI-005","SEA-SEC-028"],"patternHitAt5":true},{"scenarioId":"KI-AI-004","incidentId":"SEA-AI-004","rank":6,"top5IncidentIds":["SEA-AI-007","SEA-AI-008","SEA-SEC-028","SEA-POST-005","SEA-PUB-056"],"patternHitAt5":true},{"scenarioId":"KI-AI-005","incidentId":"SEA-AI-005","rank":1,"top5IncidentIds":["SEA-AI-005","SEA-AI-007","SEA-AI-003","SEA-AI-004","SEA-SEC-028"],"patternHitAt5":true},{"scenarioId":"KI-AI-006","incidentId":"SEA-AI-006","rank":2,"top5IncidentIds":["SEA-POST-001","SEA-AI-006","SEA-REG-003","SEA-REG-004","SEA-SEC-002"],"patternHitAt5":true},{"scenarioId":"KI-POST-024","incidentId":"SEA-POST-024","rank":1,"top5IncidentIds":["SEA-POST-024","SEA-POST-023","SEA-POST-017","SEA-PUB-053","SEA-POST-019"],"patternHitAt5":true},{"scenarioId":"KI-POST-025","incidentId":"SEA-POST-025","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-POST-026","incidentId":"SEA-POST-026","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-SEC-027","incidentId":"SEA-SEC-027","rank":2,"top5IncidentIds":["SEA-SEC-001","SEA-SEC-027","SEA-AI-009","SEA-AI-004","SEA-REG-002"],"patternHitAt5":true},{"scenarioId":"KI-SEC-028","incidentId":"SEA-SEC-028","rank":3,"top5IncidentIds":["SEA-POST-011","SEA-SEC-010","SEA-SEC-028","SEA-SEC-003","SEA-SEC-017"],"patternHitAt5":true},{"scenarioId":"KI-SEC-029","incidentId":"SEA-SEC-029","rank":null,"top5IncidentIds":["SEA-SEC-027","SEA-SEC-006"],"patternHitAt5":true},{"scenarioId":"KI-SEC-030","incidentId":"SEA-SEC-030","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-AI-007","incidentId":"SEA-AI-007","rank":1,"top5IncidentIds":["SEA-AI-007","SEA-AI-001","SEA-SEC-028","SEA-AI-008","SEA-PUB-041"],"patternHitAt5":true},{"scenarioId":"KI-AI-008","incidentId":"SEA-AI-008","rank":4,"top5IncidentIds":["SEA-AI-007","SEA-AI-001","SEA-SEC-028","SEA-AI-008","SEA-PUB-041"],"patternHitAt5":true},{"scenarioId":"KI-AI-009","incidentId":"SEA-AI-009","rank":3,"top5IncidentIds":["SEA-AI-007","SEA-AI-004","SEA-AI-009","SEA-SEC-001","SEA-SEC-020"],"patternHitAt5":true},{"scenarioId":"KI-AI-010","incidentId":"SEA-AI-010","rank":1,"top5IncidentIds":["SEA-AI-010"],"patternHitAt5":true}]},"hybrid":{"benchmarkVersion":"KI-2026.10.1","retrieval":"hybrid","datasetSnapshot":"2026.09.30","evaluationDefinition":"One query per documented incident, describing the affected system before its failure without naming the organization, a CVE, a date or the failure itself (underlying technologies may be named). hitAt5 is the share of queries whose own incident ranks in the top five of the evaluated corpus; patternHitAt5 accepts any top-five match that shares the incident's primary failure pattern.","totalScenarios":65,"hitAt1":0.7076923076923077,"hitAt5":0.8615384615384616,"hitAt5Interval":[0.7573167852534054,0.9254099381231576],"meanReciprocalRank":0.7615384615384614,"patternHitAt5":0.9230769230769231,"results":[{"scenarioId":"KI-SEC-001","incidentId":"SEA-SEC-001","rank":1,"top5IncidentIds":["SEA-SEC-001","SEA-SEC-021","SEA-SEC-024","SEA-SEC-027","SEA-SEC-020"],"patternHitAt5":true},{"scenarioId":"KI-SEC-002","incidentId":"SEA-SEC-002","rank":1,"top5IncidentIds":["SEA-SEC-002"],"patternHitAt5":true},{"scenarioId":"KI-SEC-003","incidentId":"SEA-SEC-003","rank":4,"top5IncidentIds":["SEA-AI-003","SEA-SEC-028","SEA-PUB-040","SEA-SEC-003","SEA-SEC-010"],"patternHitAt5":true},{"scenarioId":"KI-SEC-004","incidentId":"SEA-SEC-004","rank":1,"top5IncidentIds":["SEA-SEC-004","SEA-SEC-002","SEA-SEC-020","SEA-AI-004","SEA-SEC-019"],"patternHitAt5":true},{"scenarioId":"KI-SEC-005","incidentId":"SEA-SEC-005","rank":1,"top5IncidentIds":["SEA-SEC-005","SEA-SEC-014"],"patternHitAt5":true},{"scenarioId":"KI-SEC-006","incidentId":"SEA-SEC-006","rank":1,"top5IncidentIds":["SEA-SEC-006","SEA-PUB-058"],"patternHitAt5":true},{"scenarioId":"KI-SEC-007","incidentId":"SEA-SEC-007","rank":1,"top5IncidentIds":["SEA-SEC-007"],"patternHitAt5":true},{"scenarioId":"KI-SEC-008","incidentId":"SEA-SEC-008","rank":1,"top5IncidentIds":["SEA-SEC-008","SEA-SEC-022","SEA-SEC-027","SEA-SEC-016","SEA-SEC-024"],"patternHitAt5":true},{"scenarioId":"KI-SEC-009","incidentId":"SEA-SEC-009","rank":5,"top5IncidentIds":["SEA-REG-001","SEA-SEC-019","SEA-PUB-044","SEA-SEC-002","SEA-SEC-009"],"patternHitAt5":true},{"scenarioId":"KI-SEC-010","incidentId":"SEA-SEC-010","rank":1,"top5IncidentIds":["SEA-SEC-010","SEA-SEC-003","SEA-SEC-028","SEA-POST-006","SEA-SEC-017"],"patternHitAt5":true},{"scenarioId":"KI-SEC-011","incidentId":"SEA-SEC-011","rank":1,"top5IncidentIds":["SEA-SEC-011","SEA-REG-004"],"patternHitAt5":true},{"scenarioId":"KI-SEC-012","incidentId":"SEA-SEC-012","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-SEC-013","incidentId":"SEA-SEC-013","rank":1,"top5IncidentIds":["SEA-SEC-013","SEA-SEC-015","SEA-AI-009"],"patternHitAt5":true},{"scenarioId":"KI-SEC-014","incidentId":"SEA-SEC-014","rank":1,"top5IncidentIds":["SEA-SEC-014"],"patternHitAt5":true},{"scenarioId":"KI-SEC-015","incidentId":"SEA-SEC-015","rank":1,"top5IncidentIds":["SEA-SEC-015"],"patternHitAt5":true},{"scenarioId":"KI-SEC-016","incidentId":"SEA-SEC-016","rank":10,"top5IncidentIds":["SEA-SEC-020","SEA-AI-009","SEA-AI-004","SEA-SEC-001","SEA-SEC-021"],"patternHitAt5":true},{"scenarioId":"KI-SEC-017","incidentId":"SEA-SEC-017","rank":1,"top5IncidentIds":["SEA-SEC-017","SEA-SEC-028","SEA-AI-003"],"patternHitAt5":true},{"scenarioId":"KI-SEC-018","incidentId":"SEA-SEC-018","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-SEC-019","incidentId":"SEA-SEC-019","rank":1,"top5IncidentIds":["SEA-SEC-019","SEA-SEC-002","SEA-SEC-005","SEA-POST-014"],"patternHitAt5":true},{"scenarioId":"KI-SEC-020","incidentId":"SEA-SEC-020","rank":1,"top5IncidentIds":["SEA-SEC-020"],"patternHitAt5":true},{"scenarioId":"KI-SEC-021","incidentId":"SEA-SEC-021","rank":1,"top5IncidentIds":["SEA-SEC-021","SEA-SEC-020","SEA-SEC-001","SEA-SEC-019","SEA-SEC-016"],"patternHitAt5":true},{"scenarioId":"KI-SEC-022","incidentId":"SEA-SEC-022","rank":1,"top5IncidentIds":["SEA-SEC-022"],"patternHitAt5":true},{"scenarioId":"KI-SEC-023","incidentId":"SEA-SEC-023","rank":4,"top5IncidentIds":["SEA-SEC-001","SEA-SEC-027","SEA-POST-008","SEA-SEC-023","SEA-AI-004"],"patternHitAt5":true},{"scenarioId":"KI-SEC-024","incidentId":"SEA-SEC-024","rank":1,"top5IncidentIds":["SEA-SEC-024","SEA-SEC-022","SEA-SEC-006","SEA-SEC-010","SEA-POST-011"],"patternHitAt5":true},{"scenarioId":"KI-SEC-025","incidentId":"SEA-SEC-025","rank":1,"top5IncidentIds":["SEA-SEC-025"],"patternHitAt5":true},{"scenarioId":"KI-SEC-026","incidentId":"SEA-SEC-026","rank":1,"top5IncidentIds":["SEA-SEC-026"],"patternHitAt5":true},{"scenarioId":"KI-REG-001","incidentId":"SEA-REG-001","rank":1,"top5IncidentIds":["SEA-REG-001","SEA-REG-004"],"patternHitAt5":true},{"scenarioId":"KI-REG-002","incidentId":"SEA-REG-002","rank":3,"top5IncidentIds":["SEA-SEC-022","SEA-SEC-027","SEA-REG-002","SEA-SEC-008","SEA-SEC-017"],"patternHitAt5":true},{"scenarioId":"KI-REG-003","incidentId":"SEA-REG-003","rank":1,"top5IncidentIds":["SEA-REG-003","SEA-REG-004","SEA-SEC-007"],"patternHitAt5":true},{"scenarioId":"KI-REG-004","incidentId":"SEA-REG-004","rank":3,"top5IncidentIds":["SEA-SEC-005","SEA-REG-003","SEA-REG-004","SEA-REG-005","SEA-REG-001"],"patternHitAt5":true},{"scenarioId":"KI-POST-001","incidentId":"SEA-POST-001","rank":1,"top5IncidentIds":["SEA-POST-001","SEA-PUB-044","SEA-SEC-002","SEA-REG-001","SEA-REG-003"],"patternHitAt5":true},{"scenarioId":"KI-POST-002","incidentId":"SEA-POST-002","rank":1,"top5IncidentIds":["SEA-POST-002","SEA-POST-020","SEA-POST-013"],"patternHitAt5":true},{"scenarioId":"KI-POST-004","incidentId":"SEA-POST-004","rank":1,"top5IncidentIds":["SEA-POST-004"],"patternHitAt5":true},{"scenarioId":"KI-POST-005","incidentId":"SEA-POST-005","rank":1,"top5IncidentIds":["SEA-POST-005","SEA-POST-006","SEA-POST-021","SEA-POST-017","SEA-SEC-010"],"patternHitAt5":true},{"scenarioId":"KI-POST-006","incidentId":"SEA-POST-006","rank":1,"top5IncidentIds":["SEA-POST-006","SEA-SEC-003","SEA-POST-005","SEA-POST-024","SEA-POST-021"],"patternHitAt5":true},{"scenarioId":"KI-POST-007","incidentId":"SEA-POST-007","rank":3,"top5IncidentIds":["SEA-POST-009","SEA-SEC-022","SEA-POST-007","SEA-PUB-043","SEA-POST-005"],"patternHitAt5":true},{"scenarioId":"KI-POST-008","incidentId":"SEA-POST-008","rank":1,"top5IncidentIds":["SEA-POST-008","SEA-AI-003","SEA-SEC-029","SEA-SEC-028","SEA-AI-004"],"patternHitAt5":true},{"scenarioId":"KI-POST-009","incidentId":"SEA-POST-009","rank":null,"top5IncidentIds":["SEA-POST-007","SEA-POST-005","SEA-POST-011","SEA-SEC-017","SEA-SEC-029"],"patternHitAt5":false},{"scenarioId":"KI-POST-010","incidentId":"SEA-POST-010","rank":1,"top5IncidentIds":["SEA-POST-010","SEA-POST-004","SEA-SEC-018","SEA-SEC-015","SEA-SEC-008"],"patternHitAt5":true},{"scenarioId":"KI-POST-011","incidentId":"SEA-POST-011","rank":3,"top5IncidentIds":["SEA-SEC-006","SEA-AI-010","SEA-POST-011","SEA-SEC-017"],"patternHitAt5":true},{"scenarioId":"KI-POST-016","incidentId":"SEA-POST-016","rank":1,"top5IncidentIds":["SEA-POST-016","SEA-POST-004","SEA-POST-009","SEA-POST-010","SEA-SEC-004"],"patternHitAt5":true},{"scenarioId":"KI-POST-017","incidentId":"SEA-POST-017","rank":1,"top5IncidentIds":["SEA-POST-017","SEA-POST-024","SEA-POST-005","SEA-POST-026","SEA-POST-019"],"patternHitAt5":true},{"scenarioId":"KI-POST-018","incidentId":"SEA-POST-018","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-POST-019","incidentId":"SEA-POST-019","rank":1,"top5IncidentIds":["SEA-POST-019"],"patternHitAt5":true},{"scenarioId":"KI-POST-020","incidentId":"SEA-POST-020","rank":1,"top5IncidentIds":["SEA-POST-020","SEA-POST-013","SEA-POST-009","SEA-POST-017","SEA-POST-025"],"patternHitAt5":true},{"scenarioId":"KI-POST-021","incidentId":"SEA-POST-021","rank":1,"top5IncidentIds":["SEA-POST-021","SEA-SEC-010","SEA-POST-005","SEA-SEC-003","SEA-SEC-006"],"patternHitAt5":true},{"scenarioId":"KI-POST-022","incidentId":"SEA-POST-022","rank":null,"top5IncidentIds":["SEA-POST-026","SEA-POST-017","SEA-POST-005","SEA-SEC-003","SEA-POST-016"],"patternHitAt5":true},{"scenarioId":"KI-POST-023","incidentId":"SEA-POST-023","rank":1,"top5IncidentIds":["SEA-POST-023"],"patternHitAt5":true},{"scenarioId":"KI-AI-001","incidentId":"SEA-AI-001","rank":2,"top5IncidentIds":["SEA-POST-005","SEA-AI-001","SEA-PUB-041","SEA-AI-007","SEA-SEC-013"],"patternHitAt5":true},{"scenarioId":"KI-AI-002","incidentId":"SEA-AI-002","rank":6,"top5IncidentIds":["SEA-SEC-028","SEA-AI-001","SEA-AI-007","SEA-AI-008","SEA-PUB-041"],"patternHitAt5":true},{"scenarioId":"KI-AI-003","incidentId":"SEA-AI-003","rank":1,"top5IncidentIds":["SEA-AI-003","SEA-AI-004","SEA-PUB-040","SEA-SEC-028","SEA-AI-007"],"patternHitAt5":true},{"scenarioId":"KI-AI-004","incidentId":"SEA-AI-004","rank":1,"top5IncidentIds":["SEA-AI-004","SEA-AI-001","SEA-PUB-041","SEA-AI-003","SEA-POST-007"],"patternHitAt5":true},{"scenarioId":"KI-AI-005","incidentId":"SEA-AI-005","rank":null,"top5IncidentIds":["SEA-AI-004","SEA-SEC-028","SEA-PUB-041","SEA-AI-009"],"patternHitAt5":true},{"scenarioId":"KI-AI-006","incidentId":"SEA-AI-006","rank":1,"top5IncidentIds":["SEA-AI-006","SEA-AI-009"],"patternHitAt5":true},{"scenarioId":"KI-POST-024","incidentId":"SEA-POST-024","rank":1,"top5IncidentIds":["SEA-POST-024","SEA-PUB-053","SEA-POST-017","SEA-POST-023","SEA-POST-019"],"patternHitAt5":true},{"scenarioId":"KI-POST-025","incidentId":"SEA-POST-025","rank":1,"top5IncidentIds":["SEA-POST-025","SEA-SEC-005"],"patternHitAt5":true},{"scenarioId":"KI-POST-026","incidentId":"SEA-POST-026","rank":1,"top5IncidentIds":["SEA-POST-026"],"patternHitAt5":true},{"scenarioId":"KI-SEC-027","incidentId":"SEA-SEC-027","rank":2,"top5IncidentIds":["SEA-SEC-001","SEA-SEC-027","SEA-AI-009","SEA-AI-004","SEA-SEC-021"],"patternHitAt5":true},{"scenarioId":"KI-SEC-028","incidentId":"SEA-SEC-028","rank":1,"top5IncidentIds":["SEA-SEC-028","SEA-SEC-010","SEA-SEC-017","SEA-POST-011","SEA-SEC-029"],"patternHitAt5":true},{"scenarioId":"KI-SEC-029","incidentId":"SEA-SEC-029","rank":1,"top5IncidentIds":["SEA-SEC-029","SEA-SEC-028","SEA-SEC-027","SEA-SEC-017","SEA-AI-003"],"patternHitAt5":true},{"scenarioId":"KI-SEC-030","incidentId":"SEA-SEC-030","rank":null,"top5IncidentIds":[],"patternHitAt5":false},{"scenarioId":"KI-AI-007","incidentId":"SEA-AI-007","rank":1,"top5IncidentIds":["SEA-AI-007","SEA-PUB-041","SEA-AI-001","SEA-SEC-028","SEA-AI-008"],"patternHitAt5":true},{"scenarioId":"KI-AI-008","incidentId":"SEA-AI-008","rank":5,"top5IncidentIds":["SEA-AI-007","SEA-PUB-041","SEA-SEC-028","SEA-AI-001","SEA-AI-008"],"patternHitAt5":true},{"scenarioId":"KI-AI-009","incidentId":"SEA-AI-009","rank":1,"top5IncidentIds":["SEA-AI-009","SEA-AI-004","SEA-SEC-020"],"patternHitAt5":true},{"scenarioId":"KI-AI-010","incidentId":"SEA-AI-010","rank":1,"top5IncidentIds":["SEA-AI-010"],"patternHitAt5":true}]}}}