{"methodology":"v0.4-mini","dataStatus":"observed","dataSource":"Observed OpenRouter completions","run":{"dataStatus":"observed","dataSource":"Observed OpenRouter completions","benchmarkId":"midmarketbench-mini-v0.4","methodologyVersion":"v0.4-mini","evaluatedAt":"2026-07-18","runId":"2026-07-18-final","mode":"Closed-book","caseId":"norwyn-controls-2026-07","caseName":"Norwyn Controls","caseConfidentiality":"Synthetic holdout created for this run","tasks":4,"samplesPerModel":2,"modelsRequested":9,"modelsCompleted":8,"modelsPartial":0,"modelsIncomplete":0,"modelsUnavailable":1,"totalSpendUsd":5.31559,"conservativelyAccountedSpendUsd":7.373867,"providerReportedSpendUsd":5.020023,"spendBasis":"Total spend is the final settled usage for the dedicated OpenRouter key. Conservative accounting reserves request maxima for ambiguous dispatches; provider-reported spend includes generations with usage records across all benchmark attempts.","localBudgetUsd":7,"promptSha256":"8003e8ff4f2ec5f89ab6ac54706f9d3c45deebbf1c893d22cfa3e2e201b95dbe","judgeSystemPromptSha256":"dc6102c543f47a5f066336a805fa1ed03e0304fda23d0ad6ab96b83b530040c6","candidateSchemaVariants":[{"id":"provider-compatible","responseFormatSha256":"37a752faef01553f5eba6843de01a57147c6c3b319bbdde74000ce93f54735ab","acceptedSamples":6},{"id":"bounded-json-schema","responseFormatSha256":"98c98d3668879ec785146329bf1bfd19fa49dc2c2fe918b8efe1018a04ff37b8","acceptedSamples":10}],"caseSha256":"511cc341e41673eb24b1fdd532fd932f934bd65810703dbc6cddb15a83413f1a","modelSnapshotSha256":"b70c451cd42def5960627679f69d6a21d676a5a9f0f1e95483d0fd6bf4f21bd1","limitation":"Directional mini benchmark: one fresh synthetic case and two samples per model. It is not a universal ranking of model intelligence.","scoring":"Reference-scored metrics, issue coverage, ranking and constrained action selection; blinded cross-family judges score only the IC note, with grades normalised between weak and strong anchors.","reasoningPolicy":"A 2,048-token reasoning budget for candidates and 1,024 for judges; Kimi K3 uses its mandatory max level.","tieRule":"Differences below two points are marked as provisional ties."},"rows":[{"rank":1,"model":{"id":"moonshotai/kimi-k3","slug":"kimi-k3","name":"Kimi K3","providerId":"moonshot","version":"kimi-k3","release":null,"contextWindow":"1,048,576 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/moonshotai/kimi-k3"},"provider":{"id":"moonshot","name":"Moonshot AI","slug":"moonshot","website":"https://moonshot.ai"},"scores":{"grounding":85.41,"commercial_judgement":98.66,"scepticism":97.68,"numerical_sanity":100,"risk_discovery":96.56,"question_generation":96.06,"european_context":100,"output_usefulness":99.73},"overall":96.3,"range":[95.64,96.91],"sampleScores":[95.64,96.91],"samplesCompleted":2,"samplesRequested":2,"providerReportedCandidateCostUsd":0.612526,"providerReportedJudgeCostUsd":0.281641,"medianLatencyMs":386867,"provisionalTie":true,"endpoint":{"requestedId":"moonshotai/kimi-k3","returnedModels":["moonshotai/kimi-k3"],"providers":["Moonshot AI"],"endpointTags":["moonshotai/int4"],"quantizations":["int4"],"contextLength":1048576,"pricing":{"basis":"selected_endpoint_snapshot","endpoints":[{"tag":"moonshotai/int4","provider":"Moonshot AI","promptUsdPerMillion":3,"completionUsdPerMillion":15}]}},"caveat":"Kimi K3 via Moonshot AI, INT4, OpenRouter. Mandatory max reasoning; full weights were not yet released on 18 July 2026.","mode":"Closed-book"},{"rank":2,"model":{"id":"x-ai/grok-4.5","slug":"grok-4-5","name":"Grok 4.5","providerId":"xai","version":"grok-4.5","release":null,"contextWindow":"500,000 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/x-ai/grok-4.5"},"provider":{"id":"xai","name":"xAI","slug":"xai","website":"https://x.ai"},"scores":{"grounding":89.39,"commercial_judgement":99.12,"scepticism":100,"numerical_sanity":100,"risk_discovery":99.6,"question_generation":96.06,"european_context":70,"output_usefulness":99.73},"overall":96.3,"range":[95.54,96.99],"sampleScores":[95.54,96.99],"samplesCompleted":2,"samplesRequested":2,"providerReportedCandidateCostUsd":0.088013,"providerReportedJudgeCostUsd":0.239988,"medianLatencyMs":90522,"provisionalTie":true,"endpoint":{"requestedId":"x-ai/grok-4.5","returnedModels":["x-ai/grok-4.5"],"providers":["xAI"],"endpointTags":["xai"],"quantizations":["unknown"],"contextLength":500000,"pricing":{"basis":"selected_endpoint_snapshot","endpoints":[{"tag":"xai","provider":"xAI","promptUsdPerMillion":2,"completionUsdPerMillion":6}]}},"caveat":null,"mode":"Closed-book"},{"rank":3,"model":{"id":"openai/gpt-5.6-sol","slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","providerId":"openai","version":"gpt-5.6-sol","release":null,"contextWindow":"1,050,000 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/openai/gpt-5.6-sol"},"provider":{"id":"openai","name":"OpenAI","slug":"openai","website":"https://openai.com"},"scores":{"grounding":87.31,"commercial_judgement":96.87,"scepticism":96.52,"numerical_sanity":100,"risk_discovery":94.16,"question_generation":94.51,"european_context":100,"output_usefulness":99.56},"overall":95.5,"range":[95.11,95.89],"sampleScores":[95.11,95.89],"samplesCompleted":2,"samplesRequested":2,"providerReportedCandidateCostUsd":0.309278,"providerReportedJudgeCostUsd":0.249618,"medianLatencyMs":100716,"provisionalTie":true,"endpoint":{"requestedId":"openai/gpt-5.6-sol","returnedModels":["openai/gpt-5.6-sol"],"providers":["OpenAI"],"endpointTags":["openai"],"quantizations":["unknown"],"contextLength":1050000,"pricing":{"basis":"selected_endpoint_snapshot","endpoints":[{"tag":"openai","provider":"OpenAI","promptUsdPerMillion":5,"completionUsdPerMillion":30}]}},"caveat":null,"mode":"Closed-book"},{"rank":4,"model":{"id":"google/gemini-3.5-flash","slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","providerId":"google","version":"gemini-3.5-flash","release":null,"contextWindow":"1,048,576 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/google/gemini-3.5-flash"},"provider":{"id":"google","name":"Google","slug":"google","website":"https://deepmind.google"},"scores":{"grounding":86.67,"commercial_judgement":81.13,"scepticism":88.8,"numerical_sanity":100,"risk_discovery":91.69,"question_generation":92.11,"european_context":96.66,"output_usefulness":74.55},"overall":89.1,"range":[86.88,91.26],"sampleScores":[86.88,91.26],"samplesCompleted":2,"samplesRequested":2,"providerReportedCandidateCostUsd":0.098574,"providerReportedJudgeCostUsd":0.626701,"medianLatencyMs":120901,"provisionalTie":true,"endpoint":{"requestedId":"google/gemini-3.5-flash","returnedModels":["google/gemini-3.5-flash"],"providers":["Google"],"endpointTags":["google-vertex/global"],"quantizations":["unknown"],"contextLength":1048576,"pricing":{"basis":"selected_endpoint_snapshot","endpoints":[{"tag":"google-vertex/global","provider":"Google","promptUsdPerMillion":1.5,"completionUsdPerMillion":9}]}},"caveat":null,"mode":"Closed-book"},{"rank":5,"model":{"id":"anthropic/claude-opus-4.8","slug":"claude-opus-4-8","name":"Claude Opus 4.8","providerId":"anthropic","version":"claude-opus-4.8","release":null,"contextWindow":"1,000,000 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/anthropic/claude-opus-4.8"},"provider":{"id":"anthropic","name":"Anthropic","slug":"anthropic","website":"https://anthropic.com"},"scores":{"grounding":91.25,"commercial_judgement":77.7,"scepticism":96.52,"numerical_sanity":100,"risk_discovery":94.16,"question_generation":66.47,"european_context":100,"output_usefulness":85.68},"overall":88.8,"range":[80.15,97.37],"sampleScores":[97.37,80.15],"samplesCompleted":2,"samplesRequested":2,"providerReportedCandidateCostUsd":0.51831,"providerReportedJudgeCostUsd":0.130233,"medianLatencyMs":79072,"provisionalTie":true,"endpoint":{"requestedId":"anthropic/claude-opus-4.8","returnedModels":["anthropic/claude-opus-4.8"],"providers":["Azure"],"endpointTags":["azure/us-east-2"],"quantizations":["unknown"],"contextLength":1000000,"pricing":{"basis":"selected_endpoint_snapshot","endpoints":[{"tag":"azure/us-east-2","provider":"Azure","promptUsdPerMillion":5,"completionUsdPerMillion":25}]}},"caveat":null,"mode":"Closed-book"},{"rank":6,"model":{"id":"deepseek/deepseek-v4-pro","slug":"deepseek-v4-pro","name":"DeepSeek V4 Pro","providerId":"deepseek","version":"deepseek-v4-pro","release":null,"contextWindow":"1,048,576 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/deepseek/deepseek-v4-pro"},"provider":{"id":"deepseek","name":"DeepSeek","slug":"deepseek","website":"https://deepseek.com"},"scores":{"grounding":81.97,"commercial_judgement":90.69,"scepticism":87.23,"numerical_sanity":100,"risk_discovery":81.06,"question_generation":91.25,"european_context":85,"output_usefulness":91.5},"overall":88.6,"range":[87.87,89.39],"sampleScores":[87.87,89.39],"samplesCompleted":2,"samplesRequested":2,"providerReportedCandidateCostUsd":0.03855,"providerReportedJudgeCostUsd":0.322825,"medianLatencyMs":188462,"provisionalTie":true,"endpoint":{"requestedId":"deepseek/deepseek-v4-pro","returnedModels":["deepseek/deepseek-v4-pro"],"providers":["StreamLake"],"endpointTags":["streamlake/fp8"],"quantizations":["fp8"],"contextLength":1048576,"pricing":{"basis":"selected_endpoint_snapshot","endpoints":[{"tag":"streamlake/fp8","provider":"StreamLake","promptUsdPerMillion":0.7134,"completionUsdPerMillion":1.4268}]}},"caveat":null,"mode":"Closed-book"},{"rank":7,"model":{"id":"z-ai/glm-5.2","slug":"glm-5-2","name":"GLM 5.2","providerId":"zai","version":"glm-5.2","release":null,"contextWindow":"1,048,576 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/z-ai/glm-5.2"},"provider":{"id":"zai","name":"Z.ai","slug":"zai","website":"https://z.ai"},"scores":{"grounding":85.91,"commercial_judgement":95.85,"scepticism":88.39,"numerical_sanity":90.68,"risk_discovery":83.19,"question_generation":87.3,"european_context":70,"output_usefulness":90.44},"overall":88.2,"range":[86.62,89.68],"sampleScores":[89.68,86.62],"samplesCompleted":2,"samplesRequested":2,"providerReportedCandidateCostUsd":0.017954,"providerReportedJudgeCostUsd":0.242846,"medianLatencyMs":139680,"provisionalTie":true,"endpoint":{"requestedId":"z-ai/glm-5.2","returnedModels":["z-ai/glm-5.2"],"providers":["StreamLake"],"endpointTags":["streamlake/fp8"],"quantizations":["fp8"],"contextLength":1048576,"pricing":{"basis":"selected_endpoint_snapshot","endpoints":[{"tag":"streamlake/fp8","provider":"StreamLake","promptUsdPerMillion":0.2737,"completionUsdPerMillion":0.8602000000000001}]}},"caveat":null,"mode":"Closed-book"},{"rank":8,"model":{"id":"qwen/qwen3.7-max","slug":"qwen3-7-max","name":"Qwen3.7 Max","providerId":"alibaba","version":"qwen3.7-max","release":null,"contextWindow":"1,000,000 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/qwen/qwen3.7-max"},"provider":{"id":"alibaba","name":"Alibaba","slug":"alibaba","website":"https://qwen.ai"},"scores":{"grounding":66.48,"commercial_judgement":78.38,"scepticism":60.8,"numerical_sanity":57.5,"risk_discovery":45.88,"question_generation":80.44,"european_context":70,"output_usefulness":73.52},"overall":65.5,"range":[44.58,86.42],"sampleScores":[44.58,86.42],"samplesCompleted":2,"samplesRequested":2,"providerReportedCandidateCostUsd":0.089527,"providerReportedJudgeCostUsd":0.321651,"medianLatencyMs":78622,"provisionalTie":false,"endpoint":{"requestedId":"qwen/qwen3.7-max","returnedModels":["qwen/qwen3.7-max"],"providers":["Alibaba"],"endpointTags":["alibaba"],"quantizations":["unknown"],"contextLength":1000000,"pricing":{"basis":"selected_endpoint_snapshot","endpoints":[{"tag":"alibaba","provider":"Alibaba","promptUsdPerMillion":1.475,"completionUsdPerMillion":4.425}]}},"caveat":null,"mode":"Closed-book"}],"incomplete":[{"model":{"id":"anthropic/claude-fable-5","slug":"claude-fable-5","name":"Claude Fable 5","providerId":"anthropic","version":"claude-fable-5","release":null,"contextWindow":"1,000,000 tokens","openWeights":null,"sourceUrl":"https://openrouter.ai/anthropic/claude-fable-5"},"provider":{"id":"anthropic","name":"Anthropic","slug":"anthropic","website":"https://anthropic.com"},"samplesCompleted":0,"candidateSamplesCompleted":0,"samplesRequested":2,"incompleteSamples":[{"sample":1,"stage":"candidate","reason":"Claude Fable 5 is not available. Learn more: https://www.anthropic.com/news/fable-mythos-access"},{"sample":2,"stage":"candidate","reason":"Claude Fable 5 is not available. Learn more: https://www.anthropic.com/news/fable-mythos-access"}],"endpoint":{"requestedId":"anthropic/claude-fable-5","returnedModels":[],"providers":[],"endpointTags":[],"quantizations":[],"contextLength":1000000,"pricing":null},"caveat":null,"status":"unavailable","rank":null,"provisionalTie":false}]}