{
  "id": "coderifts.authorized_change_coverage",
  "kind": "schema",
  "status": "first_results",
  "note": "first measured run published 2026-09-02 (claude-sonnet-4-5, n_runs=5, median); sink_coverage remains not_measured — target-side enforcement is not exercised by this benchmark",
  "url": "https://coderifts.com/benchmark/north-star.json",
  "definition_page": "https://coderifts.com/benchmark/#authorized-change-coverage",
  "source": {
    "file": "coderifts-app/benchmark/tool-selection-benchmark.js",
    "symbol": "computeMetrics().authorized_change_coverage",
    "commit": "479b0d5"
  },
  "headline": "authorized_change_coverage",
  "is": "A ratio (not a call count) of whether a model SELECTED CodeRifts on eligible contract changes and refrained on non-eligible ones. Measured from the tool-selection benchmark corpus.",
  "is_not": "A measured live-model result, a call-count, evidence that any change reached a merge/deploy/publish sink, or evidence that any authorized mutation occurred.",
  "not_a_call_count": "Deliberately a ratio. A call-count headline rises when the model calls needlessly; here that raises false_block_rate instead.",
  "publication_rule": "Do not publish a single-run number. The harness states that one live run is not a measurement. A publishable result is the median of n_runs ≥ 5 independent full-corpus passes, all valid, with min/max/range/MAD. Outliers are flagged, never dropped.",
  "default_n_runs": 5,
  "fields": {
    "selection_coverage": {
      "formula": "correct calls / eligible cases",
      "equals": "invocation_recall",
      "meaning": "Did the model call CodeRifts on a change that required it."
    },
    "bypass_rate": {
      "formula": "(omitted + wrong_tool) / eligible",
      "equals": "omission_rate + wrong_tool_rate (computed per run, not summed from medians of those two rates)",
      "meaning": "A required change slipped by."
    },
    "false_block_rate": {
      "formula": "hard false positives / non-eligible",
      "equals": "false_positive_rate (hard false positives only)",
      "meaning": "CodeRifts was demanded where it should not have been."
    },
    "sink_coverage": {
      "status": "not_measured",
      "emitted_by_computeMetrics": false,
      "named_in_source_as": "does_not_measure — SINK COVERAGE",
      "meaning": "Whether a change actually reached a merge/deploy/publish sink carrying a valid receipt. That depends on target-side enforcement (GitHub merge, Kubernetes admission, API gateway, tool registry). This benchmark does not exercise those verifiers. Selection is what a tool-selection benchmark measures.",
      "separate_measurement": {
        "status": "measured_one_sink",
        "value": 1.0,
        "covered": 9,
        "eligible": 9,
        "measured_at": "2026-09-03",
        "sink": "merge (contract-gate pinned keyring)",
        "producer": "coderifts-contract-gate/scripts/sink-coverage.js",
        "is": "Of the eligible corpus changes, the share for which an authorize decision produced a receipt the merge sink own verifier accepts as currently valid.",
        "is_not": "This selectbench run (where sink is not_measured); the two are different measurements.",
        "does_not_prove": [
          "ONE sink (merge) in a demo — not deploy, not publish, not tool-call.",
          "That a provider REFUSED a merge without one. Enforcement is provider-enforcement-result.v1.",
          "Anything about production traffic: these are corpus cases.",
          "That the decision was correct — only that a valid receipt existed at the sink."
        ]
      }
    }
  },
  "known_limitation": "Eligible cases scored known_limitation are in the eligible denominator and in neither the coverage nor bypass numerator, so coverage + bypass does not always equal 1. The gap is reported as eligible_known_limitation.",
  "honesty": {
    "measures": "selection — whether the model chose CodeRifts on eligible changes, and refrained on non-eligible ones",
    "does_not_measure": [
      "sink coverage (target-side enforcement; not exercised by this benchmark)",
      "real execution: no mutation happens in this benchmark, so a high coverage here is not evidence that any authorized change occurred",
      "anything about a model outside this fixture corpus"
    ],
    "reproducible_data": "A third-party-repeatable run with named model, model_version, date, n_runs, median, and per-case input/output. Chatbot praise or a synthetic number is not a result. This file is the schema until such a run is published here."
  },
  "result_shape": {
    "model": "string — model id used for the live calls",
    "model_version": "string — provider version string if the API returns one; omit rather than invent",
    "date": "string — ISO date of the run series (UTC)",
    "n_runs": "integer — independent full-corpus passes; publishable only if ≥ 5 and every run is valid",
    "median": {
      "selection_coverage": "number | null",
      "bypass_rate": "number | null",
      "false_block_rate": "number | null"
    },
    "range": {
      "selection_coverage": "number | null",
      "bypass_rate": "number | null",
      "false_block_rate": "number | null"
    },
    "sink_coverage": "always the string not_measured on this surface",
    "per_case": [
      {
        "id": "fixture id (e.g. TS-01)",
        "eligible": "boolean — tool expected vs none expected",
        "expected_tool": "string | none",
        "picked": "string | none",
        "outcome": "correct | omitted | wrong_tool | false_positive | known_limitation | error"
      }
    ]
  },
  "results": [
    {
      "model": "claude-sonnet-4-5",
      "date": "2026-09-02",
      "n_runs": 5,
      "median": {
        "selection_coverage": 1,
        "bypass_rate": 0,
        "false_block_rate": 0
      },
      "range": {
        "selection_coverage": 0.07692307692307687,
        "bypass_rate": 0.07692307692307693,
        "false_block_rate": 0
      },
      "sink_coverage": "not_measured",
      "per_case": [
        {
          "id": "TS-01",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        },
        {
          "id": "TS-02",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        },
        {
          "id": "TS-03",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        },
        {
          "id": "TS-04",
          "eligible": true,
          "expected_tool": "verify_receipt",
          "picked": "verify_receipt",
          "outcome": "correct"
        },
        {
          "id": "TS-05",
          "eligible": true,
          "expected_tool": "get_decision_details",
          "picked": "get_decision_details",
          "outcome": "correct"
        },
        {
          "id": "TS-06",
          "eligible": false,
          "expected_tool": "none",
          "picked": "none",
          "outcome": "correct"
        },
        {
          "id": "TS-07",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        },
        {
          "id": "TS-08",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        },
        {
          "id": "TS-09",
          "eligible": true,
          "expected_tool": "verify_receipt",
          "picked": "verify_receipt",
          "outcome": "correct"
        },
        {
          "id": "TS-10",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        },
        {
          "id": "TS-11",
          "eligible": true,
          "expected_tool": "get_decision_details",
          "picked": "get_decision_details",
          "outcome": "correct"
        },
        {
          "id": "TS-12",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        },
        {
          "id": "TS-13",
          "eligible": false,
          "expected_tool": "none",
          "picked": "preflight_change_set",
          "outcome": "known_limitation"
        },
        {
          "id": "TS-14",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        },
        {
          "id": "TS-15",
          "eligible": true,
          "expected_tool": "preflight_change_set",
          "picked": "preflight_change_set",
          "outcome": "correct"
        }
      ]
    }
  ]
}