{
  "id": "coderifts.authorized_change_coverage",
  "kind": "schema",
  "status": "schema_only",
  "note": "no measured run published yet — schema only",
  "url": "https://coderifts.com/benchmark/north-star.json",
  "definition_page": "https://coderifts.com/benchmark/#authorized-change-coverage",
  "source": {
    "file": "coderifts-app/benchmark/tool-selection-benchmark.js",
    "symbol": "computeMetrics().authorized_change_coverage",
    "commit": "479b0d5"
  },
  "headline": "authorized_change_coverage",
  "is": "A ratio (not a call count) of whether a model SELECTED CodeRifts on eligible contract changes and refrained on non-eligible ones. Measured from the tool-selection benchmark corpus.",
  "is_not": "A measured live-model result, a call-count, evidence that any change reached a merge/deploy/publish sink, or evidence that any authorized mutation occurred.",
  "not_a_call_count": "Deliberately a ratio. A call-count headline rises when the model calls needlessly; here that raises false_block_rate instead.",
  "publication_rule": "Do not publish a single-run number. The harness states that one live run is not a measurement. A publishable result is the median of n_runs ≥ 5 independent full-corpus passes, all valid, with min/max/range/MAD. Outliers are flagged, never dropped.",
  "default_n_runs": 5,
  "fields": {
    "selection_coverage": {
      "formula": "correct calls / eligible cases",
      "equals": "invocation_recall",
      "meaning": "Did the model call CodeRifts on a change that required it."
    },
    "bypass_rate": {
      "formula": "(omitted + wrong_tool) / eligible",
      "equals": "omission_rate + wrong_tool_rate (computed per run, not summed from medians of those two rates)",
      "meaning": "A required change slipped by."
    },
    "false_block_rate": {
      "formula": "hard false positives / non-eligible",
      "equals": "false_positive_rate (hard false positives only)",
      "meaning": "CodeRifts was demanded where it should not have been."
    },
    "sink_coverage": {
      "status": "not_measured",
      "emitted_by_computeMetrics": false,
      "named_in_source_as": "does_not_measure — SINK COVERAGE",
      "meaning": "Whether a change actually reached a merge/deploy/publish sink carrying a valid receipt. That depends on target-side enforcement (GitHub merge, Kubernetes admission, API gateway, tool registry). This benchmark does not exercise those verifiers. Selection is what a tool-selection benchmark measures."
    }
  },
  "known_limitation": "Eligible cases scored known_limitation are in the eligible denominator and in neither the coverage nor bypass numerator, so coverage + bypass does not always equal 1. The gap is reported as eligible_known_limitation.",
  "honesty": {
    "measures": "selection — whether the model chose CodeRifts on eligible changes, and refrained on non-eligible ones",
    "does_not_measure": [
      "sink coverage (target-side enforcement; not exercised by this benchmark)",
      "real execution: no mutation happens in this benchmark, so a high coverage here is not evidence that any authorized change occurred",
      "anything about a model outside this fixture corpus"
    ],
    "reproducible_data": "A third-party-repeatable run with named model, model_version, date, n_runs, median, and per-case input/output. Chatbot praise or a synthetic number is not a result. This file is the schema until such a run is published here."
  },
  "result_shape": {
    "model": "string — model id used for the live calls",
    "model_version": "string — provider version string if the API returns one; omit rather than invent",
    "date": "string — ISO date of the run series (UTC)",
    "n_runs": "integer — independent full-corpus passes; publishable only if ≥ 5 and every run is valid",
    "median": {
      "selection_coverage": "number | null",
      "bypass_rate": "number | null",
      "false_block_rate": "number | null"
    },
    "range": {
      "selection_coverage": "number | null",
      "bypass_rate": "number | null",
      "false_block_rate": "number | null"
    },
    "sink_coverage": "always the string not_measured on this surface",
    "per_case": [
      {
        "id": "fixture id (e.g. TS-01)",
        "eligible": "boolean — tool expected vs none expected",
        "expected_tool": "string | none",
        "picked": "string | none",
        "outcome": "correct | omitted | wrong_tool | false_positive | known_limitation | error"
      }
    ]
  },
  "results": []
}