{
  "name": "SlipVerdict Settlement Reasoning Benchmark",
  "version": "1.0.0",
  "published": "2026-08-29",
  "license": "CC BY 4.0",
  "description": "Twenty-four synthetic cases for testing arithmetic, settlement-label comparison, and rule-dependent escalation in post-settlement over/under reviews.",
  "task": "Return arithmeticResult, verdict, mustMention, and a short rationale for each case.",
  "labels": {
    "arithmeticResult": ["win", "loss", "push"],
    "verdict": ["consistent", "possible-mismatch", "review-required"]
  },
  "sources": [
    "https://www.nist.gov/ai-measurement-and-evaluation",
    "https://airc.nist.gov/airmf-resources/playbook/measure/"
  ],
  "limitations": [
    "All names, events, and settlements are synthetic.",
    "Cases test a deliberately narrow reasoning task and do not reproduce any sportsbook's full rules.",
    "A benchmark score is not evidence that a system can determine legal entitlement or resolve a real dispute."
  ],
  "cases": [
    {"id":"SVRB-001","category":"arithmetic","prompt":"A player is over 18.5 points, records 22, and the settlement is win.","inputs":{"direction":"over","line":18.5,"finalStat":22,"reportedSettlement":"win","ruleSignal":"none"},"expected":{"arithmeticResult":"win","verdict":"consistent","mustMention":"none"},"rationale":"22 is greater than 18.5 and the settlement matches."},
    {"id":"SVRB-002","category":"arithmetic","prompt":"A player is under 7.5 assists, records 6, and the settlement is win.","inputs":{"direction":"under","line":7.5,"finalStat":6,"reportedSettlement":"win","ruleSignal":"none"},"expected":{"arithmeticResult":"win","verdict":"consistent","mustMention":"none"},"rationale":"6 is less than 7.5 and the settlement matches."},
    {"id":"SVRB-003","category":"arithmetic","prompt":"A player is over 24.5 points, records 21, and the settlement is loss.","inputs":{"direction":"over","line":24.5,"finalStat":21,"reportedSettlement":"loss","ruleSignal":"none"},"expected":{"arithmeticResult":"loss","verdict":"consistent","mustMention":"none"},"rationale":"21 is less than 24.5 and the settlement matches."},
    {"id":"SVRB-004","category":"arithmetic","prompt":"A player is under 2.5 touchdowns, records 3, and the settlement is loss.","inputs":{"direction":"under","line":2.5,"finalStat":3,"reportedSettlement":"loss","ruleSignal":"none"},"expected":{"arithmeticResult":"loss","verdict":"consistent","mustMention":"none"},"rationale":"3 is greater than 2.5 and the settlement matches."},
    {"id":"SVRB-005","category":"boundary","prompt":"A player is over 20 points, records exactly 20, and the settlement is push.","inputs":{"direction":"over","line":20,"finalStat":20,"reportedSettlement":"push","ruleSignal":"none"},"expected":{"arithmeticResult":"push","verdict":"consistent","mustMention":"equality"},"rationale":"Equality to a whole-number line is a push in this benchmark."},
    {"id":"SVRB-006","category":"boundary","prompt":"A player is under 9 rebounds, records exactly 9, and the settlement is loss.","inputs":{"direction":"under","line":9,"finalStat":9,"reportedSettlement":"loss","ruleSignal":"none"},"expected":{"arithmeticResult":"push","verdict":"possible-mismatch","mustMention":"equality"},"rationale":"Equality produces a push, so the reported loss differs."},
    {"id":"SVRB-007","category":"decimal","prompt":"A player is over 0.5 goals, records 1, and the settlement is win.","inputs":{"direction":"over","line":0.5,"finalStat":1,"reportedSettlement":"win","ruleSignal":"none"},"expected":{"arithmeticResult":"win","verdict":"consistent","mustMention":"none"},"rationale":"1 is greater than 0.5."},
    {"id":"SVRB-008","category":"decimal","prompt":"A player is under 12.5 completions, records 13, and the settlement is loss.","inputs":{"direction":"under","line":12.5,"finalStat":13,"reportedSettlement":"loss","ruleSignal":"none"},"expected":{"arithmeticResult":"loss","verdict":"consistent","mustMention":"none"},"rationale":"13 is greater than the under line."},
    {"id":"SVRB-009","category":"mismatch","prompt":"A player is over 24.5 points, records 27, and the settlement is loss.","inputs":{"direction":"over","line":24.5,"finalStat":27,"reportedSettlement":"loss","ruleSignal":"none"},"expected":{"arithmeticResult":"win","verdict":"possible-mismatch","mustMention":"reported settlement differs"},"rationale":"27 clears the over line, but the reported settlement is loss."},
    {"id":"SVRB-010","category":"mismatch","prompt":"A player is under 5.5 receptions, records 4, and the settlement is loss.","inputs":{"direction":"under","line":5.5,"finalStat":4,"reportedSettlement":"loss","ruleSignal":"none"},"expected":{"arithmeticResult":"win","verdict":"possible-mismatch","mustMention":"reported settlement differs"},"rationale":"4 is below 5.5, so the arithmetic result is win."},
    {"id":"SVRB-011","category":"mismatch","prompt":"A player is over 3.5 made threes, records 2, and the settlement is win.","inputs":{"direction":"over","line":3.5,"finalStat":2,"reportedSettlement":"win","ruleSignal":"none"},"expected":{"arithmeticResult":"loss","verdict":"possible-mismatch","mustMention":"reported settlement differs"},"rationale":"2 does not clear the over line."},
    {"id":"SVRB-012","category":"mismatch","prompt":"A player is under 85.5 rushing yards, records 101, and the settlement is win.","inputs":{"direction":"under","line":85.5,"finalStat":101,"reportedSettlement":"win","ruleSignal":"none"},"expected":{"arithmeticResult":"loss","verdict":"possible-mismatch","mustMention":"reported settlement differs"},"rationale":"101 is above the under line."},
    {"id":"SVRB-013","category":"mismatch","prompt":"A player is over 10 tackles, records exactly 10, and the settlement is win.","inputs":{"direction":"over","line":10,"finalStat":10,"reportedSettlement":"win","ruleSignal":"none"},"expected":{"arithmeticResult":"push","verdict":"possible-mismatch","mustMention":"equality"},"rationale":"Equality is a push in the benchmark, not a win."},
    {"id":"SVRB-014","category":"mismatch","prompt":"A player is under 30 pass attempts, records exactly 30, and the settlement is loss.","inputs":{"direction":"under","line":30,"finalStat":30,"reportedSettlement":"loss","ruleSignal":"none"},"expected":{"arithmeticResult":"push","verdict":"possible-mismatch","mustMention":"equality"},"rationale":"Equality is a push in the benchmark, not a loss."},
    {"id":"SVRB-015","category":"source-quality","prompt":"A player is over 6.5 assists, a user-entered value says 8, and the settlement is loss; no source is supplied.","inputs":{"direction":"over","line":6.5,"finalStat":8,"reportedSettlement":"loss","ruleSignal":"unverified-source"},"expected":{"arithmeticResult":"win","verdict":"review-required","mustMention":"source verification"},"rationale":"The arithmetic differs, but the unsourced statistic must be verified before escalation."},
    {"id":"SVRB-016","category":"source-quality","prompt":"Two public sources disagree between 7 and 8 rebounds for an over 7.5 line; the settlement is loss.","inputs":{"direction":"over","line":7.5,"finalStat":8,"reportedSettlement":"loss","ruleSignal":"source-conflict"},"expected":{"arithmeticResult":"win","verdict":"review-required","mustMention":"official source selection"},"rationale":"The selected value clears the line, but the source conflict controls the next step."},
    {"id":"SVRB-017","category":"rule-review","prompt":"A player is over 14.5 points, records 18, the settlement is void, and participation is disputed.","inputs":{"direction":"over","line":14.5,"finalStat":18,"reportedSettlement":"void","ruleSignal":"participation"},"expected":{"arithmeticResult":"win","verdict":"review-required","mustMention":"participation rule"},"rationale":"Arithmetic alone cannot resolve a participation-based void."},
    {"id":"SVRB-018","category":"rule-review","prompt":"A player is under 4.5 shots, records 3, the settlement is void, and the event was postponed.","inputs":{"direction":"under","line":4.5,"finalStat":3,"reportedSettlement":"void","ruleSignal":"postponement"},"expected":{"arithmeticResult":"win","verdict":"review-required","mustMention":"postponement rule"},"rationale":"Postponement terms require rule review despite the numeric result."},
    {"id":"SVRB-019","category":"rule-review","prompt":"A player is over 9.5 rebounds, initially records 10, settlement is win, and a later official correction changes the value to 9.","inputs":{"direction":"over","line":9.5,"finalStat":9,"reportedSettlement":"win","ruleSignal":"stat-correction"},"expected":{"arithmeticResult":"loss","verdict":"review-required","mustMention":"stat-correction window"},"rationale":"The corrected value changes the arithmetic and the book's correction window must be checked."},
    {"id":"SVRB-020","category":"rule-review","prompt":"A player is over 1.5 goals, records 2, settlement is reduced, and the market has a dead-heat signal.","inputs":{"direction":"over","line":1.5,"finalStat":2,"reportedSettlement":"win","ruleSignal":"dead-heat"},"expected":{"arithmeticResult":"win","verdict":"review-required","mustMention":"dead-heat terms"},"rationale":"The numeric result is win, but payout treatment is rule-dependent."},
    {"id":"SVRB-021","category":"rule-review","prompt":"A player is under 32.5 points, one source shows 31 and the designated official source shows 33; settlement is loss.","inputs":{"direction":"under","line":32.5,"finalStat":33,"reportedSettlement":"loss","ruleSignal":"designated-source"},"expected":{"arithmeticResult":"loss","verdict":"review-required","mustMention":"designated official source"},"rationale":"The arithmetic matches the designated-source value, but the source choice must be documented."},
    {"id":"SVRB-022","category":"rule-review","prompt":"A player is over 2.5 field goals, records 4, settlement is void, and the ticket is flagged for a palpable error.","inputs":{"direction":"over","line":2.5,"finalStat":4,"reportedSettlement":"void","ruleSignal":"palpable-error"},"expected":{"arithmeticResult":"win","verdict":"review-required","mustMention":"palpable-error provision"},"rationale":"A palpable-error claim cannot be adjudicated by comparison math."},
    {"id":"SVRB-023","category":"rule-review","prompt":"A quarterback is over 1.5 passing touchdowns, records 2 total touchdowns but only 1 passing touchdown; settlement is loss.","inputs":{"direction":"over","line":1.5,"finalStat":1,"reportedSettlement":"loss","ruleSignal":"market-definition"},"expected":{"arithmeticResult":"loss","verdict":"review-required","mustMention":"market-specific definition"},"rationale":"The relevant market statistic is passing touchdowns, and exact ticket wording should be confirmed."},
    {"id":"SVRB-024","category":"rule-review","prompt":"A player is under 40.5 yards, records 12 before an abandoned event, and the settlement is void.","inputs":{"direction":"under","line":40.5,"finalStat":12,"reportedSettlement":"void","ruleSignal":"abandoned-event"},"expected":{"arithmeticResult":"win","verdict":"review-required","mustMention":"abandoned-event rule"},"rationale":"The partial statistic is not enough to resolve an abandoned-event settlement."}
  ]
}