{
  "schema": "asn.public-study-evidence.v1",
  "study_id": "qwen-cuad-five-contract-microholdout-2026-10-20",
  "title": "Qwen 2.5 1.5B · five-contract CUAD micro-holdout",
  "study_type": "bounded_local_model_pilot",
  "evaluated_at": "2026-10-20T20:54:35.014Z",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "license": "CC BY 4.0",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073"
  },
  "protocol": {
    "suite_id": "custom-qwen-cuad-disjoint-five-20261020-v1",
    "suite_sha256": "528f5ffdf8ace348e5305ac9f46c3047f6bfa22c655c5079ccdcda01c2a4ec2b",
    "task_count": 5,
    "unique_contract_examples": 5,
    "variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 1,
    "grading": "Exact match against independently checked positive annotated CUAD answer spans in a closed JSON output.",
    "runtime": "Ollama 0.33.3 on loopback; no external calls, provider spend, production writes, or routing changes.",
    "raw_prompts_and_model_outputs_published": false
  },
  "model": {
    "name": "qwen2.5:1.5b",
    "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
    "runs": 10,
    "verified_passes": 1,
    "verified_failures": 9,
    "execution_errors": 0,
    "exact_match_rate": 0.1
  },
  "tasks": [
    {
      "task_id": "cuad-agreement-date-1341ae864f76.h1",
      "category": "agreement-date",
      "attempts": 2,
      "exact_matches": 1,
      "exact_failures": 1,
      "execution_errors": 0,
      "prompt_pair_verdict_flip": true
    },
    {
      "task_id": "cuad-effective-date-1881adc7c4e2.h1",
      "category": "effective-date",
      "attempts": 2,
      "exact_matches": 0,
      "exact_failures": 2,
      "execution_errors": 0,
      "prompt_pair_verdict_flip": false
    },
    {
      "task_id": "cuad-expiration-date-13876ff9847e.h1",
      "category": "expiration-date",
      "attempts": 2,
      "exact_matches": 0,
      "exact_failures": 2,
      "execution_errors": 0,
      "prompt_pair_verdict_flip": false
    },
    {
      "task_id": "cuad-renewal-term-419d96219ac0.h1",
      "category": "renewal-term",
      "attempts": 2,
      "exact_matches": 0,
      "exact_failures": 2,
      "execution_errors": 0,
      "prompt_pair_verdict_flip": false
    },
    {
      "task_id": "cuad-renewal-term-4fc02f84a202.h1",
      "category": "renewal-term",
      "attempts": 2,
      "exact_matches": 0,
      "exact_failures": 2,
      "execution_errors": 0,
      "prompt_pair_verdict_flip": false
    }
  ],
  "interpretation": {
    "ranking_eligible": false,
    "production_index_eligible": false,
    "customer_utility_verified": false,
    "general_capability_verified": false,
    "limitations": [
      "Only five positive-span examples and one attempt per prompt variant; this is a micro-holdout, not a benchmark estimate.",
      "Training exposure to CUAD is unknown.",
      "Exact annotated-span reproduction does not establish legal correctness, review time saved, or customer value.",
      "One of five tasks changed pass/fail verdict across the two prompt variants; the paired prompts are not independent examples."
    ]
  },
  "provenance": {
    "artifact_sha256": "3c10d3382b4c313252eab33963a6f639b51044a4504062f0fcb2431447979500",
    "public_receipt_integrity": "The original local artifact passed the independent holdout verifier before this allow-listed projection was generated.",
    "external_provider_calls": false,
    "production_measurement_written": false,
    "money_movement": false
  },
  "receipt_sha256": "5987bf98b6b4f5db884880eaa078b3fcd5b6681f69db2aa70269122aea1024f6"
}