{
  "schema": "asn.public-study-evidence.v1",
  "study_id": "cuad-five-contract-qwen-llama-shared-suite-2026-10-20",
  "title": "Qwen 2.5 1.5B vs Llama 3.2 3B · paired five-contract CUAD repeat",
  "study_type": "bounded_local_matched_model_comparison",
  "evaluated_at": "2026-10-20T21:58:24.200Z",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "license": "CC BY 4.0",
    "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073"
  },
  "protocol": {
    "suite_sha256": "528f5ffdf8ace348e5305ac9f46c3047f6bfa22c655c5079ccdcda01c2a4ec2b",
    "unique_contract_examples": 5,
    "positive_annotated_spans_only": true,
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 1,
    "attempts_per_model": 10,
    "grading": "Exact match against independently checked CUAD annotated answer spans in a closed JSON response.",
    "runtime": "Ollama 0.33.3 on 127.0.0.1 loopback only.",
    "provider_calls": 0,
    "provider_spend_microusd": 0,
    "production_measurement_writes": 0,
    "raw_prompts_published": false,
    "raw_model_outputs_published": false
  },
  "models": [
    {
      "name": "Qwen 2.5 1.5B",
      "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "quantization": "Q4_K_M",
      "attempts": 10,
      "valid_outputs": 10,
      "exact_matches": 1,
      "exact_failures": 9,
      "execution_errors": 0,
      "exact_match_rate": 0.1,
      "wilson_95_valid_outputs": {
        "confidence_level": 0.95,
        "method": "wilson_score_interval",
        "lower": 0.0179,
        "upper": 0.4042
      },
      "prompt_verdict_flips": 1
    },
    {
      "name": "Llama 3.2 3B",
      "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "quantization": "Q4_K_M",
      "attempts": 10,
      "valid_outputs": 10,
      "exact_matches": 1,
      "exact_failures": 9,
      "execution_errors": 0,
      "exact_match_rate": 0.1,
      "wilson_95_valid_outputs": {
        "confidence_level": 0.95,
        "method": "wilson_score_interval",
        "lower": 0.0179,
        "upper": 0.4042
      },
      "prompt_verdict_flips": 1
    }
  ],
  "paired_outcomes": {
    "matched_task_prompt_outcomes": 10,
    "both_exact": 1,
    "both_not_exact": 9,
    "discordant_verdicts": 0
  },
  "results_by_field": [
    {
      "field": "agreement date",
      "contract_examples": 1,
      "models": {
        "qwen2.5:1.5b": {
          "exact_matches": 1,
          "attempts": 2
        },
        "llama3.2:3b": {
          "exact_matches": 1,
          "attempts": 2
        }
      },
      "discordant_task_prompt_outcomes": 0
    },
    {
      "field": "effective date",
      "contract_examples": 1,
      "models": {
        "qwen2.5:1.5b": {
          "exact_matches": 0,
          "attempts": 2
        },
        "llama3.2:3b": {
          "exact_matches": 0,
          "attempts": 2
        }
      },
      "discordant_task_prompt_outcomes": 0
    },
    {
      "field": "expiration date",
      "contract_examples": 1,
      "models": {
        "qwen2.5:1.5b": {
          "exact_matches": 0,
          "attempts": 2
        },
        "llama3.2:3b": {
          "exact_matches": 0,
          "attempts": 2
        }
      },
      "discordant_task_prompt_outcomes": 0
    },
    {
      "field": "renewal term",
      "contract_examples": 2,
      "models": {
        "qwen2.5:1.5b": {
          "exact_matches": 0,
          "attempts": 4
        },
        "llama3.2:3b": {
          "exact_matches": 0,
          "attempts": 4
        }
      },
      "discordant_task_prompt_outcomes": 0
    }
  ],
  "interpretation": {
    "public_ranking_eligible": false,
    "production_index_eligible": false,
    "customer_utility_verified": false,
    "general_capability_verified": false,
    "statement": "Both fixed local model builds produced one exact match among ten valid outputs on this five-contract suite. Their pass/fail verdicts agreed on all ten matched task-prompt outcomes; each model changed verdict once between the canonical and paraphrased prompt on one contract. This small result does not establish model equivalence or a general winner."
  },
  "limitations": [
    "Five positive-span contract examples and one attempt per phrasing provide sparse evidence; the 95% intervals are wide and overlapping.",
    "The ten matched task-prompt outcomes are paired and correlated within five contracts; do not treat them as ten independent examples.",
    "Pretraining exposure to CUAD is unknown. Exact annotated-span reproduction is not legal correctness, full-document search, absent-clause detection, review-time savings or customer utility.",
    "Local inference time does not measure hosted service latency, provider availability, realized price or production behavior."
  ],
  "integrity": {
    "suite_id": "custom-cuad-five-shared-comparison-20261020-v1",
    "suite_sha256": "528f5ffdf8ace348e5305ac9f46c3047f6bfa22c655c5079ccdcda01c2a4ec2b",
    "suite_artifact_sha256": "de28312e9fba8bc0d14adf3490796d8cb350fcf16c4c781bba07529a78c5005e",
    "baseline_artifact_sha256": "3c10d3382b4c313252eab33963a6f639b51044a4504062f0fcb2431447979500",
    "candidate_artifact_sha256": "08db2addc6184583cec53c70cd4139a3df7beb1de630bd171a452b7b56eaa0eb",
    "verification_method": "independent_exact_holdout_oracle.v1",
    "verified_attempts": 20,
    "execution_errors": 0,
    "public_receipt_hash_method": "SHA-256 of JSON.stringify of the receipt body with receipt_sha256 omitted."
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "provider_spend_microusd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "money_movement": false,
    "authority_expansion": false
  },
  "receipt_sha256": "dbe9dfe76f319415e05fd50db1f0b81fad8f4aebd8deed6415a026d5f3b33a40"
}