{
  "schema": "asn.public-cuad-qwen-followup.v1",
  "study_id": "cuad-qwen-five-contract-unseen-followup-2026-10-20",
  "title": "Qwen five-contract CUAD follow-up",
  "created_at": "2026-10-20T10:12:32.257Z",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "license": "CC BY 4.0",
    "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073",
    "archive_sha256": "f8161d18bea4e9c05e78fa6dda61c19c846fb8087ea969c172753bc2f45b999a"
  },
  "protocol": {
    "unique_contracts": 5,
    "categories": [
      "agreement date",
      "effective date",
      "expiration date",
      "renewal term",
      "notice period to terminate renewal"
    ],
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 1,
    "grading": "Parsed answer must exactly reproduce the official CUAD annotated span; source offsets were verified before prompting.",
    "runtime": "Ollama 0.33.3 over loopback only.",
    "request_timeout_ms": 60000,
    "total_runtime_cap_ms": 900000,
    "provider_calls": 0,
    "provider_spend_microusd": 0,
    "production_writes": 0
  },
  "model": {
    "name": "qwen2.5:1.5b",
    "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
    "quantization": "Q4_K_M"
  },
  "results": {
    "attempts": 10,
    "valid_outputs": 10,
    "execution_errors": 0,
    "exact_matches": 2,
    "canonical_exact_matches": 2,
    "paraphrase_exact_matches": 0,
    "paired_task_verdict_flips": 2,
    "tasks": [
      {
        "category": "agreement date",
        "canonical_exact_matches": 1,
        "paraphrase_exact_matches": 0,
        "exact_matches": 1,
        "verdict_changed": true
      },
      {
        "category": "effective date",
        "canonical_exact_matches": 1,
        "paraphrase_exact_matches": 0,
        "exact_matches": 1,
        "verdict_changed": true
      },
      {
        "category": "expiration date",
        "canonical_exact_matches": 0,
        "paraphrase_exact_matches": 0,
        "exact_matches": 0,
        "verdict_changed": false
      },
      {
        "category": "renewal term",
        "canonical_exact_matches": 0,
        "paraphrase_exact_matches": 0,
        "exact_matches": 0,
        "verdict_changed": false
      },
      {
        "category": "notice period to terminate renewal",
        "canonical_exact_matches": 0,
        "paraphrase_exact_matches": 0,
        "exact_matches": 0,
        "verdict_changed": false
      }
    ]
  },
  "interpretation": {
    "public_model_ranking_eligible": false,
    "production_measurement_index_eligible": false,
    "general_capability_verified": false,
    "customer_utility_verified": false,
    "statement": "This Qwen-only follow-up adds five new task cases. Two exact matches occurred on canonical prompts and none on paraphrases; two task pairs changed verdict. Because no other model was run on these same cases, this is not a model comparison and is not pooled into the 25-contract matched study.",
    "limitations": [
      "Five contracts and one prompt pair per contract are a small, correlated sample.",
      "Pretraining exposure to CUAD is unknown.",
      "Exact annotated-span reproduction is not legal correctness, full-document retrieval, review-time savings or customer utility.",
      "Local CPU execution does not measure hosted provider availability, price or production behavior."
    ]
  },
  "integrity": {
    "suite_id": "custom-cuad-contract-utility-1-92f71329f421-v1",
    "suite_sha256": "df845d71f886526212b5cc832748105c1b55f930fa7f982dbff559b8570918f3",
    "suite_file_sha256": "47f6363d07b71325ee143eba63522caa6b6cfbbd2d25795abbac0ab50c3fc3e6",
    "excluded_prior_suite_ids": [
      "custom-cuad-contract-utility-1-184124248cd6-v1",
      "custom-cuad-contract-utility-1-20261019-v1",
      "custom-cuad-contract-utility-1-89cdfdb28c64-v1",
      "custom-cuad-contract-utility-1-935d413002c3-v1",
      "custom-cuad-contract-utility-1-a0c3dba4bfa9-v1",
      "custom-cuad-contract-utility-20261019-v1"
    ],
    "source_artifact_sha256": "68e39ac03c16915d80361f15fdd4a8be60244067d7bf00746829b7f97ed45876",
    "checkpoint_sha256": "01d720c717b1147bb420b890db8ae70a6f129f67bee634b9ddd300d9d512202b",
    "checkpoint_audit_sha256": "851afb2349b38b699214c7a7bc3546492313b2299cda6811d619518e6aefa4ba",
    "verified_rows": 10,
    "verification_method": "independent_exact_holdout_oracle.v1",
    "raw_prompts_outputs_and_contract_excerpts_published": false
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "provider_spend_microusd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "money_movement": false,
    "authority_expansion": false
  },
  "receipt_sha256": "394c5c1f9f45e87c9f8b3634c8aecff785c9897e464cba77d376196d46fd3994"
}