{
  "schema": "asn.public-study-evidence.v1",
  "study_id": "cuad-six-contract-qwen-llama-prompt-comparison-2026-10-27",
  "title": "Qwen 2.5 1.5B and Llama 3.2 3B · exact-span reproduction on six CUAD contracts",
  "study_type": "bounded_local_matched_protocol_model_comparison",
  "evaluated_at": "2026-10-27T00:07:18.184Z",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "dataset_page": "https://www.atticusprojectai.org/cuad/",
    "license": "CC BY 4.0",
    "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073",
    "archive_sha256": "f8161d18bea4e9c05e78fa6dda61c19c846fb8087ea969c172753bc2f45b999a"
  },
  "protocol": {
    "task_count": 6,
    "distinct_contracts": 6,
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 2,
    "attempts_per_model": 24,
    "total_attempts": 48,
    "temperature": 0,
    "seed": 42,
    "request_timeout_ms": 60000,
    "max_runtime_ms": 900000,
    "parallelism": 1,
    "grading": "Independent exact reproduction of each official CUAD annotated answer span in a closed JSON field.",
    "runtime": "Ollama 0.33.3 on local loopback only; no external inference or provider spend.",
    "execution": "Qwen and Llama runs were conducted on separate days using the same pinned task suite, prompt variants and runtime version; this is a matched protocol, not a simultaneous randomized trial."
  },
  "models": [
    {
      "name": "qwen2.5:1.5b",
      "display_name": "Qwen 2.5 1.5B",
      "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "quantization": "Q4_K_M",
      "parameter_size": "1.5B",
      "run_date": "2026-10-26",
      "attempts": 24,
      "exact_responses": 10,
      "failed_responses": 14,
      "execution_errors": 0,
      "by_prompt_variant": {
        "canonical": {
          "attempts": 12,
          "exact_responses": 6,
          "failed_responses": 6,
          "contracts_passing_both_repeats": 3,
          "contracts": 6,
          "task_level_95_interval": {
            "lower": 0.1876,
            "upper": 0.8124,
            "method": "wilson_95_percent_task_level"
          }
        },
        "paraphrase": {
          "attempts": 12,
          "exact_responses": 4,
          "failed_responses": 8,
          "contracts_passing_both_repeats": 2,
          "contracts": 6,
          "task_level_95_interval": {
            "lower": 0.0968,
            "upper": 0.7,
            "method": "wilson_95_percent_task_level"
          }
        }
      },
      "median_local_inference_latency_ms": 10913.439,
      "run_artifact_sha256": "ed5a091b14c279a12eb4f441b75c9aef3154cc35d885dea5a91dbac30b9ca296",
      "independently_verified_runs": 24
    },
    {
      "name": "llama3.2:3b",
      "display_name": "Llama 3.2 3B",
      "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "quantization": "Q4_K_M",
      "parameter_size": "3.2B",
      "run_date": "2026-10-27",
      "attempts": 24,
      "exact_responses": 2,
      "failed_responses": 22,
      "execution_errors": 0,
      "by_prompt_variant": {
        "canonical": {
          "attempts": 12,
          "exact_responses": 0,
          "failed_responses": 12,
          "contracts_passing_both_repeats": 0,
          "contracts": 6,
          "task_level_95_interval": {
            "lower": 0,
            "upper": 0.3903,
            "method": "wilson_95_percent_task_level"
          }
        },
        "paraphrase": {
          "attempts": 12,
          "exact_responses": 2,
          "failed_responses": 10,
          "contracts_passing_both_repeats": 1,
          "contracts": 6,
          "task_level_95_interval": {
            "lower": 0.0301,
            "upper": 0.5635,
            "method": "wilson_95_percent_task_level"
          }
        }
      },
      "median_local_inference_latency_ms": 20888.693,
      "run_artifact_sha256": "5ab98cdef9607e797f62766f5db697e57af3cf21338d4cecd194fe1d6a4b7847",
      "independently_verified_runs": 24
    }
  ],
  "results": {
    "by_prompt_variant": {
      "canonical": {
        "qwen2.5:1.5b": {
          "attempts": 12,
          "exact_responses": 6,
          "failed_responses": 6,
          "contracts_passing_both_repeats": 3,
          "contracts": 6,
          "task_level_95_interval": {
            "lower": 0.1876,
            "upper": 0.8124,
            "method": "wilson_95_percent_task_level"
          }
        },
        "llama3.2:3b": {
          "attempts": 12,
          "exact_responses": 0,
          "failed_responses": 12,
          "contracts_passing_both_repeats": 0,
          "contracts": 6,
          "task_level_95_interval": {
            "lower": 0,
            "upper": 0.3903,
            "method": "wilson_95_percent_task_level"
          }
        }
      },
      "paraphrase": {
        "qwen2.5:1.5b": {
          "attempts": 12,
          "exact_responses": 4,
          "failed_responses": 8,
          "contracts_passing_both_repeats": 2,
          "contracts": 6,
          "task_level_95_interval": {
            "lower": 0.0968,
            "upper": 0.7,
            "method": "wilson_95_percent_task_level"
          }
        },
        "llama3.2:3b": {
          "attempts": 12,
          "exact_responses": 2,
          "failed_responses": 10,
          "contracts_passing_both_repeats": 1,
          "contracts": 6,
          "task_level_95_interval": {
            "lower": 0.0301,
            "upper": 0.5635,
            "method": "wilson_95_percent_task_level"
          }
        }
      }
    },
    "latency_definition": "Median Ollama prompt_eval_duration + eval_duration; local runtime only and not hosted latency."
  },
  "tasks": [
    {
      "task_id": "cuad-agreement-date-1a10f44df6de.h1",
      "category": "Agreement date",
      "models": {
        "qwen2.5:1.5b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 2,
            "contracts_passing_both_repeats": true
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 2,
            "contracts_passing_both_repeats": true
          }
        },
        "llama3.2:3b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        }
      }
    },
    {
      "task_id": "cuad-agreement-date-20d15cd51192.h1",
      "category": "Agreement date",
      "models": {
        "qwen2.5:1.5b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 2,
            "contracts_passing_both_repeats": true
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 2,
            "contracts_passing_both_repeats": true
          }
        },
        "llama3.2:3b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 2,
            "contracts_passing_both_repeats": true
          }
        }
      }
    },
    {
      "task_id": "cuad-effective-date-303d59853076.h1",
      "category": "Effective date",
      "models": {
        "qwen2.5:1.5b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 2,
            "contracts_passing_both_repeats": true
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        },
        "llama3.2:3b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        }
      }
    },
    {
      "task_id": "cuad-expiration-date-17bd3d6a4f73.h1",
      "category": "Expiration date",
      "models": {
        "qwen2.5:1.5b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        },
        "llama3.2:3b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        }
      }
    },
    {
      "task_id": "cuad-renewal-term-7f34a28d5f73.h1",
      "category": "Renewal term",
      "models": {
        "qwen2.5:1.5b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        },
        "llama3.2:3b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        }
      }
    },
    {
      "task_id": "cuad-notice-period-to-terminate-renewal-7055ef47edf6.h1",
      "category": "Notice period to terminate renewal",
      "models": {
        "qwen2.5:1.5b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        },
        "llama3.2:3b": {
          "canonical": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          },
          "paraphrase": {
            "attempts": 2,
            "exact_responses": 0,
            "contracts_passing_both_repeats": false
          }
        }
      }
    }
  ],
  "evidence": {
    "suite_sha256": "611a4e3fbfacc9dc67f16d462c5d8fcb1953e30d140926f0226d0357f614150c",
    "qwen_run_artifact_sha256": "ed5a091b14c279a12eb4f441b75c9aef3154cc35d885dea5a91dbac30b9ca296",
    "llama_run_artifact_sha256": "5ab98cdef9607e797f62766f5db697e57af3cf21338d4cecd194fe1d6a4b7847",
    "llama_repeat_artifact_sha256": "00e3e9ef339976ec0dafff9d375bc20ca57b8317cc425c79786623ef52250390",
    "llama_repeat_verified_runs": 24,
    "llama_repeat_matching_output_hashes": 24,
    "llama_repeat_added_distinct_tasks": 0,
    "verification_method": "independent_exact_holdout_oracle.v1",
    "raw_prompts_published": false,
    "contract_excerpts_published": false,
    "raw_model_outputs_published": false
  },
  "interpretation": {
    "statement": "On these six public CUAD examples, Qwen had more exact responses overall (10/24) than Llama (2/24). Qwen passed both repeats on 3/6 contracts with canonical prompts and 2/6 with paraphrases; Llama passed both repeats on 0/6 and 1/6 respectively.",
    "confidence": "Wilson 95% intervals are computed over six contract-level outcomes per prompt and model, not over correlated responses. Samples are small and intervals are wide; no general model ranking is supported.",
    "ranking_eligible": false,
    "production_index_eligible": false,
    "general_capability_verified": false,
    "legal_correctness_verified": false,
    "customer_utility_verified": false
  },
  "limitations": [
    "Six positive examples from a public CUAD test split; model pretraining exposure is unknown.",
    "Repeated responses on the same contract are correlated; the independent task sample is six contracts.",
    "Runs were conducted on separate days. The results compare the same protocol and prompts, not a randomized simultaneous trial.",
    "Exact reproduction of a public annotation is not legal correctness, full-document search, review time saved or customer utility.",
    "Quantized local models only; results do not describe hosted behavior. Local electricity and hardware opportunity cost were not measured."
  ],
  "semantics": {
    "local_research_only": true,
    "external_provider_calls": false,
    "provider_spend_usd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "production_index_eligible": false,
    "ranking_eligible": false,
    "tool_execution": false,
    "authority_expansion": false
  },
  "receipt_sha256": "d546e528808b223cf0dcd1d68917c73a518b5e2b93b63f2a30f198ec8b9bd152"
}