{ "schema": "asn.public-three-model-cuad-holdout.v1", "study_id": "cuad-third-local-model-matched-holdout-2026-10-20", "title": "Third local model on a matched five-contract CUAD holdout", "source": { "name": "Contract Understanding Atticus Dataset (CUAD) v1", "url": "https://github.com/The-Atticus-Project/cuad", "license": "CC BY 4.0", "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021", "split": "official test.json; positive single-span cases only", "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073" }, "protocol": { "unique_contracts": 5, "prompt_variants": [ "canonical", "paraphrase" ], "repetitions_per_task_per_variant": 1, "attempts_per_model": 10, "grading": "Parsed answer must exactly reproduce the official annotated span. Source offsets were checked before prompts were prepared; the models received bounded excerpts and no gold answers.", "runtime": "Ollama 0.33.3 over local loopback only.", "request_timeout_ms": 60000, "total_runtime_cap_ms": 900000, "provider_calls": 0, "provider_spend_microusd": 0, "production_writes": 0 }, "models": [ { "name": "qwen2.5:1.5b", "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b", "parameter_size": "1.5B", "quantization": "Q4_K_M", "attempts": 10, "valid_outputs": 10, "exact_matches": 0, "exact_failures": 10, "execution_errors": 0, "exact_match_rate_valid_outputs": 0, "wilson_95_valid_outputs": { "confidence_level": 0.95, "method": "wilson_score_interval", "lower": 0, "upper": 0.2775 }, "median_inference_latency_ms_valid_calls": 14968.9 }, { "name": "llama3.2:3b", "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", "parameter_size": "3.2B", "quantization": "Q4_K_M", "attempts": 10, "valid_outputs": 9, "exact_matches": 1, "exact_failures": 8, "execution_errors": 1, "exact_match_rate_valid_outputs": 0.1111, "wilson_95_valid_outputs": { "confidence_level": 0.95, "method": "wilson_score_interval", "lower": 0.0199, "upper": 0.435 }, "median_inference_latency_ms_valid_calls": 32768.4 }, { "name": "deepseek-coder:latest", "digest": "3ddd2d3fc8d2b5fe039d18f859271132fd9c7960ef0be1864984442dc2a915d3", "parameter_size": "1B", "quantization": "Q4_0", "attempts": 10, "valid_outputs": 8, "exact_matches": 0, "exact_failures": 8, "execution_errors": 2, "exact_match_rate_valid_outputs": 0, "wilson_95_valid_outputs": { "confidence_level": 0.95, "method": "wilson_score_interval", "lower": 0, "upper": 0.3244 }, "median_inference_latency_ms_valid_calls": 27575.4 } ], "tasks": [ { "category": "Agreement date", "models": { "qwen2.5:1.5b": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false }, "llama3.2:3b": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false }, "deepseek-coder:latest": { "canonical": "verified_fail", "paraphrase": "execution_error", "exact_matches": 0, "execution_errors": 1, "verdict_changed": true } } }, { "category": "Effective date", "models": { "qwen2.5:1.5b": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false }, "llama3.2:3b": { "canonical": "verified_pass", "paraphrase": "verified_fail", "exact_matches": 1, "execution_errors": 0, "verdict_changed": true }, "deepseek-coder:latest": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false } } }, { "category": "Expiration date", "models": { "qwen2.5:1.5b": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false }, "llama3.2:3b": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false }, "deepseek-coder:latest": { "canonical": "verified_fail", "paraphrase": "execution_error", "exact_matches": 0, "execution_errors": 1, "verdict_changed": true } } }, { "category": "Renewal term", "models": { "qwen2.5:1.5b": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false }, "llama3.2:3b": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false }, "deepseek-coder:latest": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false } } }, { "category": "Notice period to terminate renewal", "models": { "qwen2.5:1.5b": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false }, "llama3.2:3b": { "canonical": "verified_fail", "paraphrase": "execution_error", "exact_matches": 0, "execution_errors": 1, "verdict_changed": true }, "deepseek-coder:latest": { "canonical": "verified_fail", "paraphrase": "verified_fail", "exact_matches": 0, "execution_errors": 0, "verdict_changed": false } } } ], "interpretation": { "public_ranking_eligible": false, "production_measurement_index_eligible": false, "general_capability_verified": false, "customer_utility_verified": false, "statement": "This small local holdout compares three fixed local model digests on the same five positive CUAD span-extraction tasks and two prompt phrasings. DeepSeek Coder was added after the earlier Qwen/Llama run on this suite. These are descriptive fixture results, not a general capability ranking or a measure of hosted services." }, "limitations": [ "Five contracts and one attempt per phrasing provide sparse, correlated evidence; pretraining exposure to CUAD is unknown.", "Exact annotated-span reproduction is not legal correctness, full-document search, absent-clause detection, review-time savings or customer utility.", "Execution timeouts count as errors. Local inference time does not measure hosted service latency, provider availability, realized price or production behavior.", "Qwen and Llama records are already included in the separate 25-contract pilot; this extension adds only the DeepSeek Coder run and does not pool it into that pilot." ], "integrity": { "suite_id": "custom-cuad-contract-utility-1-a0c3dba4bfa9-v1", "suite_sha256": "f8729ba5bfbb155cce094581c7a670f0dfa824bd1160444b47953a3885dafb39", "suite_file_sha256": "7f546c6ce2984275eb01fa530f8bc18c187bee96ed2eca83802fd2944548521c", "baseline_artifact_sha256": "3db84d4125051830023738824e7e36d1333dfc25c2ffb5dd25dec4eaad6aeaf1", "third_model_artifact_sha256": "5ccdaef7ea2f9f639ade200dfa3e782e85da5b30135735239b8e623e9490c3d4", "verification_method": "independent_exact_holdout_oracle.v1", "verified_attempts": 27, "execution_errors": 3, "raw_prompts_outputs_and_contract_excerpts_published": false }, "semantics": { "local_only": true, "external_provider_calls": false, "provider_spend_microusd": 0, "production_measurement_written": false, "production_routing_changed": false, "money_movement": false, "authority_expansion": false }, "receipt_sha256": "29e3cc27b36aeb60bf0c2902c9edfcb8fff4389159ee8d9479e75837060514a6" }