{
  "schema": "asn.public-cuad-supplemental-run.v1",
  "study_id": "cuad-llama-three-billion-supplemental-rerun-2026-10-27",
  "title": "Llama 3.2 3B · supplemental CUAD rerun with timeouts",
  "evaluated_at": "2026-10-27T20:11:40.349Z",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "dataset_page": "https://www.atticusprojectai.org/cuad/",
    "license": "CC BY 4.0",
    "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073",
    "archive_sha256": "f8161d18bea4e9c05e78fa6dda61c19c846fb8087ea969c172753bc2f45b999a"
  },
  "protocol": {
    "task_count": 6,
    "distinct_contracts": 6,
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 2,
    "planned_attempts": 24,
    "parallelism": 1,
    "temperature": 0,
    "seed": 42,
    "request_timeout_ms": 60000,
    "max_runtime_ms": 900000,
    "runtime": "Ollama on local loopback only; no external inference or provider spend.",
    "grading": "Independent exact reproduction of each official CUAD annotated answer span in a closed JSON field.",
    "relationship_to_primary_comparison": "A later Llama-only rerun of the same six cases. It is supplemental reliability evidence, not a matched Qwen comparison and not pooled into the primary comparison."
  },
  "model": {
    "name": "llama3.2:3b",
    "display_name": "Llama 3.2 3B",
    "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
    "parameter_size": "3.2B",
    "quantization": "Q4_K_M"
  },
  "results": {
    "attempts": 24,
    "returned_outputs": 19,
    "exact_matches": 2,
    "verified_mismatches": 17,
    "execution_errors": 5,
    "exact_match_rate_of_returned": 0.1053,
    "by_prompt_variant": {
      "canonical": {
        "attempts": 12,
        "returned_outputs": 11,
        "exact_matches": 0,
        "verified_mismatches": 11,
        "execution_errors": 1,
        "exact_match_rate_of_returned": 0
      },
      "paraphrase": {
        "attempts": 12,
        "returned_outputs": 8,
        "exact_matches": 2,
        "verified_mismatches": 6,
        "execution_errors": 4,
        "exact_match_rate_of_returned": 0.25
      }
    },
    "tasks": [
      {
        "case_id": "cuad-agreement-date-1a10f44df6de.h1",
        "label": "Agreement date · case 1",
        "attempts": 4,
        "returned_outputs": 3,
        "exact_matches": 0,
        "verified_mismatches": 3,
        "execution_errors": 1,
        "exact_match_rate_of_returned": 0,
        "by_prompt_variant": {
          "canonical": {
            "attempts": 2,
            "returned_outputs": 2,
            "exact_matches": 0,
            "verified_mismatches": 2,
            "execution_errors": 0,
            "exact_match_rate_of_returned": 0
          },
          "paraphrase": {
            "attempts": 2,
            "returned_outputs": 1,
            "exact_matches": 0,
            "verified_mismatches": 1,
            "execution_errors": 1,
            "exact_match_rate_of_returned": 0
          }
        }
      },
      {
        "case_id": "cuad-agreement-date-20d15cd51192.h1",
        "label": "Agreement date · case 2",
        "attempts": 4,
        "returned_outputs": 4,
        "exact_matches": 2,
        "verified_mismatches": 2,
        "execution_errors": 0,
        "exact_match_rate_of_returned": 0.5,
        "by_prompt_variant": {
          "canonical": {
            "attempts": 2,
            "returned_outputs": 2,
            "exact_matches": 0,
            "verified_mismatches": 2,
            "execution_errors": 0,
            "exact_match_rate_of_returned": 0
          },
          "paraphrase": {
            "attempts": 2,
            "returned_outputs": 2,
            "exact_matches": 2,
            "verified_mismatches": 0,
            "execution_errors": 0,
            "exact_match_rate_of_returned": 1
          }
        }
      },
      {
        "case_id": "cuad-effective-date-303d59853076.h1",
        "label": "Effective date",
        "attempts": 4,
        "returned_outputs": 4,
        "exact_matches": 0,
        "verified_mismatches": 4,
        "execution_errors": 0,
        "exact_match_rate_of_returned": 0,
        "by_prompt_variant": {
          "canonical": {
            "attempts": 2,
            "returned_outputs": 2,
            "exact_matches": 0,
            "verified_mismatches": 2,
            "execution_errors": 0,
            "exact_match_rate_of_returned": 0
          },
          "paraphrase": {
            "attempts": 2,
            "returned_outputs": 2,
            "exact_matches": 0,
            "verified_mismatches": 2,
            "execution_errors": 0,
            "exact_match_rate_of_returned": 0
          }
        }
      },
      {
        "case_id": "cuad-expiration-date-17bd3d6a4f73.h1",
        "label": "Expiration date",
        "attempts": 4,
        "returned_outputs": 2,
        "exact_matches": 0,
        "verified_mismatches": 2,
        "execution_errors": 2,
        "exact_match_rate_of_returned": 0,
        "by_prompt_variant": {
          "canonical": {
            "attempts": 2,
            "returned_outputs": 1,
            "exact_matches": 0,
            "verified_mismatches": 1,
            "execution_errors": 1,
            "exact_match_rate_of_returned": 0
          },
          "paraphrase": {
            "attempts": 2,
            "returned_outputs": 1,
            "exact_matches": 0,
            "verified_mismatches": 1,
            "execution_errors": 1,
            "exact_match_rate_of_returned": 0
          }
        }
      },
      {
        "case_id": "cuad-renewal-term-7f34a28d5f73.h1",
        "label": "Renewal term",
        "attempts": 4,
        "returned_outputs": 3,
        "exact_matches": 0,
        "verified_mismatches": 3,
        "execution_errors": 1,
        "exact_match_rate_of_returned": 0,
        "by_prompt_variant": {
          "canonical": {
            "attempts": 2,
            "returned_outputs": 2,
            "exact_matches": 0,
            "verified_mismatches": 2,
            "execution_errors": 0,
            "exact_match_rate_of_returned": 0
          },
          "paraphrase": {
            "attempts": 2,
            "returned_outputs": 1,
            "exact_matches": 0,
            "verified_mismatches": 1,
            "execution_errors": 1,
            "exact_match_rate_of_returned": 0
          }
        }
      },
      {
        "case_id": "cuad-notice-period-to-terminate-renewal-7055ef47edf6.h1",
        "label": "Notice period to terminate renewal",
        "attempts": 4,
        "returned_outputs": 3,
        "exact_matches": 0,
        "verified_mismatches": 3,
        "execution_errors": 1,
        "exact_match_rate_of_returned": 0,
        "by_prompt_variant": {
          "canonical": {
            "attempts": 2,
            "returned_outputs": 2,
            "exact_matches": 0,
            "verified_mismatches": 2,
            "execution_errors": 0,
            "exact_match_rate_of_returned": 0
          },
          "paraphrase": {
            "attempts": 2,
            "returned_outputs": 1,
            "exact_matches": 0,
            "verified_mismatches": 1,
            "execution_errors": 1,
            "exact_match_rate_of_returned": 0
          }
        }
      }
    ],
    "timeout_interpretation": "A request timeout is an unknown task outcome, not a verified answer failure. Exact-match rate uses only returned outputs; timeouts are shown separately."
  },
  "evidence": {
    "suite_sha256": "611a4e3fbfacc9dc67f16d462c5d8fcb1953e30d140926f0226d0357f614150c",
    "run_artifact_sha256": "c6dfef8698d4dbfaffb63b5a34eb942c88a6cdd93bb8f4e875e0ad564340dd4d",
    "checkpoint_sha256": "6e21370b7d58c6c5a502ab386880f82d580a4bbb5cbc5949ac68a3ff66de0d7b",
    "checkpoint_audit_sha256": "a7b41184a3e91043c065ea0826d2ad7ed9f43882543cb2fe16af33f879dfbc08",
    "field_analysis_sha256": "f68a181e11dd67943d0e94f6bed302235f7f07427531ba14addcb8b7dafa7f72",
    "verified_attempts": 19,
    "attempt_record_count": 24,
    "verification_method": "independent_exact_holdout_oracle.v1",
    "raw_prompts_published": false,
    "contract_excerpts_published": false,
    "raw_model_outputs_published": false
  },
  "interpretation": {
    "statement": "In this bounded rerun, two of 19 returned outputs exactly matched the CUAD annotation; five of 24 requests timed out. The earlier complete Llama run on these cases produced 2/24 exact outputs with no timeouts. The rerun is not a clean model replication or a general capability estimate.",
    "ranking_eligible": false,
    "production_index_eligible": false,
    "general_capability_verified": false,
    "legal_correctness_verified": false,
    "customer_utility_verified": false
  },
  "limitations": [
    "Six positive examples from the public CUAD test split; model pretraining exposure is unknown.",
    "Repeated prompts on the same six contracts are correlated; there is no new task coverage.",
    "Five timeout outcomes are unknown task outcomes; the runner did not preserve partial generations or host-resource telemetry.",
    "This rerun contains only Llama results and cannot be pooled with the Qwen matched comparison.",
    "Exact annotation reproduction is not legal correctness, full-document search, review time saved or customer utility.",
    "The model is a local quantized build; results do not describe hosted behavior. Local electricity and hardware opportunity cost are unmeasured."
  ],
  "semantics": {
    "local_model_research": true,
    "external_provider_calls": false,
    "provider_spend_usd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "production_index_eligible": false,
    "ranking_eligible": false,
    "tool_execution": false,
    "authority_expansion": false,
    "public_raw_prompts": false,
    "public_contract_excerpts": false,
    "public_raw_outputs": false
  },
  "receipt_sha256": "fa761a27d25e001364e6c6d5f44a9ec19af23dd1836a190ef2fd01d8e6ecaba2"
}