{
  "schema": "asn.public-cuad-timeout-followup.v1",
  "study_id": "cuad-local-timeout-diagnostic-2026-10-20",
  "title": "Llama 3.2 timeout follow-up on the same five CUAD tasks",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "license": "CC BY 4.0",
    "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073",
    "suite_id": "custom-cuad-contract-utility-1-184124248cd6-v1",
    "suite_sha256": "0933484e2c4acbb065b587df3644d0576e84701b282cc54680bb0715adfb108b"
  },
  "protocol": {
    "unique_contracts": 5,
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "previous_repetitions_per_task_per_variant": 2,
    "followup_repetitions_per_task_per_variant": 1,
    "grading": "Parsed answer must exactly reproduce the official annotated span; answers independently checked against the reviewed CUAD test annotations.",
    "runtime": "Ollama 0.33.3 over local loopback only.",
    "previous_request_timeout_ms": 90000,
    "followup_request_timeout_ms": 120000,
    "total_runtime_cap_ms": 900000,
    "provider_calls": 0,
    "provider_spend_microusd": 0,
    "production_writes": 0
  },
  "model": {
    "name": "llama3.2:3b",
    "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
    "parameter_size": "3.2B",
    "quantization": "Q4_K_M"
  },
  "baseline": {
    "attempts": 20,
    "valid_outputs": 10,
    "exact_matches": 2,
    "exact_failures": 8,
    "execution_errors": 10,
    "exact_match_rate_valid_outputs": 0.2,
    "wilson_95_valid_outputs": {
      "confidence_level": 0.95,
      "method": "wilson_score_interval",
      "lower": 0.0567,
      "upper": 0.5098,
      "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
    },
    "mean_inference_latency_ms_valid_outputs": 50048.8,
    "request_timeout_ms": 90000
  },
  "followup": {
    "attempts": 10,
    "valid_outputs": 10,
    "exact_matches": 2,
    "exact_failures": 8,
    "execution_errors": 0,
    "exact_match_rate_valid_outputs": 0.2,
    "wilson_95_valid_outputs": {
      "confidence_level": 0.95,
      "method": "wilson_score_interval",
      "lower": 0.0567,
      "upper": 0.5098,
      "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
    },
    "mean_inference_latency_ms_valid_outputs": 46094,
    "request_timeout_ms": 120000
  },
  "matched_reference": {
    "name": "qwen2.5:1.5b",
    "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
    "attempts": 20,
    "valid_outputs": 20,
    "exact_matches": 0,
    "exact_failures": 20,
    "execution_errors": 0,
    "exact_match_rate_valid_outputs": 0,
    "wilson_95_valid_outputs": {
      "confidence_level": 0.95,
      "method": "wilson_score_interval",
      "lower": 0,
      "upper": 0.1611,
      "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
    },
    "mean_inference_latency_ms_valid_outputs": 38018.9,
    "request_timeout_ms": 90000
  },
  "tasks": [
    {
      "category": "Agreement date",
      "previous": {
        "attempts": 4,
        "valid_outputs": 2,
        "exact_matches": 0,
        "exact_failures": 2,
        "execution_errors": 2,
        "exact_match_rate_valid_outputs": 0,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0,
          "upper": 0.6576,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 37699.1,
        "request_timeout_ms": 90000
      },
      "followup": {
        "attempts": 2,
        "valid_outputs": 2,
        "exact_matches": 0,
        "exact_failures": 2,
        "execution_errors": 0,
        "exact_match_rate_valid_outputs": 0,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0,
          "upper": 0.6576,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 35682.2,
        "request_timeout_ms": 120000
      },
      "both_prompt_variants_passed": false
    },
    {
      "category": "Effective date",
      "previous": {
        "attempts": 4,
        "valid_outputs": 2,
        "exact_matches": 2,
        "exact_failures": 0,
        "execution_errors": 2,
        "exact_match_rate_valid_outputs": 1,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0.3424,
          "upper": 1,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 53018.3,
        "request_timeout_ms": 90000
      },
      "followup": {
        "attempts": 2,
        "valid_outputs": 2,
        "exact_matches": 2,
        "exact_failures": 0,
        "execution_errors": 0,
        "exact_match_rate_valid_outputs": 1,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0.3424,
          "upper": 1,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 43846.6,
        "request_timeout_ms": 120000
      },
      "both_prompt_variants_passed": true
    },
    {
      "category": "Expiration date",
      "previous": {
        "attempts": 4,
        "valid_outputs": 2,
        "exact_matches": 0,
        "exact_failures": 2,
        "execution_errors": 2,
        "exact_match_rate_valid_outputs": 0,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0,
          "upper": 0.6576,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 50177,
        "request_timeout_ms": 90000
      },
      "followup": {
        "attempts": 2,
        "valid_outputs": 2,
        "exact_matches": 0,
        "exact_failures": 2,
        "execution_errors": 0,
        "exact_match_rate_valid_outputs": 0,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0,
          "upper": 0.6576,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 44831,
        "request_timeout_ms": 120000
      },
      "both_prompt_variants_passed": false
    },
    {
      "category": "Renewal term",
      "previous": {
        "attempts": 4,
        "valid_outputs": 2,
        "exact_matches": 0,
        "exact_failures": 2,
        "execution_errors": 2,
        "exact_match_rate_valid_outputs": 0,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0,
          "upper": 0.6576,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 59297.7,
        "request_timeout_ms": 90000
      },
      "followup": {
        "attempts": 2,
        "valid_outputs": 2,
        "exact_matches": 0,
        "exact_failures": 2,
        "execution_errors": 0,
        "exact_match_rate_valid_outputs": 0,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0,
          "upper": 0.6576,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 58236.2,
        "request_timeout_ms": 120000
      },
      "both_prompt_variants_passed": false
    },
    {
      "category": "Notice period to terminate renewal",
      "previous": {
        "attempts": 4,
        "valid_outputs": 2,
        "exact_matches": 0,
        "exact_failures": 2,
        "execution_errors": 2,
        "exact_match_rate_valid_outputs": 0,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0,
          "upper": 0.6576,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 50052.1,
        "request_timeout_ms": 90000
      },
      "followup": {
        "attempts": 2,
        "valid_outputs": 2,
        "exact_matches": 0,
        "exact_failures": 2,
        "execution_errors": 0,
        "exact_match_rate_valid_outputs": 0,
        "wilson_95_valid_outputs": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0,
          "upper": 0.6576,
          "assumption": "independent Bernoulli attempts; repeated runs on the same five tasks may be correlated"
        },
        "mean_inference_latency_ms_valid_outputs": 47874.1,
        "request_timeout_ms": 120000
      },
      "both_prompt_variants_passed": false
    }
  ],
  "interpretation": {
    "public_model_ranking_eligible": false,
    "production_measurement_index_eligible": false,
    "general_capability_verified": false,
    "customer_utility_verified": false,
    "statement": "This is a timeout diagnostic on the same five fixed contracts, not a new holdout or a model comparison. Llama produced two exact matches out of ten valid outputs in both the earlier and follow-up samples. The 10 additional requests completed without execution errors at the longer timeout, but the difference cannot be attributed to timeout alone or generalized beyond this run."
  },
  "limitations": [
    "The follow-up reuses the exact five contracts and prompt variants from the earlier run; it is not an independent sample.",
    "Only one additional paired repetition per task was run. Attempts on the same contracts are correlated, and intervals are descriptive.",
    "Exact annotated-span reproduction is not legal correctness, full-document search, review-time saved or customer value.",
    "Local inference does not establish hosted model behavior, availability, price, energy use or production performance.",
    "Raw prompts, contract excerpts and model outputs are not included in this public receipt."
  ],
  "integrity": {
    "baseline_artifact_sha256": "5e553ccbede8ea2f68d19b14141476d890a282659a2b641c8a427580ae8bf31b",
    "followup_artifact_sha256": "b2d2bbb1bc22a9d29123d34e730a2cef924291de3217420ec2220efdc7d2d6c4",
    "suite_file_sha256": "534a5152e121ab804fa529383b4404715f3a4a9d787b9e241f802c5f6a6d80f8",
    "baseline_verified_rows": 30,
    "baseline_execution_errors": 10,
    "followup_verified_rows": 10,
    "followup_execution_errors": 0,
    "verification_method": "independent_exact_holdout_oracle.v1",
    "checkpoint_sha256": "59da950248f2d1ec1d22cbc97f9cf89816f862977ca179e97455b6b08acc2564",
    "raw_prompts_outputs_and_contract_excerpts_published": false
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "provider_spend_microusd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "money_movement": false,
    "authority_expansion": false
  },
  "receipt_sha256": "871fef145641ab9fce2818afca0259e40fba87317321dcab4179dad362931141"
}