{
  "schema": "asn.public-study-replication.v1",
  "study_id": "cuad-five-contract-replication-2026-10-20",
  "suite_id": "custom-cuad-contract-utility-1-184124248cd6-v1",
  "suite_sha256": "0933484e2c4acbb065b587df3644d0576e84701b282cc54680bb0715adfb108b",
  "source_artifact_sha256": "5e553ccbede8ea2f68d19b14141476d890a282659a2b641c8a427580ae8bf31b",
  "source_suite_file_sha256": "534a5152e121ab804fa529383b4404715f3a4a9d787b9e241f802c5f6a6d80f8",
  "created_at": "2026-10-20T06:03:08.596Z",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "license": "CC BY 4.0",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073",
    "archive_sha256": "f8161d18bea4e9c05e78fa6dda61c19c846fb8087ea969c172753bc2f45b999a",
    "excluded_suite_ids": [
      "custom-cuad-contract-utility-1-20261019-v1",
      "custom-cuad-contract-utility-1-935d413002c3-v1",
      "custom-cuad-contract-utility-20261019-v1"
    ]
  },
  "protocol": {
    "unique_contracts": 5,
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 2,
    "planned_attempts": 40,
    "request_timeout_ms": 90000,
    "max_runtime_ms": 900000,
    "grading": "Exact reproduction of the official annotated answer span in a closed JSON field; execution errors are separate from valid-output accuracy.",
    "runtime": "Ollama 0.33.3 on loopback; no external provider calls or spend; no production writes."
  },
  "models": [
    {
      "name": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "attempts": 20,
      "valid_outputs": 20,
      "exact_matches": 0,
      "execution_errors": 0,
      "exact_match_rate_valid_outputs": 0,
      "exact_match_rate_valid_outputs_wilson_95": {
        "confidence_level": 0.95,
        "method": "wilson_score_interval",
        "lower": 0,
        "upper": 0.1611
      },
      "execution_error_rate_all_attempts": 0
    },
    {
      "name": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "attempts": 20,
      "valid_outputs": 10,
      "exact_matches": 2,
      "execution_errors": 10,
      "exact_match_rate_valid_outputs": 0.2,
      "exact_match_rate_valid_outputs_wilson_95": {
        "confidence_level": 0.95,
        "method": "wilson_score_interval",
        "lower": 0.0567,
        "upper": 0.5098
      },
      "execution_error_rate_all_attempts": 0.5
    }
  ],
  "interpretation": {
    "public_ranking_eligible": false,
    "production_measurement_index_eligible": false,
    "general_capability_verified": false,
    "customer_utility_verified": false,
    "statement": "This is a repeated run on one five-contract suite. It adds repetitions, not new task coverage. Ten Llama execution timeouts and wide intervals limit interpretation; these results are not pooled with the five-suite pilot and do not establish a general model ranking."
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "money_movement": false,
    "authority_expansion": false,
    "exact_matches_reverified_from_source_artifact": true
  },
  "replication_receipt_sha256": "72762bd54c9edf1e01cbe65e525f18be1c286d83dfb692c9e9e70fa6c14c47c7"
}