{
  "schema": "asn.public-local-holdout-summary.v1",
  "study": {
    "id": "cuad-six-contract-prompt-comparison-2026-10",
    "title": "Six-contract CUAD prompt comparison",
    "scope": "local_ollama_loopback_only",
    "evaluated_at": "2026-10-28T04:52:10.595Z"
  },
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "license": "CC BY 4.0",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073",
    "suite_sha256": "611a4e3fbfacc9dc67f16d462c5d8fcb1953e30d140926f0226d0357f614150c",
    "artifact_sha256": "4eca18e8efb8d154f77a3938de93be2736ed51e88ea245a31c6e8192ee52b080",
    "verification": "verifyLocalHoldoutArtifact recomputed run identities, output hashes, annotation verdicts, timing consistency and summary totals before this sanitized receipt was built"
  },
  "protocol": {
    "task_count": 6,
    "distinct_contracts": 6,
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 2,
    "total_attempts": 24,
    "parallelism": 1,
    "grading": "Exact reproduction of the official CUAD annotated answer span; independently checked against the pinned task definitions."
  },
  "model": {
    "name": "llama3.2:3b",
    "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
    "quantization": null
  },
  "result": {
    "attempts": 24,
    "verified_passes": 2,
    "verified_failures": 22,
    "execution_errors": 0,
    "by_prompt_variant": {
      "canonical": {
        "attempts": 12,
        "verified_passes": 0,
        "verified_failures": 12,
        "execution_errors": 0
      },
      "paraphrase": {
        "attempts": 12,
        "verified_passes": 2,
        "verified_failures": 10,
        "execution_errors": 0
      }
    },
    "tasks": [
      {
        "task_id": "cuad-agreement-date-1a10f44df6de.h1",
        "canonical_passes": 0,
        "paraphrase_passes": 0,
        "verified_failures": 4,
        "execution_errors": 0
      },
      {
        "task_id": "cuad-agreement-date-20d15cd51192.h1",
        "canonical_passes": 0,
        "paraphrase_passes": 2,
        "verified_failures": 2,
        "execution_errors": 0
      },
      {
        "task_id": "cuad-effective-date-303d59853076.h1",
        "canonical_passes": 0,
        "paraphrase_passes": 0,
        "verified_failures": 4,
        "execution_errors": 0
      },
      {
        "task_id": "cuad-expiration-date-17bd3d6a4f73.h1",
        "canonical_passes": 0,
        "paraphrase_passes": 0,
        "verified_failures": 4,
        "execution_errors": 0
      },
      {
        "task_id": "cuad-renewal-term-7f34a28d5f73.h1",
        "canonical_passes": 0,
        "paraphrase_passes": 0,
        "verified_failures": 4,
        "execution_errors": 0
      },
      {
        "task_id": "cuad-notice-period-to-terminate-renewal-7055ef47edf6.h1",
        "canonical_passes": 0,
        "paraphrase_passes": 0,
        "verified_failures": 4,
        "execution_errors": 0
      }
    ]
  },
  "baseline": {
    "receipt_path": "/qwen-cuad-prompt-robustness-20261026.json",
    "receipt_sha256": "7b7d24e4c7dcdec343a44983a638909773f9212eb8c3edb161a5be664ed1cfb0",
    "model": "qwen2.5:1.5b",
    "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
    "attempts": 24,
    "verified_passes": 10,
    "verified_failures": 14,
    "execution_errors": 0
  },
  "limitations": [
    "Six sampled positive-span public test examples only; training exposure is unknown.",
    "Prompt repetitions on the same contracts are correlated; the independent task sample is six.",
    "Exact annotation matching is not legal correctness, full-document retrieval, customer utility or hosted behavior.",
    "The public receipt contains aggregate outcomes and hashes, not prompts, contract excerpts or raw model outputs."
  ],
  "semantics": {
    "local_research_only": true,
    "external_provider_calls": false,
    "provider_spend_usd": 0,
    "production_measurement_written": false,
    "production_index_eligible": false,
    "ranking_eligible": false,
    "routing_changed": false,
    "authority_expansion": false,
    "raw_prompts_published": false,
    "contract_excerpts_published": false,
    "raw_model_outputs_published": false
  },
  "receipt_sha256": "77cc224f926d62ffac965703f2e64de4dfda9969c478a3477daa370ce502b1c1"
}