{
  "schema": "asn.public-local-holdout-summary.v1",
  "observed_at": "2026-10-23T18:24:19.349Z",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "dataset_page": "https://www.atticusprojectai.org/cuad/",
    "license": "CC BY 4.0",
    "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073"
  },
  "protocol": {
    "suite_id": "custom-cuad-five-shared-comparison-20261020-v1",
    "source_run_suite_id": "custom-qwen-cuad-disjoint-five-20261020-v1",
    "suite_sha256": "528f5ffdf8ace348e5305ac9f46c3047f6bfa22c655c5079ccdcda01c2a4ec2b",
    "task_count": 5,
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "attempts_per_task_per_variant": 1,
    "total_attempts": 10,
    "oracle_review_state": "independently_reviewed_public_annotations",
    "oracle_method": "Expected answer spans are copied from the official CUAD v1 test split. Each offset was checked against its exact source contract text before prompt construction; the model receives only the surrounding excerpt and question, never the gold answer.",
    "task_set_relation": "same_five_contract_task_set_as_prior_paired_qwen_llama_study",
    "new_contract_tasks": 0
  },
  "model": {
    "name": "Qwen 2.5 1.5B",
    "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
    "quantization": "Q4_K_M",
    "parameter_size": "1.5B"
  },
  "runtime": {
    "name": "ollama",
    "version": "0.33.3",
    "origin_class": "loopback_only",
    "request_timeout_ms": 120000,
    "max_runtime_ms": 900000,
    "provider_spend_microusd": 0
  },
  "result": {
    "verified_exact_matches": 1,
    "valid_outputs": 10,
    "verified_failures": 9,
    "execution_errors": 0,
    "exact_match_rate": 0.1,
    "mean_inference_latency_ms": 19103.993,
    "prompt_variant_verdict_flips": 1,
    "confidence_interval": null,
    "confidence_interval_note": "No binomial interval is reported: two prompt variants are paired within each of only five contracts and should not be treated as ten independent examples."
  },
  "task_results": [
    {
      "task_id": "cuad-agreement-date-1341ae864f76.h1",
      "field": "agreement date",
      "canonical": "verified_pass",
      "paraphrase": "verified_fail",
      "verdict_changed": true,
      "execution_errors": 0
    },
    {
      "task_id": "cuad-effective-date-1881adc7c4e2.h1",
      "field": "effective date",
      "canonical": "verified_fail",
      "paraphrase": "verified_fail",
      "verdict_changed": false,
      "execution_errors": 0
    },
    {
      "task_id": "cuad-expiration-date-13876ff9847e.h1",
      "field": "expiration date",
      "canonical": "verified_fail",
      "paraphrase": "verified_fail",
      "verdict_changed": false,
      "execution_errors": 0
    },
    {
      "task_id": "cuad-renewal-term-419d96219ac0.h1",
      "field": "renewal term",
      "canonical": "verified_fail",
      "paraphrase": "verified_fail",
      "verdict_changed": false,
      "execution_errors": 0
    },
    {
      "task_id": "cuad-renewal-term-4fc02f84a202.h1",
      "field": "renewal term",
      "canonical": "verified_fail",
      "paraphrase": "verified_fail",
      "verdict_changed": false,
      "execution_errors": 0
    }
  ],
  "evidence": {
    "source_artifact_sha256": "8461c048b3f1bc7055736185aa748db8418c2b5283888857a6bf9c1f5706a98e",
    "source_suite_sha256": "528f5ffdf8ace348e5305ac9f46c3047f6bfa22c655c5079ccdcda01c2a4ec2b",
    "raw_prompts_published": false,
    "raw_model_outputs_published": false
  },
  "limitations": [
    "The suite is a sampled public test split; model pretraining exposure is unknown.",
    "Tasks use bounded excerpts and positive annotated spans, not full-document search or absence detection.",
    "Exact-match grading measures annotated-span reproduction, not legal correctness, review time saved or customer utility.",
    "Local energy, opportunity cost and external production behavior are unmeasured.",
    "This is a repeat on the same five task IDs used in the prior paired Qwen–Llama study; it adds zero new contract tasks.",
    "One local model build only; this rerun is not a head-to-head comparison.",
    "Five contracts and one attempt per prompt variant are too few to estimate general performance.",
    "Exact annotated-span reproduction is not legal correctness or customer utility.",
    "Local power use and hardware context were not measured."
  ],
  "semantics": {
    "local_research_only": true,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "production_ranking_eligible": false,
    "general_capability_verified": false,
    "provider_calls": false,
    "provider_spend_microusd": 0,
    "raw_prompts_or_outputs_included": false
  },
  "receipt_sha256": "00b8a1564f4d13131ac5658039ac53f5561ae50ed154b0387b52bb413a49fd9e"
}