{
  "schema": "asn.public-study-evidence.v1",
  "study_id": "qwen-cuad-prompt-robustness-six-contracts-2026-10-26",
  "title": "Qwen 2.5 1.5B · prompt robustness on six CUAD contracts",
  "study_type": "bounded_local_prompt_sensitivity",
  "evaluated_at": "2026-10-26T15:54:25.525Z",
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "dataset_page": "https://www.atticusprojectai.org/cuad/",
    "license": "CC BY 4.0",
    "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073"
  },
  "protocol": {
    "task_count": 6,
    "distinct_contracts": 6,
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 2,
    "total_attempts": 24,
    "temperature": 0,
    "seed": 42,
    "request_timeout_ms": 60000,
    "max_runtime_ms": 900000,
    "parallelism": 1,
    "grading": "Independent exact reproduction of each official CUAD annotated answer span in a closed JSON field.",
    "runtime": "Ollama loopback only; no external inference or provider spend."
  },
  "model": {
    "name": "qwen2.5:1.5b",
    "display_name": "Qwen 2.5 1.5B",
    "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
    "quantization": "Q4_K_M"
  },
  "result": {
    "attempts": 24,
    "verified_passes": 10,
    "verified_failures": 14,
    "execution_errors": 0,
    "by_prompt_variant": {
      "canonical": {
        "attempts": 12,
        "verified_passes": 6,
        "verified_failures": 6,
        "execution_errors": 0,
        "task_passes": 3,
        "task_count": 6,
        "task_level_interval": {
          "lower": 0.1876,
          "upper": 0.8124,
          "method": "wilson_95_percent_task_level"
        }
      },
      "paraphrase": {
        "attempts": 12,
        "verified_passes": 4,
        "verified_failures": 8,
        "execution_errors": 0,
        "task_passes": 2,
        "task_count": 6,
        "task_level_interval": {
          "lower": 0.0968,
          "upper": 0.7,
          "method": "wilson_95_percent_task_level"
        }
      }
    },
    "paired_task_outcomes": {
      "both_pass": 2,
      "canonical_only": 1,
      "paraphrase_only": 0,
      "neither": 3
    },
    "latency_ms": {
      "observations": 24,
      "mean": 11042.861,
      "median": 10913.439,
      "p95": 23318.984,
      "measurement": "Ollama prompt_eval_duration + eval_duration; local runtime only; not hosted latency"
    }
  },
  "tasks": [
    {
      "task_id": "cuad-agreement-date-1a10f44df6de.h1",
      "category": "Agreement date",
      "canonical": {
        "repetitions": 2,
        "verified_passes": 2,
        "verified_failures": 0,
        "replicate_verdicts_stable": true
      },
      "paraphrase": {
        "repetitions": 2,
        "verified_passes": 2,
        "verified_failures": 0,
        "replicate_verdicts_stable": true
      },
      "task_outcome": "both_pass"
    },
    {
      "task_id": "cuad-agreement-date-20d15cd51192.h1",
      "category": "Agreement date",
      "canonical": {
        "repetitions": 2,
        "verified_passes": 2,
        "verified_failures": 0,
        "replicate_verdicts_stable": true
      },
      "paraphrase": {
        "repetitions": 2,
        "verified_passes": 2,
        "verified_failures": 0,
        "replicate_verdicts_stable": true
      },
      "task_outcome": "both_pass"
    },
    {
      "task_id": "cuad-effective-date-303d59853076.h1",
      "category": "Effective date",
      "canonical": {
        "repetitions": 2,
        "verified_passes": 2,
        "verified_failures": 0,
        "replicate_verdicts_stable": true
      },
      "paraphrase": {
        "repetitions": 2,
        "verified_passes": 0,
        "verified_failures": 2,
        "replicate_verdicts_stable": true
      },
      "task_outcome": "canonical_only"
    },
    {
      "task_id": "cuad-expiration-date-17bd3d6a4f73.h1",
      "category": "Expiration date",
      "canonical": {
        "repetitions": 2,
        "verified_passes": 0,
        "verified_failures": 2,
        "replicate_verdicts_stable": true
      },
      "paraphrase": {
        "repetitions": 2,
        "verified_passes": 0,
        "verified_failures": 2,
        "replicate_verdicts_stable": true
      },
      "task_outcome": "neither"
    },
    {
      "task_id": "cuad-renewal-term-7f34a28d5f73.h1",
      "category": "Renewal term",
      "canonical": {
        "repetitions": 2,
        "verified_passes": 0,
        "verified_failures": 2,
        "replicate_verdicts_stable": true
      },
      "paraphrase": {
        "repetitions": 2,
        "verified_passes": 0,
        "verified_failures": 2,
        "replicate_verdicts_stable": true
      },
      "task_outcome": "neither"
    },
    {
      "task_id": "cuad-notice-period-to-terminate-renewal-7055ef47edf6.h1",
      "category": "Notice period to terminate renewal",
      "canonical": {
        "repetitions": 2,
        "verified_passes": 0,
        "verified_failures": 2,
        "replicate_verdicts_stable": true
      },
      "paraphrase": {
        "repetitions": 2,
        "verified_passes": 0,
        "verified_failures": 2,
        "replicate_verdicts_stable": true
      },
      "task_outcome": "neither"
    }
  ],
  "replication": {
    "captured_at": "2026-10-27T02:47:28.511Z",
    "attempts": 24,
    "verified_passes": 10,
    "verified_failures": 14,
    "execution_errors": 0,
    "exact_output_matches_to_first_run": 24,
    "artifact_sha256": "1afd0c0d209af87dab29fafe9000339036863097c57702e33fd128d859e9c5a3",
    "independently_verified_runs": 24,
    "provider_spend_usd": 0
  },
  "evidence": {
    "suite_sha256": "611a4e3fbfacc9dc67f16d462c5d8fcb1953e30d140926f0226d0357f614150c",
    "plan_sha256": "030908b0a757a3a6d5aa58f726498067920a1dd60289bf3b9e8e0f936dbdde10",
    "run_artifact_sha256": "ed5a091b14c279a12eb4f441b75c9aef3154cc35d885dea5a91dbac30b9ca296",
    "replication_artifact_sha256": "1afd0c0d209af87dab29fafe9000339036863097c57702e33fd128d859e9c5a3",
    "field_analysis_sha256": "147df7239dafde3310820a44aa64f0019fe8e1c0116ec0df0a28ad27df5a6f3e",
    "verification_method": "independent_exact_holdout_oracle.v1",
    "raw_prompts_published": false,
    "contract_excerpts_published": false,
    "raw_model_outputs_published": false
  },
  "interpretation": {
    "statement": "Canonical wording passed on three of six contracts; paraphrased wording passed on two. A separate 24-call local run reproduced all 24 outputs and verdicts exactly. This confirms repeatability for this pinned model, prompt pair and six-contract suite, not broader capability.",
    "confidence": "Wilson 95% intervals use six contract-level outcomes, not twelve correlated attempts per wording; the intervals are wide and overlap. The exact-output replication repeats the same six contracts and therefore adds no independent task coverage.",
    "ranking_eligible": false,
    "production_index_eligible": false,
    "general_capability_verified": false,
    "legal_correctness_verified": false,
    "customer_utility_verified": false
  },
  "limitations": [
    "Only six positive CUAD test examples were measured; model pretraining exposure is unknown.",
    "Two repetitions on the same contract are correlated and do not increase the independent task sample beyond six.",
    "Exact reproduction of a public annotation is not legal correctness, full-document search, review-time saved, or customer utility.",
    "One local model build was tested; results do not describe hosted behavior or compare models.",
    "Local electricity, hardware opportunity cost and physical machine details were not measured."
  ],
  "semantics": {
    "local_research_only": true,
    "external_provider_calls": false,
    "provider_spend_usd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "production_index_eligible": false,
    "ranking_eligible": false,
    "tool_execution": false,
    "authority_expansion": false
  },
  "receipt_sha256": "7b7d24e4c7dcdec343a44983a638909773f9212eb8c3edb161a5be664ed1cfb0"
}