{
  "schema": "asn.public-study-evidence.v1",
  "study_id": "cuad-twenty-five-contract-local-extraction-pilot-2026-10",
  "title": "Twenty-five-contract CUAD clause extraction pilot",
  "study_type": "bounded_local_model_pilot",
  "evaluated_at": {
    "start": "2026-10-19T05:19:19.702Z",
    "end": "2026-10-20T09:52:41.852Z"
  },
  "source": {
    "name": "Contract Understanding Atticus Dataset (CUAD) v1",
    "url": "https://github.com/The-Atticus-Project/cuad",
    "dataset_page": "https://www.atticusprojectai.org/cuad/",
    "license": "CC BY 4.0",
    "attribution": "The Atticus Project, Inc.; CUAD v1; Hendrycks et al., NeurIPS 2021",
    "split": "official test.json; positive single-span cases only",
    "dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073",
    "archive_sha256": "f8161d18bea4e9c05e78fa6dda61c19c846fb8087ea969c172753bc2f45b999a",
    "suite_id": null,
    "suite_sha256": null,
    "suite_ids": [
      "custom-cuad-contract-utility-1-184124248cd6-v1",
      "custom-cuad-contract-utility-1-20261019-v1",
      "custom-cuad-contract-utility-1-89cdfdb28c64-v1",
      "custom-cuad-contract-utility-1-935d413002c3-v1",
      "custom-cuad-contract-utility-1-a0c3dba4bfa9-v1"
    ],
    "suite_sha256_by_id": {
      "custom-cuad-contract-utility-1-184124248cd6-v1": "0933484e2c4acbb065b587df3644d0576e84701b282cc54680bb0715adfb108b",
      "custom-cuad-contract-utility-1-20261019-v1": "c494290e35a233f0cecaf24fb914a2115905653b33c4bba1862e843d2b01000c",
      "custom-cuad-contract-utility-1-89cdfdb28c64-v1": "c3cfa2534b9967dbd7b04e6ee84bcc670af7b0792566cb29b246c0c08964e055",
      "custom-cuad-contract-utility-1-935d413002c3-v1": "d503eb4b337302f7796e817868852215481a80b19b0abc0e8c5f68cbcbcbee8e",
      "custom-cuad-contract-utility-1-a0c3dba4bfa9-v1": "f8729ba5bfbb155cce094581c7a670f0dfa824bd1160444b47953a3885dafb39"
    },
    "excluded_suite_ids": [
      "custom-cuad-contract-utility-20261019-v1"
    ],
    "oracle_method": "Expected answer spans are copied from the official CUAD v1 test split. Each offset was checked against its exact source contract text before prompt construction; the model receives only the surrounding excerpt and question, never the gold answer.",
    "oracle_review_state": "independently_reviewed_public_annotations"
  },
  "protocol": {
    "task_count": 25,
    "unique_contract_examples": 25,
    "independent_suite_count": 5,
    "positive_annotated_spans_only": true,
    "variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 1,
    "grading": "Parsed JSON answer must reproduce the official annotated source span exactly; every expected span offset was checked against the source contract.",
    "runtime": "Ollama on loopback only; no external provider calls or provider spend.",
    "ollama_version": "0.33.3",
    "request_timeout_by_suite": {
      "custom-cuad-contract-utility-1-184124248cd6-v1": [
        60000,
        60000
      ],
      "custom-cuad-contract-utility-1-20261019-v1": [
        null
      ],
      "custom-cuad-contract-utility-1-89cdfdb28c64-v1": [
        60000,
        60000
      ],
      "custom-cuad-contract-utility-1-935d413002c3-v1": [
        null
      ],
      "custom-cuad-contract-utility-1-a0c3dba4bfa9-v1": [
        60000
      ]
    },
    "execution_error_note": "Execution errors are included separately from valid exact-match denominators. Earlier suite receipts do not all record a per-request timeout limit; the two latest suites record 60 seconds and completed without execution errors.",
    "raw_contract_excerpts_or_model_outputs_published": false
  },
  "aggregate": {
    "task_scope": "structured_extraction · CUAD clause-span reproduction",
    "uncertainty_method": "Wilson score interval at 95% confidence for valid-output exact-match proportions; descriptive only and not adjusted for paired prompts or task clustering.",
    "models": [
      {
        "name": "qwen2.5:1.5b",
        "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
        "parameter_size": "1.5B",
        "quantization": "Q4_K_M",
        "attempts": 50,
        "valid_outputs": 49,
        "exact_matches": 3,
        "exact_failures": 46,
        "execution_errors": 1,
        "exact_match_rate_valid_outputs": 0.0612,
        "exact_match_rate_valid_outputs_wilson_95": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0.021,
          "upper": 0.1652
        },
        "exact_match_rate_all_attempts": 0.06,
        "source_artifact_hashes": [
          "3db84d4125051830023738824e7e36d1333dfc25c2ffb5dd25dec4eaad6aeaf1",
          "66221589c015bbdc300e71cc22a964fcb9bb723bce828b82d23c19124e36721b",
          "91d9f3968c77fa69339cea83997070be5bee98b8e5cac338113a58a92160f90a",
          "ca415f30ede3c09b99945818499ca0f45443678a8087ee2da476d3fd7f17762a",
          "e320af3914387e9fd465676819801def823137d197aea30b6280e98d2fe8b019"
        ]
      },
      {
        "name": "llama3.2:3b",
        "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
        "parameter_size": "3B",
        "quantization": "Q4_K_M",
        "attempts": 50,
        "valid_outputs": 45,
        "exact_matches": 5,
        "exact_failures": 40,
        "execution_errors": 5,
        "exact_match_rate_valid_outputs": 0.1111,
        "exact_match_rate_valid_outputs_wilson_95": {
          "confidence_level": 0.95,
          "method": "wilson_score_interval",
          "lower": 0.0484,
          "upper": 0.235
        },
        "exact_match_rate_all_attempts": 0.1,
        "source_artifact_hashes": [
          "0c33a8ea5f2446855dc7c7fc7e4aa191edaa7b37e757bfe18aa47647d94fc364",
          "3db84d4125051830023738824e7e36d1333dfc25c2ffb5dd25dec4eaad6aeaf1",
          "91d9f3968c77fa69339cea83997070be5bee98b8e5cac338113a58a92160f90a",
          "a33639f02d6bc58e332402fa9d9f2998d3412f7fe7ea8eaba6ae5cc46adac1c8",
          "ca415f30ede3c09b99945818499ca0f45443678a8087ee2da476d3fd7f17762a"
        ]
      }
    ],
    "by_suite": [
      {
        "suite_id": "custom-cuad-contract-utility-1-184124248cd6-v1",
        "suite_sha256": "0933484e2c4acbb065b587df3644d0576e84701b282cc54680bb0715adfb108b",
        "task_count": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 9,
            "exact_matches": 0,
            "execution_errors": 1,
            "exact_match_rate_valid_outputs": 0
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 9,
            "exact_matches": 2,
            "execution_errors": 1,
            "exact_match_rate_valid_outputs": 0.2222
          }
        ]
      },
      {
        "suite_id": "custom-cuad-contract-utility-1-20261019-v1",
        "suite_sha256": "c494290e35a233f0cecaf24fb914a2115905653b33c4bba1862e843d2b01000c",
        "task_count": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 2,
            "execution_errors": 0,
            "exact_match_rate_valid_outputs": 0.2
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 9,
            "exact_matches": 1,
            "execution_errors": 1,
            "exact_match_rate_valid_outputs": 0.1111
          }
        ]
      },
      {
        "suite_id": "custom-cuad-contract-utility-1-89cdfdb28c64-v1",
        "suite_sha256": "c3cfa2534b9967dbd7b04e6ee84bcc670af7b0792566cb29b246c0c08964e055",
        "task_count": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 0,
            "execution_errors": 0,
            "exact_match_rate_valid_outputs": 0
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 0,
            "execution_errors": 0,
            "exact_match_rate_valid_outputs": 0
          }
        ]
      },
      {
        "suite_id": "custom-cuad-contract-utility-1-935d413002c3-v1",
        "suite_sha256": "d503eb4b337302f7796e817868852215481a80b19b0abc0e8c5f68cbcbcbee8e",
        "task_count": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 1,
            "execution_errors": 0,
            "exact_match_rate_valid_outputs": 0.1
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 8,
            "exact_matches": 1,
            "execution_errors": 2,
            "exact_match_rate_valid_outputs": 0.125
          }
        ]
      },
      {
        "suite_id": "custom-cuad-contract-utility-1-a0c3dba4bfa9-v1",
        "suite_sha256": "f8729ba5bfbb155cce094581c7a670f0dfa824bd1160444b47953a3885dafb39",
        "task_count": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 0,
            "execution_errors": 0,
            "exact_match_rate_valid_outputs": 0
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 9,
            "exact_matches": 1,
            "execution_errors": 1,
            "exact_match_rate_valid_outputs": 0.1111
          }
        ]
      }
    ],
    "by_category": [
      {
        "category": "Agreement Date",
        "unique_contracts": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 2,
            "execution_errors": 0
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 0,
            "execution_errors": 0
          }
        ]
      },
      {
        "category": "Effective Date",
        "unique_contracts": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 1,
            "execution_errors": 0
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 9,
            "exact_matches": 3,
            "execution_errors": 1
          }
        ]
      },
      {
        "category": "Expiration Date",
        "unique_contracts": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 0,
            "execution_errors": 0
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 9,
            "exact_matches": 1,
            "execution_errors": 1
          }
        ]
      },
      {
        "category": "Renewal Term",
        "unique_contracts": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 9,
            "exact_matches": 0,
            "execution_errors": 1
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 8,
            "exact_matches": 0,
            "execution_errors": 2
          }
        ]
      },
      {
        "category": "Notice Period To Terminate Renewal",
        "unique_contracts": 5,
        "models": [
          {
            "name": "qwen2.5:1.5b",
            "attempts": 10,
            "valid_outputs": 10,
            "exact_matches": 0,
            "execution_errors": 0
          },
          {
            "name": "llama3.2:3b",
            "attempts": 10,
            "valid_outputs": 9,
            "exact_matches": 1,
            "execution_errors": 1
          }
        ]
      }
    ]
  },
  "task_results": [
    {
      "task_id": "cuad-agreement-date-003155dd4f9c.h1",
      "task_class": "structured_extraction",
      "category": "Agreement Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 1,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_pass",
              "output_sha256": "a2ac5f2bdc40ffae217a7e2353adcabb6fb2c304500430e729c061528e54242a",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "53c0b5d0361bd0a3bdcabde8b193aafe18d31d68284a2b47b31d43ed59bf150f",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "f52ed94829434c1b46a065addbc3cdff256ec0556e371b47d16a3727158c8943",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-agreement-date-039a49e8fba3.h1",
      "task_class": "structured_extraction",
      "category": "Agreement Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 1,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_pass",
              "output_sha256": "bb81f5146a284e176ca6b26c6370a277d569e64a86a2975925f43032ae3d3274",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "29d83812b6b59340e3f60e8752b58922e33d6949b5ac856e86dbd4894d2b60ee",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "8125cf5c8e023249c96503b15fdde4cc7e44b6ef0e47816c6a01b415a1118a40",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-agreement-date-0583b8777c49.h1",
      "task_class": "structured_extraction",
      "category": "Agreement Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "093f0d88c24f445043c0adddeea6e6ba75888e5ef637a06ebdc45560edc849d3",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "093f0d88c24f445043c0adddeea6e6ba75888e5ef637a06ebdc45560edc849d3",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "944d50ac8b4457c12d0c50e687d8e244371f506cda48a56740f70c131588270c",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "322a77b59de15f3044d65a5073fec0bbdc3664928beaffda0faec9431ba1bfa1",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-agreement-date-115842a1eb7c.h1",
      "task_class": "structured_extraction",
      "category": "Agreement Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "093f0d88c24f445043c0adddeea6e6ba75888e5ef637a06ebdc45560edc849d3",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "093f0d88c24f445043c0adddeea6e6ba75888e5ef637a06ebdc45560edc849d3",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "f0013f692255f29404aa403bea56d2e877e24466816d566736443d5832ec75c0",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-agreement-date-13f294339966.h1",
      "task_class": "structured_extraction",
      "category": "Agreement Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "50b297b29a1f176e8bf2276500f0245347e2345c7455904205de818226aa439f",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "a02253126bd8db401dd65a227fec5b59a9c863e04d5965532f456c79ce652def",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "c0d3fd0809439ba9986a3218275c6e5cc4beb9dec7f745289017fb1ea948b2db",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-effective-date-041cef3ca2c1.h1",
      "task_class": "structured_extraction",
      "category": "Effective Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 1,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_pass",
              "output_sha256": "d70833788e622ff2ccd87b78c6a5ea5d51a928ebb69e6a0ac6dc27a1ad58e2a9",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "f9d33749c8897c86f24199c20518820a676ec925f0bc2afdc68b57b2ec05d728",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "ec57b6982b2c345115be6301ba101a8f98421dc456b2df7cc05efc190516cda1",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "f9d33749c8897c86f24199c20518820a676ec925f0bc2afdc68b57b2ec05d728",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-effective-date-0a3cdb64f4fb.h1",
      "task_class": "structured_extraction",
      "category": "Effective Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "0bc610e81f2517cc2dbae241526be96f696cd583de325579fb7a2d3d368ee163",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "d9811bb6eb99453e6e42915cc4e9ff2bc46993758318deb14586624a233fac90",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 1,
          "exact_matches": 0,
          "execution_errors": 1,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "execution_error",
              "output_sha256": null,
              "verification_method": null
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-effective-date-0bfe99b9a2ae.h1",
      "task_class": "structured_extraction",
      "category": "Effective Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "f9d33749c8897c86f24199c20518820a676ec925f0bc2afdc68b57b2ec05d728",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "f9d33749c8897c86f24199c20518820a676ec925f0bc2afdc68b57b2ec05d728",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 1,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_pass",
              "output_sha256": "25d8088d665cbabb0e996009787454a2caeca323362e6486d70b563c31249b0a",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "8cde20eaa420896e6e0954442ee968fb5c01ebeb41317576373159286e7fb7fc",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-effective-date-159c248d476e.h1",
      "task_class": "structured_extraction",
      "category": "Effective Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "f9d33749c8897c86f24199c20518820a676ec925f0bc2afdc68b57b2ec05d728",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "26f0f424351e48112addffba8a2c4816b5e1fdbfc1236559316487eecb4b58eb",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "f9d33749c8897c86f24199c20518820a676ec925f0bc2afdc68b57b2ec05d728",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-effective-date-1a0768c7e14d.h1",
      "task_class": "structured_extraction",
      "category": "Effective Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "a2506b948f130f4004d8de37c9a2dd7e432ec790c4c34606084537fa64b42588",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "db8b60d016b588e604e3e6873d20e2a7049e8387557d71a02b3d0056a9473f15",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 2,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_pass",
              "output_sha256": "48210394094bca20ce224cd984b46c45417570ad86ee5fae19e8a5237c60477e",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_pass",
              "output_sha256": "48210394094bca20ce224cd984b46c45417570ad86ee5fae19e8a5237c60477e",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-expiration-date-0a56035fc46d.h1",
      "task_class": "structured_extraction",
      "category": "Expiration Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "09cd634a6a4b17d3640e180faee1f417b0d6ed94402ba1f1aeaec8d5c09c474f",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "6379b3cf973e25f28e1089776c2d5fd9151fde54159036d85a6080da3ae2fffa",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 1,
          "exact_matches": 0,
          "execution_errors": 1,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "a669a63de2ce6bcf23576be0bdfa7a3cbda91b21cbc0b510088c4e8dd5ad7ab8",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "execution_error",
              "output_sha256": null,
              "verification_method": null
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-expiration-date-0b26726812b5.h1",
      "task_class": "structured_extraction",
      "category": "Expiration Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "bfcde74c91dbf737d772b3e54117f7468009b4c616a1b65767f080287c9fc544",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "bfcde74c91dbf737d772b3e54117f7468009b4c616a1b65767f080287c9fc544",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 1,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "f055f479b23e7469c4662535552fca7ab306b77c5775429ea76baba302d80842",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_pass",
              "output_sha256": "cf19faa079a419312a3b422192287127023a17cdc5b89fa08f09d4d42774c685",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-expiration-date-0b8248e471ba.h1",
      "task_class": "structured_extraction",
      "category": "Expiration Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "6336762b2fae947e31401cda7ca0f2090ee17aa01d96e48396c68f4df5ef19eb",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "6336762b2fae947e31401cda7ca0f2090ee17aa01d96e48396c68f4df5ef19eb",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "eaff86624e35e2d38b8f77fc69d166350c337976d6b2bb2dc45dcc8bdcaa849b",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-expiration-date-11ddc9335adb.h1",
      "task_class": "structured_extraction",
      "category": "Expiration Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "ac4cccc1401a161fe8bf333dc253345319c39e454762c87235728ffffce6dc31",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "c6abfed43b4d488d454058cebf8c888a28a4be5ddcfd7ab44e3092695268f1d8",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "f3ec8796d923f3d65f83c7216fe505dbab0aeb85f651145f0cae8514366f9cb0",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-expiration-date-14fa082b54b9.h1",
      "task_class": "structured_extraction",
      "category": "Expiration Date",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "8548783cba26bb5c15d37f7727cf75c84fe1c21ed2e86a6c215899f4a7a007a5",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "89ac819b5e99f1ce1973d701fadf067b11db121c780392364bc92df4ec12ab8e",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "900dd64ed71a9815b6e79bcf118fa71f5703ca807b71214a639463b47fb9f1ad",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-notice-period-to-terminate-renewal-0bb0b7de61b7.h1",
      "task_class": "structured_extraction",
      "category": "Notice Period To Terminate Renewal",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "37b594f9d18ade8fa79b90fd3bfa9616b784c49cab772aa54c85d936740281d2",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "b938862e9df5c7f6a1302f58d1847091af1d5bfe668ea9a45bb1145cfcb8416c",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 1,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_pass",
              "output_sha256": "7414bef53436c602e927a01c15b8babe63338acf5612cda66686aa863e26ba3f",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-notice-period-to-terminate-renewal-209db78ec51c.h1",
      "task_class": "structured_extraction",
      "category": "Notice Period To Terminate Renewal",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "cf80fa34e98f59b2c2d7921626b27fcd637df23de0ea74668e0d86992c07b64c",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "0490487bbbd13714c9413474536bf83a002a9650e89762ec0d86b8efc6139b01",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "cf80fa34e98f59b2c2d7921626b27fcd637df23de0ea74668e0d86992c07b64c",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "cf80fa34e98f59b2c2d7921626b27fcd637df23de0ea74668e0d86992c07b64c",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-notice-period-to-terminate-renewal-240601976697.h1",
      "task_class": "structured_extraction",
      "category": "Notice Period To Terminate Renewal",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "86f713afc30766df6edb92ac0a39118807e566608fe8ab24751f5b7b664d605f",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "865d9f0cf9d62bcf01552e5edd26f97c37212bcf9b1cb16bd98b567adf361bcb",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "88dfcc407cb8c14cb41bdb8654247693f09c950dde5214dc8be6d85f5de5e08e",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-notice-period-to-terminate-renewal-2db46c2c735e.h1",
      "task_class": "structured_extraction",
      "category": "Notice Period To Terminate Renewal",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "1d98dfecd916f83f9a6e7652713193785be78e12577c9d29a40941bbc35771f5",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "ecc46d819ff380c76c899f364bf1278aaf3859b4e0f2af703c54db3898b7c073",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 1,
          "exact_matches": 0,
          "execution_errors": 1,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "b9f6b2ddc34749765e4e0fd747ef582d30c94e8e13c4259d25a28fe7cf7b1120",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "execution_error",
              "output_sha256": null,
              "verification_method": null
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-notice-period-to-terminate-renewal-2ecb16b4fc78.h1",
      "task_class": "structured_extraction",
      "category": "Notice Period To Terminate Renewal",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "7610e30550b727cce45d299c11b90864c91b96d48ebbeebb61a29f6decbcdfe6",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "79722bf64d656d257955bf40985df018639ea80773d0a5b719268b01462c3021",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "e65f44a4b37b86417f3ff74401bc023f030ab4f41a714f1e523ff154bdedfee0",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-renewal-term-06d1553ad59d.h1",
      "task_class": "structured_extraction",
      "category": "Renewal Term",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "10a5d1f2c4491c640f9a1dd5d50b715190b7dd65d8d5eca61b27a40c56270d63",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "6c8297dada65732a45ce489d5d25d72cda3bbe7d367d882c58a98c204a1f7ac3",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "7d29d0da0b7406dfc7537644b61b5fcb723902d280cbda1088b7eeafc4c3b04f",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-renewal-term-214cb35b8d41.h1",
      "task_class": "structured_extraction",
      "category": "Renewal Term",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "5e44e14755cc25e6118e6faf394b96ce1ad65be4ac80d6925622720a3ad95b5b",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "5e44e14755cc25e6118e6faf394b96ce1ad65be4ac80d6925622720a3ad95b5b",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 1,
          "exact_matches": 0,
          "execution_errors": 1,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "da12f9ed8e0900ffddefd3fa11ff91c72c462c0691527a7a4ffc565c9548368d",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "execution_error",
              "output_sha256": null,
              "verification_method": null
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-renewal-term-21c7f671da51.h1",
      "task_class": "structured_extraction",
      "category": "Renewal Term",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "6fd3933c0d6d5f8bbb01408e039addacee6b85b5dbbf82c49235cfe52dd80ab9",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "6e9b4463ef9a52bfaf82af531010b6893d80853665e4f65a09b6f79bc7cc598a",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "00e582f3661893fb44cd7eaba1e03df045505c6febf8f04850d0e51194e0610c",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "bcd0c65d6dbb13a80812dfb306e2eda4740519b881ce9bad4791d9e0eb3305a4",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-renewal-term-22fad784f724.h1",
      "task_class": "structured_extraction",
      "category": "Renewal Term",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "1266e43a50508e24075b49a38698f6a89b6362ba8d7e776cf5a89779ae8ddbcf",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "08fe964cb894ee1e5466ddd387ec09e39f693dd4d6fdbc058a15ab47a1658753",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "08fe964cb894ee1e5466ddd387ec09e39f693dd4d6fdbc058a15ab47a1658753",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail",
              "output_sha256": "08fe964cb894ee1e5466ddd387ec09e39f693dd4d6fdbc058a15ab47a1658753",
              "verification_method": "independent_exact_holdout_oracle.v1"
            }
          ]
        }
      ]
    },
    {
      "task_id": "cuad-renewal-term-289daceecade.h1",
      "task_class": "structured_extraction",
      "category": "Renewal Term",
      "models": [
        {
          "model": "qwen2.5:1.5b",
          "attempted": 2,
          "valid_outputs": 1,
          "exact_matches": 0,
          "execution_errors": 1,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "13bead7234140d826d27f46004e54c3579305d67f4eee73bc015f8789e046dfc",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "execution_error",
              "output_sha256": null,
              "verification_method": null
            }
          ]
        },
        {
          "model": "llama3.2:3b",
          "attempted": 2,
          "valid_outputs": 1,
          "exact_matches": 0,
          "execution_errors": 1,
          "variants": [
            {
              "name": "canonical",
              "state": "verified_fail",
              "output_sha256": "3bdf68cc1aec55545f98aa543b2f023f23ac6ec1293c4b67bc998cc1e33b3c60",
              "verification_method": "independent_exact_holdout_oracle.v1"
            },
            {
              "name": "paraphrase",
              "state": "execution_error",
              "output_sha256": null,
              "verification_method": null
            }
          ]
        }
      ]
    }
  ],
  "interpretation": {
    "public_model_ranking_eligible": false,
    "production_measurement_index_eligible": false,
    "customer_utility_verified": false,
    "general_model_capability_verified": false,
    "statement": "Across five disjoint five-contract CUAD suites, Qwen 2.5 1.5B exactly reproduced 3 of 49 valid outputs (1 execution error); Llama 3.2 3B reproduced 5 of 45 valid outputs (5 execution errors). Both models share all 25 contract tasks, but this small descriptive study is not a general ranking.",
    "limitations": [
      "Only 25 unique contracts, selected across five non-overlapping suites, and one repetition per prompt variant were measured; paired variants are not independent examples.",
      "Execution errors are not counted as valid outputs; timeout caps are absent from the first two suite receipts, so failure causes cannot be harmonized across all suites.",
      "Pretraining exposure to the public CUAD dataset is unknown.",
      "Tasks use bounded excerpts with positive annotations; they do not measure full-document retrieval, absence detection, legal correctness, review time saved or customer utility.",
      "Local inference is not provider-hosted; external availability, prices, energy, hardware-normalized speed and production behavior were not measured.",
      "Exact-span match follows CUAD labels and is not legal advice or an assessment of legal correctness."
    ]
  },
  "integrity": {
    "source_dataset_sha256": "007b6a40b0c65247f881627375c3d2e9b6aeeb5dfa89957494feed0765a1a073",
    "source_archive_sha256": "f8161d18bea4e9c05e78fa6dda61c19c846fb8087ea969c172753bc2f45b999a",
    "suites": [
      {
        "suite_id": "custom-cuad-contract-utility-1-184124248cd6-v1",
        "suite_sha256": "0933484e2c4acbb065b587df3644d0576e84701b282cc54680bb0715adfb108b",
        "task_count": 5
      },
      {
        "suite_id": "custom-cuad-contract-utility-1-20261019-v1",
        "suite_sha256": "c494290e35a233f0cecaf24fb914a2115905653b33c4bba1862e843d2b01000c",
        "task_count": 5
      },
      {
        "suite_id": "custom-cuad-contract-utility-1-89cdfdb28c64-v1",
        "suite_sha256": "c3cfa2534b9967dbd7b04e6ee84bcc670af7b0792566cb29b246c0c08964e055",
        "task_count": 5
      },
      {
        "suite_id": "custom-cuad-contract-utility-1-935d413002c3-v1",
        "suite_sha256": "d503eb4b337302f7796e817868852215481a80b19b0abc0e8c5f68cbcbcbee8e",
        "task_count": 5
      },
      {
        "suite_id": "custom-cuad-contract-utility-1-a0c3dba4bfa9-v1",
        "suite_sha256": "f8729ba5bfbb155cce094581c7a670f0dfa824bd1160444b47953a3885dafb39",
        "task_count": 5
      }
    ],
    "source_artifacts": [
      {
        "path": "brain/evidence/cuad-public-contract-utility-quick5-20261019.json",
        "artifact_sha256": "91d9f3968c77fa69339cea83997070be5bee98b8e5cac338113a58a92160f90a",
        "suite_sha256": "c494290e35a233f0cecaf24fb914a2115905653b33c4bba1862e843d2b01000c",
        "verified_rows": 19
      },
      {
        "path": "brain/evidence/cuad-public-contract-utility-quick5-replication-20261019.json",
        "artifact_sha256": "ca415f30ede3c09b99945818499ca0f45443678a8087ee2da476d3fd7f17762a",
        "suite_sha256": "d503eb4b337302f7796e817868852215481a80b19b0abc0e8c5f68cbcbcbee8e",
        "verified_rows": 18
      },
      {
        "path": "brain/evidence/cuad-public-contract-utility-next5-20261019.json",
        "artifact_sha256": "66221589c015bbdc300e71cc22a964fcb9bb723bce828b82d23c19124e36721b",
        "suite_sha256": "0933484e2c4acbb065b587df3644d0576e84701b282cc54680bb0715adfb108b",
        "verified_rows": 9
      },
      {
        "path": "brain/evidence/cuad-public-contract-utility-llama3-20261020.json",
        "artifact_sha256": "a33639f02d6bc58e332402fa9d9f2998d3412f7fe7ea8eaba6ae5cc46adac1c8",
        "suite_sha256": "0933484e2c4acbb065b587df3644d0576e84701b282cc54680bb0715adfb108b",
        "verified_rows": 9
      },
      {
        "path": "brain/evidence/cuad-public-contract-utility-next5-20261020.json",
        "artifact_sha256": "3db84d4125051830023738824e7e36d1333dfc25c2ffb5dd25dec4eaad6aeaf1",
        "suite_sha256": "f8729ba5bfbb155cce094581c7a670f0dfa824bd1160444b47953a3885dafb39",
        "verified_rows": 19
      },
      {
        "path": "brain/evidence/cuad-public-contract-utility-next5-20261020-v2.json",
        "artifact_sha256": "e320af3914387e9fd465676819801def823137d197aea30b6280e98d2fe8b019",
        "suite_sha256": "c3cfa2534b9967dbd7b04e6ee84bcc670af7b0792566cb29b246c0c08964e055",
        "verified_rows": 10
      },
      {
        "path": "brain/evidence/cuad-public-contract-utility-next5-20261020-llama.json",
        "artifact_sha256": "0c33a8ea5f2446855dc7c7fc7e4aa191edaa7b37e757bfe18aa47647d94fc364",
        "suite_sha256": "c3cfa2534b9967dbd7b04e6ee84bcc670af7b0792566cb29b246c0c08964e055",
        "verified_rows": 10
      }
    ],
    "all_valid_outputs_include_output_sha256": true,
    "run_verification": "independent_exact_holdout_oracle.v1"
  },
  "artifact_sha256": "e3272d28388e7968af5c2b7380b1d865c55345a2ac9cabcb571be5e5bc5d552c"
}