{
  "schema": "asn.local-model-holdout.v1",
  "created_at": "2026-10-20T06:23:03.238Z",
  "suite_id": "custom-local-generalization-20261019-v1",
  "suite_sha256": "1032d3c6f775e39500eba03f1d0b296fb430603a562ac0a6259104d5e3509326",
  "suite_metadata": {
    "source_kind": "operator_authored_json",
    "oracle_method": "Operator-authored expected values are checked against explicit integer arithmetic and literal source fields. This small holdout tests measurement mechanics and two task classes; it is not a broad or blinded model benchmark.",
    "oracle_review_state": "mechanically_checked_operator_authored",
    "source_sha256": "f6666255cc5583a105754fb48bbffed7f8176175df5b0b861840312309bbb200"
  },
  "runtime": {
    "name": "ollama",
    "version": "0.33.3",
    "origin_class": "loopback_only",
    "origin": "http://127.0.0.1:11434",
    "host_platform": "win32",
    "host_arch": "x64",
    "request_timeout_ms": 60000,
    "max_runtime_ms": 900000
  },
  "models": [
    {
      "name": "qwen2.5:1.5b",
      "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "size_bytes": 986061892,
      "details": {
        "parent_model": "",
        "format": "gguf",
        "family": "qwen2",
        "families": [
          "qwen2"
        ],
        "parameter_size": "1.5B",
        "quantization_level": "Q4_K_M",
        "context_length": 32768,
        "embedding_length": 1536
      }
    },
    {
      "name": "llama3.2:3b",
      "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "size_bytes": 2019393189,
      "details": {
        "parent_model": "",
        "format": "gguf",
        "family": "llama",
        "families": [
          "llama"
        ],
        "parameter_size": "3.2B",
        "quantization_level": "Q4_K_M",
        "context_length": 131072,
        "embedding_length": 3072
      }
    }
  ],
  "repetitions_per_task_per_variant": 1,
  "prompt_variants": [
    "canonical",
    "paraphrase"
  ],
  "tasks": [
    {
      "id": "generalization-pack-total-01.h1",
      "task_class": "verified_arithmetic",
      "schema": {
        "type": "object",
        "properties": {
          "pack_id": {
            "type": "string"
          },
          "units": {
            "type": "integer"
          },
          "unit_price_cents": {
            "type": "integer"
          },
          "merchandise_total_cents": {
            "type": "integer"
          },
          "shipping_cents": {
            "type": "integer"
          },
          "amount_due_cents": {
            "type": "integer"
          }
        },
        "required": [
          "pack_id",
          "units",
          "unit_price_cents",
          "merchandise_total_cents",
          "shipping_cents",
          "amount_due_cents"
        ],
        "additionalProperties": false
      },
      "expected": {
        "pack_id": "PK-731",
        "units": 7,
        "unit_price_cents": 132,
        "merchandise_total_cents": 924,
        "shipping_cents": 245,
        "amount_due_cents": 1169
      },
      "prompt": "For pack PK-731, 7 units cost 132 cents each and shipping is 245 cents. Calculate merchandise and total due in integer cents. Return only the six required fields.",
      "paraphrase_prompt": "PK-731: quantity 7, unit price $1.32, delivery $2.45. Give the identifier, counts, all unit and extended prices in cents, and the final amount due as one exact JSON object.",
      "critical_fields": [
        "merchandise_total_cents",
        "amount_due_cents"
      ],
      "condition_family": "integer_unit_price_and_shipping",
      "definition_sha256": "52c6447e35a0ef56c292441db9de3533f18f73d5f09e59db6bb256da9d52a063"
    },
    {
      "id": "generalization-discount-total-02.h1",
      "task_class": "verified_arithmetic",
      "schema": {
        "type": "object",
        "properties": {
          "invoice_id": {
            "type": "string"
          },
          "gross_subtotal_cents": {
            "type": "integer"
          },
          "discount_percent": {
            "type": "integer"
          },
          "discount_cents": {
            "type": "integer"
          },
          "net_subtotal_cents": {
            "type": "integer"
          },
          "shipping_cents": {
            "type": "integer"
          },
          "tax_cents": {
            "type": "integer"
          },
          "total_cents": {
            "type": "integer"
          }
        },
        "required": [
          "invoice_id",
          "gross_subtotal_cents",
          "discount_percent",
          "discount_cents",
          "net_subtotal_cents",
          "shipping_cents",
          "tax_cents",
          "total_cents"
        ],
        "additionalProperties": false
      },
      "expected": {
        "invoice_id": "INV-2904",
        "gross_subtotal_cents": 4800,
        "discount_percent": 15,
        "discount_cents": 720,
        "net_subtotal_cents": 4080,
        "shipping_cents": 220,
        "tax_cents": 315,
        "total_cents": 4615
      },
      "prompt": "Invoice INV-2904 has a $48.00 subtotal. Apply a 15% discount, then add $2.20 shipping and $3.15 tax. Report every amount in integer cents and the final total. Return only the required JSON fields.",
      "paraphrase_prompt": "Compute INV-2904: merchandise $48.00, discount 15%, freight $2.20, tax $3.15. The discount is taken before freight and tax are added. Give the gross, discount, net, freight, tax, and final values in cents.",
      "critical_fields": [
        "discount_cents",
        "net_subtotal_cents",
        "total_cents"
      ],
      "condition_family": "percentage_discount_then_additions",
      "definition_sha256": "a16f6498503aff4c3eb30a0ae7a2fa773c4a248fd18db352c9bf4819a6a26f31"
    },
    {
      "id": "generalization-waybill-fields-03.h1",
      "task_class": "structured_extraction",
      "schema": {
        "type": "object",
        "properties": {
          "waybill_id": {
            "type": "string"
          },
          "origin_code": {
            "type": "string"
          },
          "destination_code": {
            "type": "string"
          },
          "ship_date": {
            "type": "string"
          },
          "service_level": {
            "type": "string"
          }
        },
        "required": [
          "waybill_id",
          "origin_code",
          "destination_code",
          "ship_date",
          "service_level"
        ],
        "additionalProperties": false
      },
      "expected": {
        "waybill_id": "WB-4418",
        "origin_code": "BOS",
        "destination_code": "MKE",
        "ship_date": "2026-10-24",
        "service_level": "two_day"
      },
      "prompt": "Read the carrier record and extract its fields exactly.\n\nWaybill WB-4418\nOrigin: BOS\nDestination: MKE\nShip date: 2026-10-24\nService: two day\n\nReturn one JSON object with the requested identifiers and labels.",
      "paraphrase_prompt": "For shipment WB-4418, provide the origin and destination airport codes, dispatch date and service class. The dispatch date is 24 October 2026 and service is two-day. Return the date as YYYY-MM-DD and use the schema fields.",
      "critical_fields": [
        "waybill_id",
        "origin_code",
        "destination_code"
      ],
      "condition_family": "waybill_field_assignment",
      "definition_sha256": "9babc3dc8fc3ee2a1840c1ed3765405ba470a379d4b9b8f62c35134466ddad73"
    },
    {
      "id": "generalization-contract-fields-04.h1",
      "task_class": "structured_extraction",
      "schema": {
        "type": "object",
        "properties": {
          "contract_id": {
            "type": "string"
          },
          "vendor": {
            "type": "string"
          },
          "term_months": {
            "type": "integer"
          },
          "auto_renews": {
            "type": "boolean"
          },
          "notice_days": {
            "type": "integer"
          }
        },
        "required": [
          "contract_id",
          "vendor",
          "term_months",
          "auto_renews",
          "notice_days"
        ],
        "additionalProperties": false
      },
      "expected": {
        "contract_id": "CTR-918",
        "vendor": "Northstar Data",
        "term_months": 18,
        "auto_renews": true,
        "notice_days": 45
      },
      "prompt": "Extract the contract facts.\n\nCTR-918 with Northstar Data has an initial term of eighteen months. It renews automatically for successive eighteen-month terms unless either party gives written notice at least forty-five days before the current term ends.\n\nReturn only the five requested fields.",
      "paraphrase_prompt": "Summarize CTR-918: supplier Northstar Data; term 18 months; automatic renewal applies; cancellation notice must arrive 45 days before expiration. Return the contract ID, vendor, term, renewal flag and notice period.",
      "critical_fields": [
        "contract_id",
        "auto_renews",
        "notice_days"
      ],
      "condition_family": "contract_clause_field_assignment",
      "definition_sha256": "2b3c49cd3f7d4e51aafa0dad10f5f2c318b76ad1af07fc02bc44b68a6990b21b"
    }
  ],
  "runs": [
    {
      "model": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "task_id": "generalization-pack-total-01.h1",
      "task_class": "verified_arithmetic",
      "variant": "canonical",
      "replicate": 1,
      "run_sequence": 0,
      "model_order_position": 0,
      "state": "verified_pass",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "exact_match",
      "output": {
        "pack_id": "PK-731",
        "units": 7,
        "unit_price_cents": 132,
        "merchandise_total_cents": 924,
        "shipping_cents": 245,
        "amount_due_cents": 1169
      },
      "output_sha256": "074d6a36694e5521979e0ddddd9e8faefabb105b63d69d5749a5234d8abbd57b",
      "inference_latency_ms": 8053.913,
      "reported_timings_ms": {
        "prompt_eval_duration": 997.252,
        "eval_duration": 7056.661,
        "total_duration": 11673.885
      },
      "elapsed_ms": 11679
    },
    {
      "model": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "task_id": "generalization-pack-total-01.h1",
      "task_class": "verified_arithmetic",
      "variant": "canonical",
      "replicate": 1,
      "run_sequence": 1,
      "model_order_position": 1,
      "state": "verified_pass",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "exact_match",
      "output": {
        "pack_id": "PK-731",
        "units": 7,
        "unit_price_cents": 132,
        "merchandise_total_cents": 924,
        "shipping_cents": 245,
        "amount_due_cents": 1169
      },
      "output_sha256": "074d6a36694e5521979e0ddddd9e8faefabb105b63d69d5749a5234d8abbd57b",
      "inference_latency_ms": 16132.914,
      "reported_timings_ms": {
        "prompt_eval_duration": 2896.662,
        "eval_duration": 13236.252,
        "total_duration": 23605.187
      },
      "elapsed_ms": 23609
    },
    {
      "model": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "task_id": "generalization-discount-total-02.h1",
      "task_class": "verified_arithmetic",
      "variant": "canonical",
      "replicate": 1,
      "run_sequence": 2,
      "model_order_position": 0,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "invoice_id": "INV-2904",
        "gross_subtotal_cents": 4800,
        "discount_percent": 15,
        "discount_cents": 720,
        "net_subtotal_cents": 4080,
        "shipping_cents": 220,
        "tax_cents": 315,
        "total_cents": 6615
      },
      "output_sha256": "1c37260af3390e868e186f7df59c569e91397e73ea65ab450e72aea42f258eff",
      "inference_latency_ms": 21221.022,
      "reported_timings_ms": {
        "prompt_eval_duration": 2712.493,
        "eval_duration": 18508.529,
        "total_duration": 21251.043
      },
      "elapsed_ms": 21256
    },
    {
      "model": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "task_id": "generalization-discount-total-02.h1",
      "task_class": "verified_arithmetic",
      "variant": "canonical",
      "replicate": 1,
      "run_sequence": 3,
      "model_order_position": 1,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "invoice_id": "INV-2904",
        "gross_subtotal_cents": 4800,
        "discount_percent": 15,
        "discount_cents": 720,
        "net_subtotal_cents": 4080,
        "shipping_cents": 220,
        "tax_cents": 315,
        "total_cents": 4625
      },
      "output_sha256": "2cbf0035bb36f6fc8e23abf35cabbcfe7924da3438ac22a9545e4c002a282418",
      "inference_latency_ms": 12734.282,
      "reported_timings_ms": {
        "prompt_eval_duration": 2262.801,
        "eval_duration": 10471.481,
        "total_duration": 16926.66
      },
      "elapsed_ms": 16931
    },
    {
      "model": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "task_id": "generalization-waybill-fields-03.h1",
      "task_class": "structured_extraction",
      "variant": "canonical",
      "replicate": 1,
      "run_sequence": 4,
      "model_order_position": 0,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "waybill_id": "WB-4418",
        "origin_code": "BOS",
        "destination_code": "MKE",
        "ship_date": "2026-10-24",
        "service_level": "two day"
      },
      "output_sha256": "091c551a896cc194637c317ebdc5c93935cd6c8b0c495d16d5ab12ff1f909634",
      "inference_latency_ms": 7724.371,
      "reported_timings_ms": {
        "prompt_eval_duration": 1429.734,
        "eval_duration": 6294.637,
        "total_duration": 7741.584
      },
      "elapsed_ms": 7745
    },
    {
      "model": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "task_id": "generalization-waybill-fields-03.h1",
      "task_class": "structured_extraction",
      "variant": "canonical",
      "replicate": 1,
      "run_sequence": 5,
      "model_order_position": 1,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "waybill_id": "WB-4418",
        "origin_code": "BOS",
        "destination_code": "MKE",
        "ship_date": "2026-10-24",
        "service_level": "two day"
      },
      "output_sha256": "091c551a896cc194637c317ebdc5c93935cd6c8b0c495d16d5ab12ff1f909634",
      "inference_latency_ms": 13551.85,
      "reported_timings_ms": {
        "prompt_eval_duration": 2709.239,
        "eval_duration": 10842.611,
        "total_duration": 13577.729
      },
      "elapsed_ms": 13581
    },
    {
      "model": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "task_id": "generalization-contract-fields-04.h1",
      "task_class": "structured_extraction",
      "variant": "canonical",
      "replicate": 1,
      "run_sequence": 6,
      "model_order_position": 0,
      "state": "verified_pass",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "exact_match",
      "output": {
        "contract_id": "CTR-918",
        "vendor": "Northstar Data",
        "term_months": 18,
        "auto_renews": true,
        "notice_days": 45
      },
      "output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007",
      "inference_latency_ms": 12412.365,
      "reported_timings_ms": {
        "prompt_eval_duration": 2502.535,
        "eval_duration": 9909.83,
        "total_duration": 12435.376
      },
      "elapsed_ms": 12441
    },
    {
      "model": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "task_id": "generalization-contract-fields-04.h1",
      "task_class": "structured_extraction",
      "variant": "canonical",
      "replicate": 1,
      "run_sequence": 7,
      "model_order_position": 1,
      "state": "verified_pass",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "exact_match",
      "output": {
        "contract_id": "CTR-918",
        "vendor": "Northstar Data",
        "term_months": 18,
        "auto_renews": true,
        "notice_days": 45
      },
      "output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007",
      "inference_latency_ms": 6828.653,
      "reported_timings_ms": {
        "prompt_eval_duration": 1261.65,
        "eval_duration": 5567.003,
        "total_duration": 6843.559
      },
      "elapsed_ms": 6850
    },
    {
      "model": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "task_id": "generalization-pack-total-01.h1",
      "task_class": "verified_arithmetic",
      "variant": "paraphrase",
      "replicate": 1,
      "run_sequence": 8,
      "model_order_position": 0,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "pack_id": "PK-731",
        "units": 7,
        "unit_price_cents": 132,
        "merchandise_total_cents": 132,
        "shipping_cents": 245,
        "amount_due_cents": 377
      },
      "output_sha256": "ded5d09d9550df4a1c74d26fa57127f07fff18c7a039232fefdcb419792f25f3",
      "inference_latency_ms": 15851.727,
      "reported_timings_ms": {
        "prompt_eval_duration": 2312.731,
        "eval_duration": 13538.996,
        "total_duration": 15878.216
      },
      "elapsed_ms": 15882
    },
    {
      "model": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "task_id": "generalization-pack-total-01.h1",
      "task_class": "verified_arithmetic",
      "variant": "paraphrase",
      "replicate": 1,
      "run_sequence": 9,
      "model_order_position": 1,
      "state": "verified_pass",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "exact_match",
      "output": {
        "pack_id": "PK-731",
        "units": 7,
        "unit_price_cents": 132,
        "merchandise_total_cents": 924,
        "shipping_cents": 245,
        "amount_due_cents": 1169
      },
      "output_sha256": "074d6a36694e5521979e0ddddd9e8faefabb105b63d69d5749a5234d8abbd57b",
      "inference_latency_ms": 9362.194,
      "reported_timings_ms": {
        "prompt_eval_duration": 1233.642,
        "eval_duration": 8128.552,
        "total_duration": 9376.613
      },
      "elapsed_ms": 9380
    },
    {
      "model": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "task_id": "generalization-discount-total-02.h1",
      "task_class": "verified_arithmetic",
      "variant": "paraphrase",
      "replicate": 1,
      "run_sequence": 10,
      "model_order_position": 0,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "invoice_id": "INV-2904",
        "gross_subtotal_cents": 4800,
        "discount_percent": 15,
        "discount_cents": 720,
        "net_subtotal_cents": 4080,
        "shipping_cents": 220,
        "tax_cents": 315,
        "total_cents": 4625
      },
      "output_sha256": "2cbf0035bb36f6fc8e23abf35cabbcfe7924da3438ac22a9545e4c002a282418",
      "inference_latency_ms": 12663.747,
      "reported_timings_ms": {
        "prompt_eval_duration": 1446.69,
        "eval_duration": 11217.057,
        "total_duration": 12680.583
      },
      "elapsed_ms": 12684
    },
    {
      "model": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "task_id": "generalization-discount-total-02.h1",
      "task_class": "verified_arithmetic",
      "variant": "paraphrase",
      "replicate": 1,
      "run_sequence": 11,
      "model_order_position": 1,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "invoice_id": "INV-2904",
        "gross_subtotal_cents": 4800,
        "discount_percent": 15,
        "discount_cents": 720,
        "net_subtotal_cents": 4080,
        "shipping_cents": 220,
        "tax_cents": 315,
        "total_cents": 4715
      },
      "output_sha256": "e6fdd094c24e7deb1be57219cfaea8f6d0c1f9f6db3c71f995a6e2ddee162962",
      "inference_latency_ms": 22624.586,
      "reported_timings_ms": {
        "prompt_eval_duration": 2763.322,
        "eval_duration": 19861.264,
        "total_duration": 22644.069
      },
      "elapsed_ms": 22646
    },
    {
      "model": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "task_id": "generalization-waybill-fields-03.h1",
      "task_class": "structured_extraction",
      "variant": "paraphrase",
      "replicate": 1,
      "run_sequence": 12,
      "model_order_position": 0,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "waybill_id": "WB-4418",
        "origin_code": "KJFK",
        "destination_code": "LAX",
        "ship_date": "2026-10-24",
        "service_level": "2-Day"
      },
      "output_sha256": "8a3b5d0f6915d773dfb61d9f59a80c0d9b721cc006f41ba2416bef1c3c499023",
      "inference_latency_ms": 15614.288,
      "reported_timings_ms": {
        "prompt_eval_duration": 2860.46,
        "eval_duration": 12753.828,
        "total_duration": 15653.97
      },
      "elapsed_ms": 15658
    },
    {
      "model": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "task_id": "generalization-waybill-fields-03.h1",
      "task_class": "structured_extraction",
      "variant": "paraphrase",
      "replicate": 1,
      "run_sequence": 13,
      "model_order_position": 1,
      "state": "verified_fail",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "value_mismatch",
      "output": {
        "waybill_id": "WB-4418",
        "origin_code": "XXX",
        "destination_code": "YYY",
        "ship_date": "2026-10-24",
        "service_level": "2-day"
      },
      "output_sha256": "d9fc95d2528c8a9fcd1404b86a9268243d194a67ce8798732a9ea972573a37fb",
      "inference_latency_ms": 8405.49,
      "reported_timings_ms": {
        "prompt_eval_duration": 1940.982,
        "eval_duration": 6464.508,
        "total_duration": 11328.596
      },
      "elapsed_ms": 11330
    },
    {
      "model": "qwen2.5:1.5b",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "task_id": "generalization-contract-fields-04.h1",
      "task_class": "structured_extraction",
      "variant": "paraphrase",
      "replicate": 1,
      "run_sequence": 14,
      "model_order_position": 0,
      "state": "verified_pass",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "exact_match",
      "output": {
        "contract_id": "CTR-918",
        "vendor": "Northstar Data",
        "term_months": 18,
        "auto_renews": true,
        "notice_days": 45
      },
      "output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007",
      "inference_latency_ms": 6375.966,
      "reported_timings_ms": {
        "prompt_eval_duration": 1082.037,
        "eval_duration": 5293.929,
        "total_duration": 6391.395
      },
      "elapsed_ms": 6393
    },
    {
      "model": "llama3.2:3b",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "task_id": "generalization-contract-fields-04.h1",
      "task_class": "structured_extraction",
      "variant": "paraphrase",
      "replicate": 1,
      "run_sequence": 15,
      "model_order_position": 1,
      "state": "verified_pass",
      "verification_method": "independent_exact_holdout_oracle.v1",
      "verification_reason": "exact_match",
      "output": {
        "contract_id": "CTR-918",
        "vendor": "Northstar Data",
        "term_months": 18,
        "auto_renews": true,
        "notice_days": 45
      },
      "output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007",
      "inference_latency_ms": 11579.699,
      "reported_timings_ms": {
        "prompt_eval_duration": 2094.816,
        "eval_duration": 9484.883,
        "total_duration": 11602.851
      },
      "elapsed_ms": 11605
    }
  ],
  "summary": {
    "by_model": {
      "qwen2.5:1.5b": {
        "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
        "runs": 8,
        "verified_passes": 4,
        "accuracy": 0.5,
        "execution_errors": 0
      },
      "llama3.2:3b": {
        "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
        "runs": 8,
        "verified_passes": 3,
        "accuracy": 0.375,
        "execution_errors": 0
      }
    },
    "tasks": [
      {
        "task_id": "generalization-pack-total-01.h1",
        "task_class": "verified_arithmetic",
        "models": {
          "qwen2.5:1.5b": {
            "paired_repetitions": 1,
            "complete_pairs": 1,
            "canonical_passes": 1,
            "paraphrase_passes": 1,
            "paired_verdict_flips": 0,
            "execution_error_pairs": 0,
            "strict_holdout_candidate": true,
            "mean_inference_latency_ms": 8708.053,
            "consistent_output_sha256": "074d6a36694e5521979e0ddddd9e8faefabb105b63d69d5749a5234d8abbd57b"
          },
          "llama3.2:3b": {
            "paired_repetitions": 1,
            "complete_pairs": 1,
            "canonical_passes": 1,
            "paraphrase_passes": 0,
            "paired_verdict_flips": 1,
            "execution_error_pairs": 0,
            "strict_holdout_candidate": false,
            "mean_inference_latency_ms": 15992.321,
            "consistent_output_sha256": null
          }
        }
      },
      {
        "task_id": "generalization-discount-total-02.h1",
        "task_class": "verified_arithmetic",
        "models": {
          "qwen2.5:1.5b": {
            "paired_repetitions": 1,
            "complete_pairs": 1,
            "canonical_passes": 0,
            "paraphrase_passes": 0,
            "paired_verdict_flips": 0,
            "execution_error_pairs": 0,
            "strict_holdout_candidate": false,
            "mean_inference_latency_ms": 12699.014,
            "consistent_output_sha256": null
          },
          "llama3.2:3b": {
            "paired_repetitions": 1,
            "complete_pairs": 1,
            "canonical_passes": 0,
            "paraphrase_passes": 0,
            "paired_verdict_flips": 0,
            "execution_error_pairs": 0,
            "strict_holdout_candidate": false,
            "mean_inference_latency_ms": 21922.804,
            "consistent_output_sha256": null
          }
        }
      },
      {
        "task_id": "generalization-waybill-fields-03.h1",
        "task_class": "structured_extraction",
        "models": {
          "qwen2.5:1.5b": {
            "paired_repetitions": 1,
            "complete_pairs": 1,
            "canonical_passes": 0,
            "paraphrase_passes": 0,
            "paired_verdict_flips": 0,
            "execution_error_pairs": 0,
            "strict_holdout_candidate": false,
            "mean_inference_latency_ms": 8064.931,
            "consistent_output_sha256": null
          },
          "llama3.2:3b": {
            "paired_repetitions": 1,
            "complete_pairs": 1,
            "canonical_passes": 0,
            "paraphrase_passes": 0,
            "paired_verdict_flips": 0,
            "execution_error_pairs": 0,
            "strict_holdout_candidate": false,
            "mean_inference_latency_ms": 14583.069,
            "consistent_output_sha256": null
          }
        }
      },
      {
        "task_id": "generalization-contract-fields-04.h1",
        "task_class": "structured_extraction",
        "models": {
          "qwen2.5:1.5b": {
            "paired_repetitions": 1,
            "complete_pairs": 1,
            "canonical_passes": 1,
            "paraphrase_passes": 1,
            "paired_verdict_flips": 0,
            "execution_error_pairs": 0,
            "strict_holdout_candidate": true,
            "mean_inference_latency_ms": 6602.31,
            "consistent_output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007"
          },
          "llama3.2:3b": {
            "paired_repetitions": 1,
            "complete_pairs": 1,
            "canonical_passes": 1,
            "paraphrase_passes": 1,
            "paired_verdict_flips": 0,
            "execution_error_pairs": 0,
            "strict_holdout_candidate": true,
            "mean_inference_latency_ms": 11996.032,
            "consistent_output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007"
          }
        }
      }
    ]
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "money_movement": false,
    "authority_changed": false,
    "independent_exact_verification": true,
    "holdout_is_separate_from_development_suite": true,
    "ranking_scope": "4 exact held-out task definitions, two specified prompt variants, repetitions and listed local model digests only.",
    "limitations": [
      "Fixtures are authored by the operator and are not blinded third-party benchmarks.",
      "Exact-match grading rewards only the declared schema and oracle.",
      "Two prompt variants and limited repetitions do not establish general capability.",
      "Local energy, opportunity cost and external production behavior are unmeasured."
    ]
  },
  "artifact_sha256": "47bb7f3bec88094e1d09b35954b35e71d9ed2d85f1584bc960f715a9b90cb3ad"
}