{
  "schema": "asn.public-local-fixture-study.v1",
  "study_id": "local-functional-fixtures-2026-10-20",
  "title": "Four local functional task fixtures",
  "evaluated_at": {
    "start": "2026-10-20T06:23:03.238Z",
    "end": "2026-10-20T06:23:03.238Z"
  },
  "source": {
    "kind": "operator_authored_json",
    "oracle_review_state": "mechanically_checked_operator_authored",
    "suite_id": "custom-local-generalization-20261019-v1",
    "suite_sha256": "1032d3c6f775e39500eba03f1d0b296fb430603a562ac0a6259104d5e3509326",
    "suite_file_sha256": "f6666255cc5583a105754fb48bbffed7f8176175df5b0b861840312309bbb200",
    "artifact_sha256": "47bb7f3bec88094e1d09b35954b35e71d9ed2d85f1584bc960f715a9b90cb3ad",
    "evidence_path": "/local-task-fixtures-evidence-20261020.json"
  },
  "protocol": {
    "task_count": 4,
    "task_classes": [
      "structured_extraction",
      "verified_arithmetic"
    ],
    "prompt_variants": [
      "canonical",
      "paraphrase"
    ],
    "repetitions_per_task_per_variant": 1,
    "attempts": 16,
    "request_timeout_ms": 60000,
    "max_runtime_ms": 900000,
    "grading": "Exact match of the complete required JSON object against the operator-authored expected values; each task also has explicit source-field or arithmetic checks.",
    "runtime": "Ollama 0.33.3 on loopback only; provider calls, spend and production writes were zero.",
    "raw_prompts_or_model_outputs_published": true
  },
  "models": [
    {
      "name": "qwen2.5:1.5b",
      "label": "Qwen 2.5 1.5B",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "parameter_size": "1.5B",
      "quantization": "Q4_K_M",
      "attempts": 8,
      "valid_outputs": 8,
      "exact_matches": 4,
      "exact_failures": 4,
      "execution_errors": 0,
      "exact_match_rate": 0.5,
      "tasks_passing_both_prompt_variants": 2,
      "mean_inference_latency_ms": 9019,
      "source_run_hashes": [
        {
          "task_id": "generalization-contract-fields-04.h1",
          "variant": "canonical",
          "state": "verified_pass",
          "output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007"
        },
        {
          "task_id": "generalization-contract-fields-04.h1",
          "variant": "paraphrase",
          "state": "verified_pass",
          "output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007"
        },
        {
          "task_id": "generalization-discount-total-02.h1",
          "variant": "canonical",
          "state": "verified_fail",
          "output_sha256": "2cbf0035bb36f6fc8e23abf35cabbcfe7924da3438ac22a9545e4c002a282418"
        },
        {
          "task_id": "generalization-discount-total-02.h1",
          "variant": "paraphrase",
          "state": "verified_fail",
          "output_sha256": "2cbf0035bb36f6fc8e23abf35cabbcfe7924da3438ac22a9545e4c002a282418"
        },
        {
          "task_id": "generalization-pack-total-01.h1",
          "variant": "canonical",
          "state": "verified_pass",
          "output_sha256": "074d6a36694e5521979e0ddddd9e8faefabb105b63d69d5749a5234d8abbd57b"
        },
        {
          "task_id": "generalization-pack-total-01.h1",
          "variant": "paraphrase",
          "state": "verified_pass",
          "output_sha256": "074d6a36694e5521979e0ddddd9e8faefabb105b63d69d5749a5234d8abbd57b"
        },
        {
          "task_id": "generalization-waybill-fields-03.h1",
          "variant": "canonical",
          "state": "verified_fail",
          "output_sha256": "091c551a896cc194637c317ebdc5c93935cd6c8b0c495d16d5ab12ff1f909634"
        },
        {
          "task_id": "generalization-waybill-fields-03.h1",
          "variant": "paraphrase",
          "state": "verified_fail",
          "output_sha256": "d9fc95d2528c8a9fcd1404b86a9268243d194a67ce8798732a9ea972573a37fb"
        }
      ]
    },
    {
      "name": "llama3.2:3b",
      "label": "Llama 3.2 3B",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "parameter_size": "3.2B",
      "quantization": "Q4_K_M",
      "attempts": 8,
      "valid_outputs": 8,
      "exact_matches": 3,
      "exact_failures": 5,
      "execution_errors": 0,
      "exact_match_rate": 0.375,
      "tasks_passing_both_prompt_variants": 1,
      "mean_inference_latency_ms": 16124,
      "source_run_hashes": [
        {
          "task_id": "generalization-contract-fields-04.h1",
          "variant": "canonical",
          "state": "verified_pass",
          "output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007"
        },
        {
          "task_id": "generalization-contract-fields-04.h1",
          "variant": "paraphrase",
          "state": "verified_pass",
          "output_sha256": "38008ffdbcef8f85bb98565e9ca1cbac45e0e713dc6a74e618d29b7180fa1007"
        },
        {
          "task_id": "generalization-discount-total-02.h1",
          "variant": "canonical",
          "state": "verified_fail",
          "output_sha256": "1c37260af3390e868e186f7df59c569e91397e73ea65ab450e72aea42f258eff"
        },
        {
          "task_id": "generalization-discount-total-02.h1",
          "variant": "paraphrase",
          "state": "verified_fail",
          "output_sha256": "e6fdd094c24e7deb1be57219cfaea8f6d0c1f9f6db3c71f995a6e2ddee162962"
        },
        {
          "task_id": "generalization-pack-total-01.h1",
          "variant": "canonical",
          "state": "verified_pass",
          "output_sha256": "074d6a36694e5521979e0ddddd9e8faefabb105b63d69d5749a5234d8abbd57b"
        },
        {
          "task_id": "generalization-pack-total-01.h1",
          "variant": "paraphrase",
          "state": "verified_fail",
          "output_sha256": "ded5d09d9550df4a1c74d26fa57127f07fff18c7a039232fefdcb419792f25f3"
        },
        {
          "task_id": "generalization-waybill-fields-03.h1",
          "variant": "canonical",
          "state": "verified_fail",
          "output_sha256": "091c551a896cc194637c317ebdc5c93935cd6c8b0c495d16d5ab12ff1f909634"
        },
        {
          "task_id": "generalization-waybill-fields-03.h1",
          "variant": "paraphrase",
          "state": "verified_fail",
          "output_sha256": "8a3b5d0f6915d773dfb61d9f59a80c0d9b721cc006f41ba2416bef1c3c499023"
        }
      ]
    }
  ],
  "tasks": [
    {
      "task_id": "generalization-pack-total-01.h1",
      "label": "Pack total · arithmetic",
      "task_class": "verified_arithmetic",
      "condition_family": "integer_unit_price_and_shipping",
      "models": [
        {
          "name": "qwen2.5:1.5b",
          "attempts": 2,
          "exact_matches": 2,
          "execution_errors": 0,
          "prompt_variants": [
            {
              "name": "canonical",
              "state": "verified_pass"
            },
            {
              "name": "paraphrase",
              "state": "verified_pass"
            }
          ]
        },
        {
          "name": "llama3.2:3b",
          "attempts": 2,
          "exact_matches": 1,
          "execution_errors": 0,
          "prompt_variants": [
            {
              "name": "canonical",
              "state": "verified_pass"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail"
            }
          ]
        }
      ]
    },
    {
      "task_id": "generalization-discount-total-02.h1",
      "label": "Discounted invoice · arithmetic",
      "task_class": "verified_arithmetic",
      "condition_family": "percentage_discount_then_additions",
      "models": [
        {
          "name": "qwen2.5:1.5b",
          "attempts": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "prompt_variants": [
            {
              "name": "canonical",
              "state": "verified_fail"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail"
            }
          ]
        },
        {
          "name": "llama3.2:3b",
          "attempts": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "prompt_variants": [
            {
              "name": "canonical",
              "state": "verified_fail"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail"
            }
          ]
        }
      ]
    },
    {
      "task_id": "generalization-waybill-fields-03.h1",
      "label": "Waybill fields · extraction",
      "task_class": "structured_extraction",
      "condition_family": "waybill_field_assignment",
      "models": [
        {
          "name": "qwen2.5:1.5b",
          "attempts": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "prompt_variants": [
            {
              "name": "canonical",
              "state": "verified_fail"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail"
            }
          ]
        },
        {
          "name": "llama3.2:3b",
          "attempts": 2,
          "exact_matches": 0,
          "execution_errors": 0,
          "prompt_variants": [
            {
              "name": "canonical",
              "state": "verified_fail"
            },
            {
              "name": "paraphrase",
              "state": "verified_fail"
            }
          ]
        }
      ]
    },
    {
      "task_id": "generalization-contract-fields-04.h1",
      "label": "Contract terms · extraction",
      "task_class": "structured_extraction",
      "condition_family": "contract_clause_field_assignment",
      "models": [
        {
          "name": "qwen2.5:1.5b",
          "attempts": 2,
          "exact_matches": 2,
          "execution_errors": 0,
          "prompt_variants": [
            {
              "name": "canonical",
              "state": "verified_pass"
            },
            {
              "name": "paraphrase",
              "state": "verified_pass"
            }
          ]
        },
        {
          "name": "llama3.2:3b",
          "attempts": 2,
          "exact_matches": 2,
          "execution_errors": 0,
          "prompt_variants": [
            {
              "name": "canonical",
              "state": "verified_pass"
            },
            {
              "name": "paraphrase",
              "state": "verified_pass"
            }
          ]
        }
      ]
    }
  ],
  "interpretation": {
    "public_ranking_eligible": false,
    "production_measurement_index_eligible": false,
    "general_model_capability_verified": false,
    "customer_utility_verified": false,
    "statement": "This descriptive comparison covers four operator-authored fixtures, two prompt phrasings per task and one run per phrasing. The paired prompt attempts are not independent examples; these results do not establish broad capability or a model ranking.",
    "limitations": [
      "Fixtures are authored by the operator and are not blinded third-party benchmarks.",
      "Exact-match grading rewards only the declared schema and oracle.",
      "Two prompt variants and limited repetitions do not establish general capability.",
      "Local energy, opportunity cost and external production behavior are unmeasured."
    ]
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "money_movement": false,
    "authority_changed": false,
    "independent_exact_verification": true
  },
  "receipt_sha256": "16d44ca3dd6388f7d2d62d59d421523d8cd25daf00c29b0156f833dc2c32a875"
}