{
  "schema": "asn.public-ifeval-local-pilot.v1",
  "study_id": "ifeval-five-instruction-local-feasibility-2026-10-20",
  "title": "Five instruction checks: a local evaluation feasibility pilot",
  "evaluated_at": {
    "baseline": "2026-10-20T15:02:52.905Z",
    "timeout_replication": "2026-10-20T16:16:57.612Z",
    "generation_cap_replication": "2026-10-20T16:32:31.023Z",
    "residency_replication": "2026-10-20T16:49:26.596Z"
  },
  "source": {
    "name": "IFEval · Instruction Following Evaluation",
    "dataset_url": "https://huggingface.co/datasets/google/IFEval",
    "data_url": "https://huggingface.co/datasets/google/IFEval/blob/966cd89545d6b6acfd7638bc708b98261ca58e84/ifeval_input_data.jsonl",
    "dataset_revision": "966cd89545d6b6acfd7638bc708b98261ca58e84",
    "dataset_sha256": "6a85310ca8ce15eff755aa08a3a4ff931c7e273e7515ebb3c492ea85fd8288f2",
    "license": "Apache-2.0",
    "task": "text-generation instruction following",
    "language": "English"
  },
  "protocol": {
    "suite_id": "asn.ifeval-bounded-pilot.v1",
    "baseline_suite_sha256": "7c6ffd82764c9ded2d6e6bc48faa310f4e3d9599b9fbae526ecaf6879048182d",
    "timeout_replication_suite_sha256": "810d6950b3f4d3e16667f13a817ed20f9d92d414d58e8703cbe3dbcfb8fb3962",
    "generation_cap_suite_sha256": "53542254d78ecb6823b4c5b9ab4a4bf2f9eccc2f8b4394d8a33055e6f11f0d92",
    "residency_suite_sha256": "cca99da902e2ab0ee7e92a085c2b92cb061c3b36f546d0bdc0d9f096fb65cc21",
    "task_count": 5,
    "task_ids": [
      1001,
      102,
      1075,
      122,
      2567
    ],
    "instruction_types": [
      "punctuation:no_comma",
      "detectable_format:number_bullet_lists",
      "detectable_format:json_format",
      "startend:quotation",
      "keywords:existence"
    ],
    "attempts_per_model_per_run": 5,
    "prompt_variants": 1,
    "repetitions_per_task": 1,
    "verifier": {
      "name": "Google Research · Instruction Following Evaluation",
      "source_revision": "e6890f85757dd84e27ca6df2dd30651dafad28e0",
      "semantics": "five named deterministic checks only; no model judge; not the full benchmark"
    },
    "runtime": {
      "name": "ollama",
      "version": "0.33.3",
      "origin_class": "loopback_only",
      "total_runtime_cap_ms": 900000,
      "model_digests": [
        {
          "name": "qwen2.5:1.5b",
          "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b"
        },
        {
          "name": "llama3.2:3b",
          "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72"
        }
      ]
    },
    "execution_profiles": [
      {
        "request_timeout_ms": 60000,
        "max_output_tokens": 768,
        "keep_alive": 0
      },
      {
        "request_timeout_ms": 80000,
        "max_output_tokens": 768,
        "keep_alive": 0
      },
      {
        "request_timeout_ms": 80000,
        "max_output_tokens": 384,
        "keep_alive": 0
      },
      {
        "request_timeout_ms": 80000,
        "max_output_tokens": 384,
        "keep_alive": "5m"
      }
    ]
  },
  "timeout_comparison": [
    {
      "name": "qwen2.5:1.5b",
      "baseline": {
        "timeout_ms": 60000,
        "attempts": 5,
        "verified_passes": 2,
        "verified_failures": 1,
        "execution_errors": 2,
        "valid_outputs": 3
      },
      "replication": {
        "timeout_ms": 80000,
        "attempts": 5,
        "verified_passes": 2,
        "verified_failures": 1,
        "execution_errors": 2,
        "valid_outputs": 3
      },
      "task_outcome_states_unchanged": true
    },
    {
      "name": "llama3.2:3b",
      "baseline": {
        "timeout_ms": 60000,
        "attempts": 5,
        "verified_passes": 0,
        "verified_failures": 0,
        "execution_errors": 5,
        "valid_outputs": 0
      },
      "replication": {
        "timeout_ms": 80000,
        "attempts": 5,
        "verified_passes": 0,
        "verified_failures": 0,
        "execution_errors": 5,
        "valid_outputs": 0
      },
      "task_outcome_states_unchanged": true
    }
  ],
  "generation_cap_comparison": [
    {
      "name": "qwen2.5:1.5b",
      "timeout80_num_predict768": {
        "verified_passes": 2,
        "verified_failures": 1,
        "execution_errors": 2,
        "valid_outputs": 3
      },
      "timeout80_num_predict384": {
        "verified_passes": 2,
        "verified_failures": 3,
        "execution_errors": 0,
        "valid_outputs": 5
      },
      "task_results": [
        {
          "source_task_id": 1001,
          "instruction_id": "punctuation:no_comma",
          "state": "verified_fail"
        },
        {
          "source_task_id": 102,
          "instruction_id": "detectable_format:number_bullet_lists",
          "state": "verified_pass"
        },
        {
          "source_task_id": 1075,
          "instruction_id": "detectable_format:json_format",
          "state": "verified_pass"
        },
        {
          "source_task_id": 122,
          "instruction_id": "startend:quotation",
          "state": "verified_fail"
        },
        {
          "source_task_id": 2567,
          "instruction_id": "keywords:existence",
          "state": "verified_fail"
        }
      ]
    },
    {
      "name": "llama3.2:3b",
      "timeout80_num_predict768": {
        "verified_passes": 0,
        "verified_failures": 0,
        "execution_errors": 5,
        "valid_outputs": 0
      },
      "timeout80_num_predict384": {
        "verified_passes": 0,
        "verified_failures": 0,
        "execution_errors": 5,
        "valid_outputs": 0
      },
      "task_results": [
        {
          "source_task_id": 1001,
          "instruction_id": "punctuation:no_comma",
          "state": "execution_error"
        },
        {
          "source_task_id": 102,
          "instruction_id": "detectable_format:number_bullet_lists",
          "state": "execution_error"
        },
        {
          "source_task_id": 1075,
          "instruction_id": "detectable_format:json_format",
          "state": "execution_error"
        },
        {
          "source_task_id": 122,
          "instruction_id": "startend:quotation",
          "state": "execution_error"
        },
        {
          "source_task_id": 2567,
          "instruction_id": "keywords:existence",
          "state": "execution_error"
        }
      ]
    }
  ],
  "residency_comparison": [
    {
      "name": "qwen2.5:1.5b",
      "keep_alive_0": {
        "verified_passes": 2,
        "verified_failures": 3,
        "execution_errors": 0,
        "task_results": [
          {
            "source_task_id": 1001,
            "instruction_id": "punctuation:no_comma",
            "state": "verified_fail"
          },
          {
            "source_task_id": 102,
            "instruction_id": "detectable_format:number_bullet_lists",
            "state": "verified_pass"
          },
          {
            "source_task_id": 1075,
            "instruction_id": "detectable_format:json_format",
            "state": "verified_pass"
          },
          {
            "source_task_id": 122,
            "instruction_id": "startend:quotation",
            "state": "verified_fail"
          },
          {
            "source_task_id": 2567,
            "instruction_id": "keywords:existence",
            "state": "verified_fail"
          }
        ]
      },
      "keep_alive_5m": {
        "verified_passes": 3,
        "verified_failures": 2,
        "execution_errors": 0,
        "task_results": [
          {
            "source_task_id": 1001,
            "instruction_id": "punctuation:no_comma",
            "state": "verified_fail"
          },
          {
            "source_task_id": 102,
            "instruction_id": "detectable_format:number_bullet_lists",
            "state": "verified_pass"
          },
          {
            "source_task_id": 1075,
            "instruction_id": "detectable_format:json_format",
            "state": "verified_pass"
          },
          {
            "source_task_id": 122,
            "instruction_id": "startend:quotation",
            "state": "verified_fail"
          },
          {
            "source_task_id": 2567,
            "instruction_id": "keywords:existence",
            "state": "verified_pass"
          }
        ]
      },
      "changed_task_outcomes": 1
    },
    {
      "name": "llama3.2:3b",
      "keep_alive_0": {
        "verified_passes": 0,
        "verified_failures": 0,
        "execution_errors": 5,
        "task_results": [
          {
            "source_task_id": 1001,
            "instruction_id": "punctuation:no_comma",
            "state": "execution_error"
          },
          {
            "source_task_id": 102,
            "instruction_id": "detectable_format:number_bullet_lists",
            "state": "execution_error"
          },
          {
            "source_task_id": 1075,
            "instruction_id": "detectable_format:json_format",
            "state": "execution_error"
          },
          {
            "source_task_id": 122,
            "instruction_id": "startend:quotation",
            "state": "execution_error"
          },
          {
            "source_task_id": 2567,
            "instruction_id": "keywords:existence",
            "state": "execution_error"
          }
        ]
      },
      "keep_alive_5m": {
        "verified_passes": 0,
        "verified_failures": 0,
        "execution_errors": 5,
        "task_results": [
          {
            "source_task_id": 1001,
            "instruction_id": "punctuation:no_comma",
            "state": "execution_error"
          },
          {
            "source_task_id": 102,
            "instruction_id": "detectable_format:number_bullet_lists",
            "state": "execution_error"
          },
          {
            "source_task_id": 1075,
            "instruction_id": "detectable_format:json_format",
            "state": "execution_error"
          },
          {
            "source_task_id": 122,
            "instruction_id": "startend:quotation",
            "state": "execution_error"
          },
          {
            "source_task_id": 2567,
            "instruction_id": "keywords:existence",
            "state": "execution_error"
          }
        ]
      },
      "changed_task_outcomes": 0
    }
  ],
  "models": [
    {
      "name": "qwen2.5:1.5b",
      "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "attempts": 5,
      "verified_passes": 3,
      "verified_failures": 2,
      "execution_errors": 0,
      "valid_outputs": 5,
      "pass_rate_valid_outputs": 0.6,
      "wilson_95_valid_outputs": {
        "confidence_level": 0.95,
        "method": "wilson_score_interval",
        "lower": 0.2307,
        "upper": 0.8824
      },
      "task_results": [
        {
          "source_task_id": 1001,
          "instruction_id": "punctuation:no_comma",
          "state": "verified_fail"
        },
        {
          "source_task_id": 102,
          "instruction_id": "detectable_format:number_bullet_lists",
          "state": "verified_pass"
        },
        {
          "source_task_id": 1075,
          "instruction_id": "detectable_format:json_format",
          "state": "verified_pass"
        },
        {
          "source_task_id": 122,
          "instruction_id": "startend:quotation",
          "state": "verified_fail"
        },
        {
          "source_task_id": 2567,
          "instruction_id": "keywords:existence",
          "state": "verified_pass"
        }
      ]
    },
    {
      "name": "llama3.2:3b",
      "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "attempts": 5,
      "verified_passes": 0,
      "verified_failures": 0,
      "execution_errors": 5,
      "valid_outputs": 0,
      "pass_rate_valid_outputs": null,
      "wilson_95_valid_outputs": null,
      "task_results": [
        {
          "source_task_id": 1001,
          "instruction_id": "punctuation:no_comma",
          "state": "execution_error"
        },
        {
          "source_task_id": 102,
          "instruction_id": "detectable_format:number_bullet_lists",
          "state": "execution_error"
        },
        {
          "source_task_id": 1075,
          "instruction_id": "detectable_format:json_format",
          "state": "execution_error"
        },
        {
          "source_task_id": 122,
          "instruction_id": "startend:quotation",
          "state": "execution_error"
        },
        {
          "source_task_id": 2567,
          "instruction_id": "keywords:existence",
          "state": "execution_error"
        }
      ]
    }
  ],
  "interpretation": {
    "comparison_state": "not_comparable_due_to_execution_errors",
    "timeout_extension_effect": "outcome_states_unchanged_between_60s_and_80s",
    "qwen_generation_cap_effect": "all_five_outputs_returned_at_384_tokens_with_two_length_capped_verified_failures",
    "llama_generation_cap_effect": "five_execution_errors_persisted_at_384_tokens_and_five_minutes_residency",
    "residency_effect": "single_replication_changed_one_qwen_task_state_without_reducing_llama_timeouts; insufficient_to_infer_stable_improvement",
    "public_ranking_eligible": false,
    "production_measurement_index_eligible": false,
    "general_capability_verified": false,
    "statement": "This feasibility pilot cannot support a model comparison. At 384 tokens with five-minute model residency, Qwen returned five checkable outputs (three passes and two failures) versus two passes and three failures in the single otherwise-matched keep_alive:0 run; one task outcome changed. This one replication is insufficient to infer a stable performance improvement. Llama timed out on all five tasks in both runs. The comparison remains local and descriptive."
  },
  "limitations": [
    "Five prompts and one response per task per run do not represent the full IFEval benchmark or general instruction following.",
    "The verifier reimplements five named deterministic checks only; it is not the complete official evaluator and uses no model judge.",
    "The source benchmark is public; possible training exposure is unknown.",
    "Execution errors are included in attempted counts and are not treated as failed verified outputs.",
    "Local Ollama results do not establish hosted API latency, availability, cost, reliability, user value or production behavior.",
    "Raw prompts and model outputs are not included in this public receipt."
  ],
  "integrity": {
    "source_suite_file_sha256": "e5f77df63a2c875d82a57c7824c03c5f71c02164cb21cb2d142f501cce07a8c5",
    "source_run_artifact_sha256": "c1d9f76ec2f3bdf37f1e8bedb7e19a78815968eee6ffc4b9e679d472fed9aff4",
    "timeout_replication_suite_file_sha256": "e3f0f84be8482a3e5e2bd643faa545be508d404a4de16d0d5c22d0637b50617e",
    "timeout_replication_run_artifact_sha256": "4638b536284b6702b62c51837e29a92d2f30e6861b78342f9db4f7fb757a844b",
    "generation_cap_suite_file_sha256": "2f6988a0f868a1a1c71d98cca6f21468cca975075732ae0006220ed8f55c0c7a",
    "generation_cap_run_artifact_sha256": "a110a77ff5e6068c4d8d683e9142edf5ff60743321c12d4d7aca1fc38becf817",
    "residency_suite_file_sha256": "11f0c1b52bd843c353ffa0bde899d363f75b5886b5b20ad415bc3da3b142f34b",
    "residency_run_artifact_sha256": "5c13a5e815cb65688fc9112fc2f1951c0c9d04ca7178de402cc524b563bb925a",
    "raw_prompts_outputs_and_error_details_published": false
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "external_spend_microusd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "public_ranking_eligible": false,
    "authority_changed": false
  },
  "receipt_sha256": "72bd116cf97040c1ee4077a2d715429b4fef9f204d5d43c6983c10dd981dccfc"
}