{
  "schema": "asn.ifeval-supported-rules-followup-receipt.v1",
  "published_summary_status": "prepared_for_operator_review",
  "created_at": "2026-10-28T18:47:31.790Z",
  "study": {
    "id": "asn-ifeval-supported-rules-followup-10.v1",
    "title": "IFEval supported-rules follow-up",
    "task_count": 10,
    "source_task_ids": [
      2195,
      2239,
      3540,
      1094,
      1658,
      2691,
      2324,
      1162,
      1187,
      1286
    ],
    "selection": {
      "method": "From source tasks containing only supported deterministic rules, select a SHA-256-ordered coverage set that covers every supported rule, then fill to ten by the same order. Exclude every task ID in the initial five-task pilot and prior 30-task holdout; no output-dependent filtering.",
      "namespace": "asn.ifeval-supported-rules-followup-10.v1",
      "excluded_prior_task_ids": [
        13,
        102,
        122,
        321,
        1001,
        1075,
        1148,
        1531,
        1776,
        1934,
        2010,
        2015,
        2078,
        2245,
        2374,
        2404,
        2417,
        2457,
        2485,
        2532,
        2567,
        2602,
        2662,
        2798,
        2801,
        2832,
        2857,
        2985,
        3025,
        3223,
        3323,
        3439,
        3536,
        3631,
        3732
      ],
      "eligible_disjoint_population": 32,
      "eligible_population_by_rule": {
        "punctuation:no_comma": 12,
        "detectable_format:number_bullet_lists": 9,
        "detectable_format:json_format": 3,
        "startend:quotation": 10,
        "keywords:existence": 6
      },
      "selected_task_ids": [
        2195,
        2239,
        3540,
        1094,
        1658,
        2691,
        2324,
        1162,
        1187,
        1286
      ],
      "selected_rule_counts": {
        "punctuation:no_comma": 4,
        "detectable_format:number_bullet_lists": 2,
        "detectable_format:json_format": 1,
        "startend:quotation": 3,
        "keywords:existence": 1
      }
    },
    "source": {
      "name": "IFEval · Instruction Following Evaluation",
      "url": "https://huggingface.co/datasets/google/IFEval",
      "revision": "966cd89545d6b6acfd7638bc708b98261ca58e84",
      "dataset_sha256": "6a85310ca8ce15eff755aa08a3a4ff931c7e273e7515ebb3c492ea85fd8288f2",
      "license": "Apache-2.0"
    },
    "verifier": {
      "name": "asn.ifeval-official-rules-subset.v1",
      "source_url": "https://github.com/google-research/google-research/blob/e6890f85757dd84e27ca6df2dd30651dafad28e0/instruction_following_eval/instructions.py",
      "source_revision": "e6890f85757dd84e27ca6df2dd30651dafad28e0",
      "supported_instructions": [
        "punctuation:no_comma",
        "detectable_format:number_bullet_lists",
        "detectable_format:json_format",
        "startend:quotation",
        "keywords:existence"
      ],
      "full_ifeval_coverage": false,
      "model_as_judge": false
    }
  },
  "protocol": {
    "models": [
      {
        "name": "qwen2.5:1.5b",
        "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b"
      },
      {
        "name": "llama3.2:3b",
        "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72"
      }
    ],
    "attempts_per_model": 10,
    "total_attempts": 20,
    "request_timeout_ms": 80000,
    "total_runtime_cap_ms": 1200000,
    "inference_options": {
      "temperature": 0,
      "seed": 42,
      "num_predict": 384,
      "keep_alive": "5m"
    },
    "execution": "serial Ollama loopback only",
    "external_provider_calls": false,
    "external_spend_microusd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false
  },
  "models": {
    "qwen2.5:1.5b": {
      "display_name": "Qwen 2.5 1.5B",
      "model_digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "attempts": 10,
      "verified_passes": 5,
      "verified_failures": 2,
      "execution_errors": 3,
      "verified_outputs": 7,
      "pass_rate_among_verified_outputs": 0.7143,
      "pass_rate_interval": {
        "confidence_level": 0.95,
        "method": "wilson_score_interval",
        "lower": 0.3589,
        "upper": 0.9178,
        "denominator": "verified_outputs_only_timeout_excluded"
      }
    },
    "llama3.2:3b": {
      "display_name": "Llama 3.2 3B",
      "model_digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "attempts": 10,
      "verified_passes": 4,
      "verified_failures": 0,
      "execution_errors": 6,
      "verified_outputs": 4,
      "pass_rate_among_verified_outputs": 1,
      "pass_rate_interval": {
        "confidence_level": 0.95,
        "method": "wilson_score_interval",
        "lower": 0.5101,
        "upper": 1,
        "denominator": "verified_outputs_only_timeout_excluded"
      }
    }
  },
  "paired_outcomes": {
    "complete_verified_pairs": 3,
    "incomplete_pairs": [
      {
        "task_id": 2195,
        "states": {
          "qwen2.5:1.5b": "execution_error",
          "llama3.2:3b": "execution_error"
        }
      },
      {
        "task_id": 3540,
        "states": {
          "qwen2.5:1.5b": "verified_fail",
          "llama3.2:3b": "execution_error"
        }
      },
      {
        "task_id": 1094,
        "states": {
          "qwen2.5:1.5b": "verified_pass",
          "llama3.2:3b": "execution_error"
        }
      },
      {
        "task_id": 1658,
        "states": {
          "qwen2.5:1.5b": "verified_pass",
          "llama3.2:3b": "execution_error"
        }
      },
      {
        "task_id": 2691,
        "states": {
          "qwen2.5:1.5b": "execution_error",
          "llama3.2:3b": "execution_error"
        }
      },
      {
        "task_id": 1162,
        "states": {
          "qwen2.5:1.5b": "execution_error",
          "llama3.2:3b": "verified_pass"
        }
      },
      {
        "task_id": 1286,
        "states": {
          "qwen2.5:1.5b": "verified_pass",
          "llama3.2:3b": "execution_error"
        }
      }
    ],
    "both_pass": 2,
    "qwen_only_pass": 0,
    "llama_only_pass": 1,
    "both_fail": 0
  },
  "task_outcomes": [
    {
      "task_id": 2195,
      "instruction_ids": [
        "detectable_format:number_bullet_lists"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "execution_error",
        "llama3.2:3b": "execution_error"
      }
    },
    {
      "task_id": 2239,
      "instruction_ids": [
        "startend:quotation"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      }
    },
    {
      "task_id": 3540,
      "instruction_ids": [
        "punctuation:no_comma"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "execution_error"
      }
    },
    {
      "task_id": 1094,
      "instruction_ids": [
        "detectable_format:json_format"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      }
    },
    {
      "task_id": 1658,
      "instruction_ids": [
        "startend:quotation",
        "keywords:existence"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      }
    },
    {
      "task_id": 2691,
      "instruction_ids": [
        "startend:quotation"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "execution_error",
        "llama3.2:3b": "execution_error"
      }
    },
    {
      "task_id": 2324,
      "instruction_ids": [
        "punctuation:no_comma"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      }
    },
    {
      "task_id": 1162,
      "instruction_ids": [
        "punctuation:no_comma"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "execution_error",
        "llama3.2:3b": "verified_pass"
      }
    },
    {
      "task_id": 1187,
      "instruction_ids": [
        "punctuation:no_comma"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      }
    },
    {
      "task_id": 1286,
      "instruction_ids": [
        "detectable_format:number_bullet_lists"
      ],
      "outcomes": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      }
    }
  ],
  "interpretation": {
    "public_ranking_eligible": false,
    "general_capability_verified": false,
    "production_measurement_index_eligible": false,
    "statement": "This is a ten-prompt descriptive slice covering five deterministic format rules, not full IFEval. 3 task pairs had two verified outputs; 3 Qwen 2.5 1.5B attempts timed out and 6 Llama 3.2 3B attempts timed out. The small sample, timeouts and public training exposure uncertainty do not support a general model ranking."
  },
  "limitations": [
    "Ten public prompts are a small slice of IFEval; public training exposure is unknown.",
    "The verifier implements only five named deterministic rules and does not assess answer truth, usefulness, safety or quality.",
    "One response per task and model does not establish stable performance.",
    "A timeout is an execution error and an unknown task outcome, not a verifier failure.",
    "Local Ollama results do not establish hosted latency, cost, availability or production behavior.",
    "Raw prompts, model outputs and detailed error text are omitted.",
    "This receipt is not production measurement and does not change rankings or routing."
  ],
  "integrity": {
    "source_suite_sha256": "f8fe39c51125503bd27b39e43b16813a247463bd3b75a283e3d8a87a1e007eed",
    "source_run_artifact_sha256": "c00a62dd32b2ba0b94066a32601be0fbfa168740868ed1af0df32516fca8b688",
    "raw_prompts_outputs_and_error_details_published": false
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "external_spend_microusd": 0,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "public_ranking_eligible": false,
    "authority_changed": false,
    "requires_operator_review_before_publication": true
  },
  "receipt_sha256": "f52ca85c821075c28b9965ad8d9025cea6fa197842591d579c2e56475d5252f6"
}