{
  "schema": "asn.public-ifeval-output-cap-diagnostic.v1",
  "state": "verified_targeted_diagnostic",
  "created_at": "2026-10-21T03:54:24.932Z",
  "question": "For the ten Qwen outputs that reached the 384-token limit in the original held-out run, does raising the cap to 768 change truncation or deterministic verifier outcomes?",
  "selection": {
    "method": "Include every Qwen task in the held-out run whose Ollama done_reason was length at 384 generated tokens; this is a targeted diagnostic sample, not a random sample.",
    "task_count": 10,
    "task_ids": [
      2374,
      3631,
      2857,
      3223,
      1776,
      2010,
      2985,
      3536,
      1531,
      3439
    ],
    "sample_is_outcome_selected": true,
    "not_an_overall_rate": true
  },
  "source": {
    "name": "IFEval · Instruction Following Evaluation",
    "dataset_url": "https://huggingface.co/datasets/google/IFEval",
    "dataset_revision": "966cd89545d6b6acfd7638bc708b98261ca58e84",
    "license": "Apache-2.0"
  },
  "verifier": {
    "project": "Google Research · Instruction Following Evaluation",
    "upstream": "https://github.com/google-research/google-research/blob/e6890f85757dd84e27ca6df2dd30651dafad28e0/instruction_following_eval/instructions.py",
    "commit": "e6890f85757dd84e27ca6df2dd30651dafad28e0",
    "license": "Apache-2.0",
    "implemented_instructions": [
      "punctuation:no_comma",
      "detectable_format:number_bullet_lists",
      "detectable_format:json_format",
      "startend:quotation",
      "keywords:existence"
    ],
    "note": "Narrow compatible reimplementation of the named deterministic rules; not the complete official evaluator or benchmark.",
    "semantics": "five deterministic checks only; not the full official IFEval benchmark"
  },
  "model": {
    "name": "qwen2.5:1.5b",
    "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b"
  },
  "protocol": {
    "runtime": "Ollama loopback only",
    "baseline_output_cap": 384,
    "replication_output_cap": 768,
    "request_timeout_ms": 80000,
    "total_runtime_cap_ms": 900000,
    "temperature": 0,
    "seed": 42,
    "keep_alive": "5m",
    "requests": 10,
    "external_provider_calls": false,
    "external_spend_microusd": 0
  },
  "provenance": {
    "baseline_suite_sha256": "3152ab43fe8c211946fc9c53f5c6bce98455ded93ec34af334c14371deb37a5d",
    "baseline_run_sha256": "0d1366a989d79c4e271f25ceaca329cf04c91f4fff32d654939f4978b34d5f41",
    "diagnostic_suite_sha256": "5db159ac4cfdfdd01db87e2985a4bc2d94e932572ad72245bd3dcc9838e08232",
    "diagnostic_run_sha256": "18d5013f263ed1695c14d8dc4bb5987739f2ef527b60b3ea0da0d18966b68585",
    "verifier_independently_replayed": true
  },
  "summary": {
    "tasks": 10,
    "baseline_384": {
      "verified_passes": 1,
      "verified_failures": 9,
      "execution_errors": 0
    },
    "replication_768": {
      "verified_passes": 0,
      "verified_failures": 6,
      "execution_errors": 4,
      "outputs_reaching_768_cap": 0
    },
    "baseline_failures_rescued": 0,
    "paired_outcomes": [
      {
        "task_id": 2374,
        "instruction_id": "punctuation:no_comma",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "verified_fail",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 3631,
        "instruction_id": "punctuation:no_comma",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "verified_fail",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 2857,
        "instruction_id": "detectable_format:json_format",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "execution_error",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 3223,
        "instruction_id": "detectable_format:json_format",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "execution_error",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 1776,
        "instruction_id": "startend:quotation",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "execution_error",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 2010,
        "instruction_id": "startend:quotation",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "verified_fail",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 2985,
        "instruction_id": "startend:quotation",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "verified_fail",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 3536,
        "instruction_id": "startend:quotation",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "verified_fail",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 1531,
        "instruction_id": "keywords:existence",
        "baseline_384_state": "verified_fail",
        "baseline_384_reached_cap": true,
        "replication_768_state": "verified_fail",
        "replication_768_reached_cap": false
      },
      {
        "task_id": 3439,
        "instruction_id": "keywords:existence",
        "baseline_384_state": "verified_pass",
        "baseline_384_reached_cap": true,
        "replication_768_state": "execution_error",
        "replication_768_reached_cap": false
      }
    ]
  },
  "interpretation": {
    "finding": "Raising the generation cap from 384 to 768 did not produce a verified pass among six completed responses; four requests timed out and remain inconclusive.",
    "scope": "Ten tasks selected because the Qwen baseline reached the 384-token cap; this is a targeted diagnostic, not an unbiased IFEval estimate.",
    "ranking_eligible": false,
    "production_measurement_written": false,
    "general_capability_verified": false,
    "customer_value_measured": false,
    "raw_prompts_or_outputs_included": false
  },
  "artifact_sha256": "2e6afaba117d11d3041a22432896135e223d09ba0f37636f2a90567a79f3b044"
}