{
  "schema": "asn.ifeval-heldout-census-summary-audit.v1",
  "state": "audited_descriptive_summary",
  "created_at": "2026-10-21T09:30:30.361Z",
  "source_artifact_path": "brain/evidence/ifeval-heldout-rules-30-run-20261021.json",
  "source_artifact_sha256": "0d1366a989d79c4e271f25ceaca329cf04c91f4fff32d654939f4978b34d5f41",
  "suite_sha256": "3152ab43fe8c211946fc9c53f5c6bce98455ded93ec34af334c14371deb37a5d",
  "summary": {
    "qwen2.5:1.5b": {
      "attempts": 30,
      "verified_passes": 15,
      "verified_failures": 15,
      "execution_errors": 0,
      "valid_outputs": 30,
      "outputs_reaching_token_cap": 10,
      "verified_failures_reaching_token_cap": 9,
      "verified_passes_reaching_token_cap": 1,
      "pass_rate_among_returned": 0.5,
      "pass_rate_among_returned_95_interval": {
        "lower": 0.3315,
        "upper": 0.6685,
        "level": 0.95,
        "method": "wilson_score_interval",
        "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
      },
      "completion_rate": 1,
      "by_rule": {
        "punctuation:no_comma": {
          "attempts": 6,
          "verified_passes": 2,
          "verified_failures": 4,
          "execution_errors": 0,
          "outputs_reaching_token_cap": 2,
          "failures_reaching_token_cap": 2,
          "valid_outputs": 6,
          "pass_rate_among_returned": 0.3333,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.0968,
            "upper": 0.7,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 1
        },
        "detectable_format:number_bullet_lists": {
          "attempts": 6,
          "verified_passes": 4,
          "verified_failures": 2,
          "execution_errors": 0,
          "outputs_reaching_token_cap": 0,
          "failures_reaching_token_cap": 0,
          "valid_outputs": 6,
          "pass_rate_among_returned": 0.6667,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.3,
            "upper": 0.9032,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 1
        },
        "detectable_format:json_format": {
          "attempts": 6,
          "verified_passes": 4,
          "verified_failures": 2,
          "execution_errors": 0,
          "outputs_reaching_token_cap": 2,
          "failures_reaching_token_cap": 2,
          "valid_outputs": 6,
          "pass_rate_among_returned": 0.6667,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.3,
            "upper": 0.9032,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 1
        },
        "startend:quotation": {
          "attempts": 6,
          "verified_passes": 2,
          "verified_failures": 4,
          "execution_errors": 0,
          "outputs_reaching_token_cap": 4,
          "failures_reaching_token_cap": 4,
          "valid_outputs": 6,
          "pass_rate_among_returned": 0.3333,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.0968,
            "upper": 0.7,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 1
        },
        "keywords:existence": {
          "attempts": 6,
          "verified_passes": 3,
          "verified_failures": 3,
          "execution_errors": 0,
          "outputs_reaching_token_cap": 2,
          "failures_reaching_token_cap": 1,
          "valid_outputs": 6,
          "pass_rate_among_returned": 0.5,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.1876,
            "upper": 0.8124,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 1
        }
      }
    },
    "llama3.2:3b": {
      "attempts": 30,
      "verified_passes": 16,
      "verified_failures": 1,
      "execution_errors": 13,
      "valid_outputs": 17,
      "outputs_reaching_token_cap": 0,
      "verified_failures_reaching_token_cap": 0,
      "verified_passes_reaching_token_cap": 0,
      "pass_rate_among_returned": 0.9412,
      "pass_rate_among_returned_95_interval": {
        "lower": 0.7302,
        "upper": 0.9895,
        "level": 0.95,
        "method": "wilson_score_interval",
        "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
      },
      "completion_rate": 0.5667,
      "by_rule": {
        "punctuation:no_comma": {
          "attempts": 6,
          "verified_passes": 5,
          "verified_failures": 0,
          "execution_errors": 1,
          "outputs_reaching_token_cap": 0,
          "failures_reaching_token_cap": 0,
          "valid_outputs": 5,
          "pass_rate_among_returned": 1,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.5655,
            "upper": 1,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 0.8333
        },
        "detectable_format:number_bullet_lists": {
          "attempts": 6,
          "verified_passes": 4,
          "verified_failures": 0,
          "execution_errors": 2,
          "outputs_reaching_token_cap": 0,
          "failures_reaching_token_cap": 0,
          "valid_outputs": 4,
          "pass_rate_among_returned": 1,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.5101,
            "upper": 1,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 0.6667
        },
        "detectable_format:json_format": {
          "attempts": 6,
          "verified_passes": 2,
          "verified_failures": 0,
          "execution_errors": 4,
          "outputs_reaching_token_cap": 0,
          "failures_reaching_token_cap": 0,
          "valid_outputs": 2,
          "pass_rate_among_returned": 1,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.3424,
            "upper": 1,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 0.3333
        },
        "startend:quotation": {
          "attempts": 6,
          "verified_passes": 3,
          "verified_failures": 0,
          "execution_errors": 3,
          "outputs_reaching_token_cap": 0,
          "failures_reaching_token_cap": 0,
          "valid_outputs": 3,
          "pass_rate_among_returned": 1,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.4385,
            "upper": 1,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 0.5
        },
        "keywords:existence": {
          "attempts": 6,
          "verified_passes": 2,
          "verified_failures": 1,
          "execution_errors": 3,
          "outputs_reaching_token_cap": 0,
          "failures_reaching_token_cap": 0,
          "valid_outputs": 3,
          "pass_rate_among_returned": 0.6667,
          "pass_rate_among_returned_95_interval": {
            "lower": 0.2077,
            "upper": 0.9385,
            "level": 0.95,
            "method": "wilson_score_interval",
            "assumption": "independent prompts; six-task rule groups remain small and may not represent the source benchmark"
          },
          "completion_rate": 0.5
        }
      }
    }
  },
  "paired_outcomes": [
    {
      "task_id": 2245,
      "instruction_id": "punctuation:no_comma",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2374,
      "instruction_id": "punctuation:no_comma",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 2417,
      "instruction_id": "punctuation:no_comma",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2798,
      "instruction_id": "punctuation:no_comma",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 3323,
      "instruction_id": "punctuation:no_comma",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 3631,
      "instruction_id": "punctuation:no_comma",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 1934,
      "instruction_id": "detectable_format:number_bullet_lists",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 2078,
      "instruction_id": "detectable_format:number_bullet_lists",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2457,
      "instruction_id": "detectable_format:number_bullet_lists",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2801,
      "instruction_id": "detectable_format:number_bullet_lists",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2832,
      "instruction_id": "detectable_format:number_bullet_lists",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 3025,
      "instruction_id": "detectable_format:number_bullet_lists",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 13,
      "instruction_id": "detectable_format:json_format",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 321,
      "instruction_id": "detectable_format:json_format",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 1148,
      "instruction_id": "detectable_format:json_format",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2404,
      "instruction_id": "detectable_format:json_format",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2857,
      "instruction_id": "detectable_format:json_format",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 3223,
      "instruction_id": "detectable_format:json_format",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 1776,
      "instruction_id": "startend:quotation",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2010,
      "instruction_id": "startend:quotation",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2015,
      "instruction_id": "startend:quotation",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2532,
      "instruction_id": "startend:quotation",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 2985,
      "instruction_id": "startend:quotation",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 3536,
      "instruction_id": "startend:quotation",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 1531,
      "instruction_id": "keywords:existence",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 2485,
      "instruction_id": "keywords:existence",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 2602,
      "instruction_id": "keywords:existence",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "verified_fail"
      },
      "both_verified": true
    },
    {
      "task_id": 2662,
      "instruction_id": "keywords:existence",
      "models": {
        "qwen2.5:1.5b": "verified_fail",
        "llama3.2:3b": "verified_pass"
      },
      "both_verified": true
    },
    {
      "task_id": 3439,
      "instruction_id": "keywords:existence",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    },
    {
      "task_id": 3732,
      "instruction_id": "keywords:existence",
      "models": {
        "qwen2.5:1.5b": "verified_pass",
        "llama3.2:3b": "execution_error"
      },
      "both_verified": false
    }
  ],
  "execution_error_codes": {
    "llama3.2:3b:The_operation_was_aborted_due_to_timeout": 13
  },
  "interpretation_limits": {
    "scope": "30 held-out prompts across five deterministic IFEval rules, two pinned local model builds",
    "timeout_is_execution_error_not_verified_format_failure": true,
    "public_ranking_eligible": false,
    "general_capability_verified": false,
    "production_measurement_written": false,
    "external_provider_calls": false
  },
  "artifact_sha256": "f50597ddb3a23bc739e756f4946b875fe9a4916407b562c47c4e2021c22e5643"
}