{
  "schema": "asn.public-local-tool-choice-pilot.v1",
  "study_id": "bfcl-v4-multiple-local-tool-choice-2026-10-24",
  "title": "Five-case local tool-choice pilot",
  "evaluated_at": {
    "start": "2026-10-24T03:35:04.165Z",
    "end": "2026-10-24T03:38:05.769Z"
  },
  "source": {
    "benchmark": "Berkeley Function Calling Leaderboard (BFCL) V4",
    "repository": "https://github.com/ShishirPatil/gorilla/tree/58f57e9124ea981403792dd51e00a6577e621fae/berkeley-function-call-leaderboard",
    "commit": "58f57e9124ea981403792dd51e00a6577e621fae",
    "license": "Apache-2.0",
    "category": "BFCL_v4_multiple",
    "cases_sha256": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
    "answer_file_sha256": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047",
    "source_artifact_sha256": "5e10acb7a2df07d2d3d2af8b77e8c90abf9a3de1193b8e13a142e5a12f7c833f",
    "provenance_correction": {
      "state": "verified",
      "prior_recorded_hashes": {
        "cases": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e8d6ee82c6969b2a",
        "answers": "244e00ce9395df948bca0fc7bee64e8f9c87ef70887587d83cae45b13699f3047"
      },
      "correction_artifact_sha256": "acd8bc90250622342c3046c5d8be7ea08106256de8b1987a965257d5e3e6d44d",
      "correction_artifact_url": "/bfcl-v4-source-provenance-correction-20261024.json"
    }
  },
  "protocol": {
    "models": [
      {
        "name": "qwen2.5:1.5b",
        "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b"
      },
      {
        "name": "llama3.2:3b",
        "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72"
      }
    ],
    "task_count": 5,
    "attempts": 10,
    "prompt_variants": 1,
    "repetitions_per_model_per_task": 1,
    "request_timeout_ms": 60000,
    "max_runtime_ms": 900000,
    "grading": "Tool selection is counted separately from complete function-and-argument verification. Arguments must match the source oracle and declared input types; only listed source aliases are accepted.",
    "runtime": "Ollama loopback only. Provider spend $0; local electricity and hardware opportunity cost unmeasured.",
    "tool_execution": false
  },
  "models": [
    {
      "name": "qwen2.5:1.5b",
      "digest": "65ec06548149b04c096a120e4a6da9d4017ea809c91734ea5631e89f96ddc57b",
      "attempts": 5,
      "tool_selections_correct": 5,
      "tool_selection_trials": 5,
      "exact_function_argument_passes": 3,
      "verified_failures": 2,
      "execution_errors": 0,
      "tool_selection_wilson_95": {
        "lower": 0.5655,
        "upper": 1
      },
      "exact_call_wilson_95": {
        "lower": 0.2307,
        "upper": 0.8824
      }
    },
    {
      "name": "llama3.2:3b",
      "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
      "attempts": 5,
      "tool_selections_correct": 5,
      "tool_selection_trials": 5,
      "exact_function_argument_passes": 2,
      "verified_failures": 3,
      "execution_errors": 0,
      "tool_selection_wilson_95": {
        "lower": 0.5655,
        "upper": 1
      },
      "exact_call_wilson_95": {
        "lower": 0.1176,
        "upper": 0.7693
      }
    }
  ],
  "tasks": [
    {
      "id": "multiple_95",
      "question": "Convert 200 euros to US dollars using current exchange rate.",
      "candidate_functions": [
        {
          "name": "currency_exchange.convert",
          "description": "Converts a value from one currency to another using the latest exchange rate."
        },
        {
          "name": "unit_conversion.convert",
          "description": "Converts a value from one unit to another."
        }
      ],
      "expected_function": "currency_exchange.convert",
      "expected_arguments": {
        "amount": 200,
        "from_currency": "EUR",
        "to_currency": "USD",
        "live_conversion": true
      },
      "accepted_arguments": {},
      "results": [
        {
          "model": "qwen2.5:1.5b",
          "selected_function": "currency_exchange.convert",
          "arguments": {
            "from_currency": "EUR",
            "to_currency": "USD",
            "live_conversion": false,
            "amount": 200
          },
          "tool_selection_correct": true,
          "exact_call_passed": false,
          "verification_reason": "argument_mismatch",
          "elapsed_ms": 20150
        },
        {
          "model": "llama3.2:3b",
          "selected_function": "currency_exchange.convert",
          "arguments": {
            "to_currency": "USD",
            "live_conversion": "true",
            "amount": "200",
            "from_currency": "EUR"
          },
          "tool_selection_correct": true,
          "exact_call_passed": false,
          "verification_reason": "arguments_schema_invalid",
          "elapsed_ms": 35761
        }
      ]
    },
    {
      "id": "multiple_165",
      "question": "Find the price of a used Gibson Les Paul guitar in excellent condition in the Chicago area.",
      "candidate_functions": [
        {
          "name": "identify_color_rgb",
          "description": "This function identifies the RGB values of a named color."
        },
        {
          "name": "board_game.chess.get_top_players",
          "description": "Find top chess players in a location based on rating."
        },
        {
          "name": "guitar_price.find",
          "description": "Retrieve the price of a specific used guitar model based on its condition and location."
        }
      ],
      "expected_function": "guitar_price.find",
      "expected_arguments": {
        "model": "Gibson Les Paul",
        "condition": "Excellent",
        "location": "Chicago"
      },
      "accepted_arguments": {
        "location": [
          "Chicago",
          "Chicago area"
        ]
      },
      "results": [
        {
          "model": "llama3.2:3b",
          "selected_function": "guitar_price.find",
          "arguments": {
            "model": "Gibson Les Paul",
            "condition": "Excellent",
            "location": "Chicago"
          },
          "tool_selection_correct": true,
          "exact_call_passed": true,
          "verification_reason": "exact_function_and_arguments",
          "elapsed_ms": 18525
        },
        {
          "model": "qwen2.5:1.5b",
          "selected_function": "guitar_price.find",
          "arguments": {
            "model": "Les Paul",
            "condition": "Excellent",
            "location": "Chicago"
          },
          "tool_selection_correct": true,
          "exact_call_passed": false,
          "verification_reason": "argument_mismatch",
          "elapsed_ms": 17030
        }
      ]
    },
    {
      "id": "multiple_44",
      "question": "How much will I weigh on Mars if my weight on Earth is 70 kg?",
      "candidate_functions": [
        {
          "name": "unit_conversion.convert",
          "description": "Convert a value from one unit to another."
        },
        {
          "name": "currency_conversion",
          "description": "Convert a value from one currency to another."
        },
        {
          "name": "calculate.weight_in_space",
          "description": "Calculate your weight on different planets given your weight on earth."
        }
      ],
      "expected_function": "calculate.weight_in_space",
      "expected_arguments": {
        "weight_earth_kg": 70,
        "planet": "Mars"
      },
      "accepted_arguments": {},
      "results": [
        {
          "model": "qwen2.5:1.5b",
          "selected_function": "calculate.weight_in_space",
          "arguments": {
            "weight_earth_kg": 70,
            "planet": "Mars"
          },
          "tool_selection_correct": true,
          "exact_call_passed": true,
          "verification_reason": "exact_function_and_arguments",
          "elapsed_ms": 9578
        },
        {
          "model": "llama3.2:3b",
          "selected_function": "calculate.weight_in_space",
          "arguments": {
            "planet": "Mars",
            "weight_earth_kg": "70"
          },
          "tool_selection_correct": true,
          "exact_call_passed": false,
          "verification_reason": "arguments_schema_invalid",
          "elapsed_ms": 15680
        }
      ]
    },
    {
      "id": "multiple_162",
      "question": "What is the structural dynamic analysis of the building with building Id B1004 for 2nd, 3rd and 4th floors?",
      "candidate_functions": [
        {
          "name": "player_statistic",
          "description": "Retrieves detailed player statistics for a specific year."
        },
        {
          "name": "analyze_structure",
          "description": "Analyze a structure of a building based on its Id and floor numbers."
        }
      ],
      "expected_function": "analyze_structure",
      "expected_arguments": {
        "building_id": "B1004",
        "floors": [
          2,
          3,
          4
        ],
        "mode": "dynamic"
      },
      "accepted_arguments": {},
      "results": [
        {
          "model": "llama3.2:3b",
          "selected_function": "analyze_structure",
          "arguments": {
            "building_id": "B1004",
            "floors": [
              "2",
              "3",
              "4"
            ],
            "mode": "dynamic"
          },
          "tool_selection_correct": true,
          "exact_call_passed": false,
          "verification_reason": "arguments_schema_invalid",
          "elapsed_ms": 16622
        },
        {
          "model": "qwen2.5:1.5b",
          "selected_function": "analyze_structure",
          "arguments": {
            "floors": [
              2,
              3,
              4
            ],
            "mode": "dynamic",
            "building_id": "B1004"
          },
          "tool_selection_correct": true,
          "exact_call_passed": true,
          "verification_reason": "exact_function_and_arguments",
          "elapsed_ms": 17595
        }
      ]
    },
    {
      "id": "multiple_56",
      "question": "Find the average closing price of Apple stock in the past 60 days.",
      "candidate_functions": [
        {
          "name": "volume_traded",
          "description": "Calculate total stock volume traded over a period."
        },
        {
          "name": "total_revenue",
          "description": "Calculate total company revenue over a period."
        },
        {
          "name": "avg_closing_price",
          "description": "Calculate the average closing price of a company over a period."
        }
      ],
      "expected_function": "avg_closing_price",
      "expected_arguments": {
        "company": "Apple",
        "days": 60
      },
      "accepted_arguments": {
        "data_source": [
          "yahoo finance",
          ""
        ]
      },
      "results": [
        {
          "model": "qwen2.5:1.5b",
          "selected_function": "avg_closing_price",
          "arguments": {
            "company": "Apple",
            "days": 60
          },
          "tool_selection_correct": true,
          "exact_call_passed": true,
          "verification_reason": "exact_function_and_arguments",
          "elapsed_ms": 10092
        },
        {
          "model": "llama3.2:3b",
          "selected_function": "avg_closing_price",
          "arguments": {
            "company": "Apple",
            "days": 60
          },
          "tool_selection_correct": true,
          "exact_call_passed": true,
          "verification_reason": "exact_function_and_arguments",
          "elapsed_ms": 20346
        }
      ]
    }
  ],
  "interpretation": {
    "public_ranking_eligible": false,
    "production_measurement_index_eligible": false,
    "general_tool_selection_capability_verified": false,
    "customer_utility_verified": false,
    "statement": "Both models selected the expected function on all five examples. Exact function-and-argument checks passed 5/10 calls overall (Qwen 3/5; Llama 2/5). The one-case difference is descriptive and does not support a model ranking.",
    "limitations": [
      "Only five public benchmark examples and one response per model-task pair.",
      "The examples are public and model training exposure is unknown.",
      "This is a project pilot, not official BFCL scoring or a reproduction of the leaderboard.",
      "No downstream tool was run; correct selection does not prove that a tool exists, is available, or returns a correct result.",
      "Latency is host-specific local wall time and is not a general speed comparison."
    ]
  },
  "semantics": {
    "local_only": true,
    "external_provider_calls": false,
    "provider_spend_usd": 0,
    "tool_execution": false,
    "production_measurement_written": false,
    "production_routing_changed": false,
    "public_ranking": false,
    "authority_changed": false
  },
  "receipt_sha256": "1aeac6517005551fba535520a9cc129a4ba1c71486e73c28f5618b4488dd903b"
}