{
  "report_id": "dulyrun-vllm-53745-20261022",
  "manifest_id": "3916aa19fa2c1e9ed32eb54815400ceb3c9da9e58ae342a2563c03fb1ad6cb64",
  "measured_at_utc": {
    "baseline": "2026-10-22T09:55:46.629611+00:00",
    "candidate": "2026-10-22T09:58:58.665698+00:00"
  },
  "published_date": "2026-10-22",
  "status": "INVESTIGATE",
  "scope": "Strict-tool parser configuration and eight short tool-calling requests. Public minimal reproduction, not a customer workload.",
  "source_issue": "https://github.com/vllm-project/vllm/issues/53745",
  "issue_state_when_checked": "closed",
  "versions": {
    "baseline": "0.27.1",
    "candidate": "0.28.0"
  },
  "environment": {
    "gpu": "one NVIDIA H100 NVL, 94 GB, same physical GPU for both releases",
    "driver": "595.91.07",
    "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8",
    "revision": "dcaee4d4dfc5ee71ad501f01f530e5652438fde0",
    "license": "Apache-2.0",
    "runtime_dependencies": {
      "baseline": {
        "vllm": "0.27.1",
        "torch": "2.13.0+cu130",
        "transformers": "5.15.0"
      },
      "candidate": {
        "vllm": "0.28.0",
        "torch": "2.13.0+cu130",
        "transformers": "5.15.1"
      }
    },
    "server_argv": [
      "/usr/bin/python3",
      "-m",
      "vllm.entrypoints.openai.api_server",
      "--model",
      "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8",
      "--revision",
      "dcaee4d4dfc5ee71ad501f01f530e5652438fde0",
      "--tokenizer-revision",
      "dcaee4d4dfc5ee71ad501f01f530e5652438fde0",
      "--host",
      "127.0.0.1",
      "--port",
      "8000",
      "--tensor-parallel-size",
      "1",
      "--gpu-memory-utilization",
      "0.60",
      "--max-model-len",
      "4096",
      "--max-num-seqs",
      "1",
      "--max-num-batched-tokens",
      "2048",
      "--enforce-eager",
      "--kernel-config",
      "{\"linear_backend\":\"triton\",\"moe_backend\":\"triton\"}",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "qwen3_coder",
      "--reasoning-parser",
      "qwen3"
    ]
  },
  "protocol": {
    "tools": "strict assign function; two required string fields; additionalProperties false",
    "requests": "auto and required tool choice; two repeats each per version",
    "temperature": 0,
    "max_tokens": 96,
    "top_logprobs": 5,
    "thinking": false,
    "comparison": "Normalized response content, reasoning, tool names and decoded arguments; excludes response metadata and generated tool-call IDs.",
    "control": "deepseek_r1 reasoning with qwen3_coder tool parser, both auto and required, unmocked parser probe only; no model requests claimed for this contrast."
  },
  "observations": {
    "parser_structural_tag": {
      "qwen3_reasoning_qwen3_coder_tool": {
        "baseline": {
          "auto": true,
          "required": true
        },
        "candidate": {
          "auto": false,
          "required": false
        }
      },
      "deepseek_r1_contrast": {
        "baseline": {
          "auto": true,
          "required": true
        },
        "candidate": {
          "auto": true,
          "required": true
        }
      }
    },
    "required_sampled_top5_masked_entries": {
      "baseline": [
        53,
        53
      ],
      "candidate": [
        0,
        0
      ]
    },
    "required_tool_call_count_per_response": {
      "baseline": [
        2,
        2
      ],
      "candidate": [
        1,
        1
      ]
    },
    "normalized_auto_outputs_equal_between_versions": true,
    "normalized_required_outputs_equal_between_versions": false,
    "within_release_repeats_equal": true,
    "all_observed_tool_arguments_schema_valid": true,
    "actual_gpu_requests": 8
  },
  "checks": {
    "both_servers_start": "PASS",
    "all_eight_requests_complete": "PASS",
    "all_observed_arguments_match_schema": "PASS",
    "strict_tag_preserved_for_qwen3_pair": "INVESTIGATE",
    "answer_quality": "NOT_TESTED",
    "speed": "NOT_CALIBRATED",
    "production_release_safety": "NOT_TESTED"
  },
  "limits": [
    "Strict tag observed in the unmocked parser-adjustment probe; logprob observation separately comes from the GPU requests.",
    "53 is the recorded sampled top-five masked-entry count, not all masked vocabulary tokens.",
    "No malformed argument keys were observed.",
    "Required-mode duplicate-call difference is not attributed to this defect.",
    "Adjacent stable versions were tested; no commit bisection. Later releases were not tested.",
    "Eager mode and explicit Triton linear/MoE backends in both arms; not the reporter production traffic.",
    "Release-bundled Transformers differs: 5.15.0 versus 5.15.1.",
    "Two earlier allocations failed in our serializer and in default FP8 kernel warmup; they are not successful reproductions."
  ],
  "compute_cost": {
    "final_attributable_usage_usd": null,
    "status": "PENDING_FINAL_PROVIDER_LEDGER",
    "new_cash_payment_usd": 0,
    "all_dedicated_resources_deleted": true
  },
  "evidence": {
    "source_result_sha256": "a789ba28e5d744e652a2b9c44266fd13c00e4865a0eb3d4ff76a6433980faef1",
    "raw_artifact_sha256": {
      "baseline/serve-command.json": "c9d7f0dd1c3068e2a34648f1f2d5f3c84026d1e7795556b4588d1ec747e2ea65",
      "baseline/unmocked-parser-observations.json": "3655fe5dc710191ff04045afff25888f82025165d965bb5fdd4ade2aa3c1d195",
      "baseline/required-request.json": "28ad55b626d81476933b88a6b110d685808d7aa19a9716bc303b179c8a17ebeb",
      "baseline/required-1-response.json": "aaff2d36585c0b38252355c66c9f21ccfcae8474f6769a07f8fa8c7f28441394",
      "baseline/required-2-response.json": "cbaad75fa558e54cc35b0a0374056eb798c9a0cefcecaf52a5b2895ecee38233",
      "baseline/auto-request.json": "d7150279f1218897b60b907346ad348a0eedd902580ae2484441cb60ca3184a8",
      "baseline/auto-1-response.json": "18b36999b877394da7542126afff5d956b20241a7eecb33e1396a911e25b8293",
      "baseline/auto-2-response.json": "f65af4b0d7917385b40b41792b80b92c72d0d65481132325658bb14ce160c851",
      "candidate/serve-command.json": "c9d7f0dd1c3068e2a34648f1f2d5f3c84026d1e7795556b4588d1ec747e2ea65",
      "candidate/unmocked-parser-observations.json": "6e8bb8643d836d98043cc87b949cb64463aa3e0299e649f04967851e059182ab",
      "candidate/required-request.json": "28ad55b626d81476933b88a6b110d685808d7aa19a9716bc303b179c8a17ebeb",
      "candidate/required-1-response.json": "1e82693767f9ab8fbf9fca822a9a02bb76017cb9833374b7eb0a77cd27c3d63a",
      "candidate/required-2-response.json": "38be320c8d67f764892a992af1d14d0edd80613b5fc59a7bdbcf643d1e0ecb84",
      "candidate/auto-request.json": "d7150279f1218897b60b907346ad348a0eedd902580ae2484441cb60ca3184a8",
      "candidate/auto-1-response.json": "32cb54a5fce4139673353e00d1490e0b1247bccb55c07874b5ea1bd48fe401d1",
      "candidate/auto-2-response.json": "0e679a84365bd311697d25707085adeeacadff5b94ba57b606e730f0a3440b17",
      "baseline/runtime.json": "b2c7ab7d0da64d29ff238b46b0c6c58105d2ef5752bc9ea7fa8aac9fbd1fef1a",
      "baseline/phase-complete.json": "46dc375ac35ac7285d3a1eea452ae5b1fa5610a78b6408b1cfd390da657d80f2",
      "candidate/runtime.json": "6bfa23bfc99e291547033bb94a85c283ebae7ea5f254309769779a547003cd47",
      "candidate/phase-complete.json": "46dc375ac35ac7285d3a1eea452ae5b1fa5610a78b6408b1cfd390da657d80f2"
    },
    "availability": "Full request/response and environment files retained privately. This public summary omits literal fixture values, host identifiers and operational paths; it does not replace them with a purported measured transcript."
  }
}