From 4142c1feea3e161a5de78ab455a8595a05e33bee Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Wed, 15 Apr 2026 12:54:54 +0200 Subject: [PATCH 01/33] save --- src/llm/io_processing/gemma4/tool_parser.cpp | 467 +++++++++++++++++++ src/llm/io_processing/gemma4/tool_parser.hpp | 93 ++++ 2 files changed, 560 insertions(+) create mode 100644 src/llm/io_processing/gemma4/tool_parser.cpp create mode 100644 src/llm/io_processing/gemma4/tool_parser.hpp diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp new file mode 100644 index 0000000000..a2498bd5a9 --- /dev/null +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -0,0 +1,467 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "tool_parser.hpp" +#include "../utils.hpp" +#include "../../../logging.hpp" +#include "../../../stringutils.hpp" +#include "rapidjson/error/en.h" +#include +#include +#include + +namespace ovms { + +const std::string Gemma4ToolParser::TOOL_CALL_START_TAG = "<|tool_call>"; +const std::string Gemma4ToolParser::TOOL_CALL_END_TAG = ""; +const std::string Gemma4ToolParser::TOOL_CALL_NAME_PREFIX = "call:"; + +const std::string Gemma4ToolParser::TOOL_ARGS_START_INDICATOR = "{"; +const std::string Gemma4ToolParser::TOOL_ARGS_END_INDICATOR = "}"; +const std::string Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR = "<\">"; +const std::string Gemma4ToolParser::TOOL_SEPARATOR_STR = ","; + +const int64_t Gemma4ToolParser::botTokenId = 10; +const int64_t Gemma4ToolParser::eotTokenId = 11; //to be changed + +std::string Gemma4ToolParser::parseArrayParameter(std::string argumentStr) { + int quoteDepth = 0; + + for (size_t i = 1; i < argumentStr.size() - 1; ++i) { + if (argumentStr[i] != '\'') { + continue; + } + + bool isLastElement = (i == argumentStr.size() - 2); + bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; + + if (quoteDepth == 0) { + argumentStr[i] = '"'; + quoteDepth++; + } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement)) { + argumentStr[i] = '"'; + quoteDepth--; + } + } + + return argumentStr; +} + +std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { + int quoteDepth = 0; + + for (size_t i = 1; i < argumentStr.size() - 1; ++i) { + if (argumentStr[i] != '\'') { + continue; + } + + bool isLastElement = (i == argumentStr.size() - 2); + bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; + bool isFollowedByColon = !isLastElement && argumentStr[i + 1] == ':'; + + if (quoteDepth == 0) { + argumentStr[i] = '"'; + quoteDepth++; + } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement || isFollowedByColon)) { + argumentStr[i] = '"'; + quoteDepth--; + } + } + + return argumentStr; +} + +std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { + if (arg.empty()) { + return arg; + } + + std::string normalized = arg; + trim(normalized); + std::string lower = normalized; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + + if (lower == "true" || lower == "false" || lower == "null") { + return lower; + } + + const char first = normalized.front(); + const char last = normalized.back(); + if (first == '{' && last == '}') { + normalized = parseObjectParameter(normalized); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument contains is an object, replaced single quotes with double quotes for JSON parsing. Modified string: {}", normalized); + } + + if (first == '[' && last == ']') { + normalized = parseArrayParameter(normalized); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an array, normalized quotes for JSON parsing. Modified string: {}", normalized); + } + + if ((first == '\'' && last == '\'')) { + normalized[0] = '"'; + normalized[normalized.size() - 1] = '"'; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is enclosed in quotes, replaced outer quotes with double quotes for JSON parsing. Modified string: {}", normalized); + } + + rapidjson::Document tempDoc; + rapidjson::Value finalValue; + tempDoc.Parse(normalized.c_str()); + if (tempDoc.HasParseError()) { + auto errorCode = tempDoc.GetParseError(); + auto errorMessage = rapidjson::GetParseError_En(errorCode); + size_t errorOffset = tempDoc.GetErrorOffset(); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse argument string as JSON. Argument string: {}, Error: {} Offset: {}", normalized, errorMessage, errorOffset); + + if (first == '\"' && last == '\"') { + normalized = normalized.substr(1, normalized.size() - 2); + } + finalValue.SetString(normalized.c_str(), static_cast(normalized.size()), tempDoc.GetAllocator()); + } else { + finalValue.CopyFrom(tempDoc, tempDoc.GetAllocator()); + } + + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + finalValue.Accept(writer); + normalized = buffer.GetString(); + } + + return normalized; +} + +void Gemma4ToolParser::writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer) { + // std::string normalized = normalizeArgStr(arg); to be fitted to actual normalization with corner cases handled + + rapidjson::Document doc; + doc.Parse(normalized.c_str()); + + rapidjson::Value& argumentDoc = doc; + writeArgumentOfAnyType(argumentDoc, writer); +} + +std::pair Gemma4ToolParser::parseSingleArgument(const std::string& argumentStr) { + std::pair argument; + + size_t equalPos = argumentStr.find(':'); + if (equalPos != std::string::npos) { + argument.first = argumentStr.substr(0, equalPos); + argument.second = argumentStr.substr(equalPos + 1); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument - name: {}, value: {}", argument.first, argument.second); + } else { + argument.first = argumentStr; + argument.second = ""; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument string: {} does not contain ':', setting name as entire string and value as empty", argumentStr); + } + return argument; +} + +std::vector> Gemma4ToolParser::parseArguments(const std::string& argumentsStr) { + std::vector args; + std::vector> parsedArgs; + + size_t argPos = 0; + while (argPos < argumentsStr.length()) { + size_t commaPos = findInStringRespectingSpecialChars(argumentsStr, TOOL_SEPARATOR_STR, argPos); + if (commaPos == std::string::npos) { + auto remainingStr = argumentsStr.substr(argPos); + args.push_back(remainingStr); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more commas found, adding remaining argument string: {}", remainingStr); + break; + } + auto argStr = argumentsStr.substr(argPos, commaPos - argPos); + args.push_back(argStr); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument string: {}", argStr); + argPos = commaPos + TOOL_SEPARATOR_STR.length(); + } + + for (const std::string& arg : args) { + parsedArgs.push_back(parseSingleArgument(arg)); + } + return parsedArgs; +} + +bool Gemma4ToolParser::parseInContentState() { + size_t toolCallStartTagPos = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); + size_t toolCallEndTagPos = this->streamingContent.find(TOOL_CALL_END_TAG, this->streamingPosition); + if (toolCallEndTagPos != std::string::npos && toolCallStartTagPos == std::string::npos) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); + this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); + return false; + } + if (toolCallStartTagPos != std::string::npos) { + if (toolCallStartTagPos > this->streamingPosition) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Content found before tool call start tag at position: {}", toolCallStartTagPos); + return true; + } + this->streamingPosition = toolCallStartTagPos + TOOL_CALL_START_TAG.length(); + this->currentState = State::ToolCallStarted; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected start of tool call at position: {}", toolCallStartTagPos); + return false; + } + + return true; +} + +bool Gemma4ToolParser::parseInToolCallState() { + size_t toolListStartPos = this->streamingContent.find(TOOL_LIST_START_INDICATOR, this->streamingPosition); + size_t argsPos = this->streamingContent.find(TOOL_ARGS_START_INDICATOR, this->streamingPosition); + + if (toolListStartPos != std::string::npos) { + this->streamingPosition = toolListStartPos + TOOL_LIST_START_INDICATOR.length(); + } + + if (argsPos == std::string::npos) { + return false; + } + + std::string toolName = this->streamingContent.substr(this->streamingPosition, argsPos - this->streamingPosition); + this->toolCall = ToolCall{generateRandomId(), toolName, ""}; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); + this->streamingPosition = argsPos + TOOL_ARGS_START_INDICATOR.length(); + this->currentState = State::ToolCallParameters; + this->toolCallIndex++; + return true; +} + +bool Gemma4ToolParser::parseToolCallParametersState() { + size_t pos = findInStringRespectingSpecialChars(this->streamingContent, TOOL_ARGS_END_INDICATOR, this->streamingPosition); + if (pos == std::string::npos) { + return false; + } + std::string argumentsStr = this->streamingContent.substr(this->streamingPosition, pos - this->streamingPosition); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed arguments string: {}", argumentsStr); + std::vector> arguments = parseArguments(argumentsStr); + + rapidjson::Document argsDoc(rapidjson::kObjectType); + rapidjson::StringBuffer sb; + rapidjson::Writer argsWriter(sb); + argsWriter.StartObject(); + + for (const std::pair& argument : arguments) { + argsWriter.Key(argument.first.c_str()); + writeArgumentToWriter(argument.second, argsWriter); + } + + argsWriter.EndObject(); + this->toolCall.arguments = sb.GetString(); + this->currentState = State::ToolCallEnded; + this->streamingPosition = pos + TOOL_ARGS_END_INDICATOR.length(); + + return true; +} + +bool Gemma4ToolParser::parseInToolCallEndedState() { + size_t pos = this->streamingContent.find(TOOL_LIST_END_INDICATOR, this->streamingPosition); + size_t toolSeparatorPos = this->streamingContent.find(TOOL_SEPARATOR_STR, this->streamingPosition); + size_t toolCallEndTagPos = this->streamingContent.find(TOOL_CALL_END_TAG, this->streamingPosition); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Current state: ToolCallEnded. Streaming content from current position: {}", this->streamingContent.substr(this->streamingPosition)); + if (pos == std::string::npos && toolSeparatorPos == std::string::npos && toolCallEndTagPos == std::string::npos) { + return false; + } else if (toolSeparatorPos != std::string::npos && toolSeparatorPos < pos) { + this->streamingPosition = toolSeparatorPos + TOOL_SEPARATOR_STR.length(); + this->currentState = State::ToolCallStarted; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected separator between tool calls at position: {}, expecting another tool call to start", toolSeparatorPos); + } else if (toolCallEndTagPos != std::string::npos) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); + this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); + this->currentState = State::AfterToolCall; + } else { + this->streamingPosition = pos + TOOL_LIST_END_INDICATOR.length(); + this->currentState = State::AfterToolCall; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool list at position: {}, returning to content state", pos); + } + return true; +} + +bool Gemma4ToolParser::parseNewContent() { + switch (this->currentState) { + case State::Content: { + return parseInContentState(); + } + case State::ToolCallStarted: { + return parseInToolCallState(); + } + case State::ToolCallParameters: { + return parseToolCallParametersState(); + } + case State::ToolCallEnded: { + return parseInToolCallEndedState(); + } + case State::AfterToolCall: + break; + } + return false; +} + +rapidjson::Document Gemma4ToolParser::wrapDeltaContent(const std::string& content) { + rapidjson::Document doc(rapidjson::kObjectType); + rapidjson::Value deltaObj(rapidjson::kObjectType); + deltaObj.AddMember("content", rapidjson::Value(content.c_str(), doc.GetAllocator()), doc.GetAllocator()); + doc.AddMember("delta", deltaObj, doc.GetAllocator()); + return doc; +} + +rapidjson::Document Gemma4ToolParser::wrapDeltaArgs(const std::string& argsStr, int toolCallIndex) { + rapidjson::Document doc(rapidjson::kObjectType); + doc.AddMember("arguments", rapidjson::Value(argsStr.c_str(), doc.GetAllocator()), doc.GetAllocator()); + + return BaseOutputParser::wrapDelta(doc, toolCallIndex); +} + +std::optional Gemma4ToolParser::parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) { + if (chunk.empty()) { + return std::nullopt; + } + + this->streamingContent += chunk; + + if (parseNewContent()) { + if (this->currentState == State::ToolCallParameters) { + return BaseOutputParser::wrapFirstDelta(this->toolCall.name, toolCallIndex); + } + if (this->currentState == State::ToolCallEnded) { + return wrapDeltaArgs(this->toolCall.arguments, toolCallIndex); + } + if (this->currentState == State::Content) { + size_t contentEnd = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); + std::string content; + if (contentEnd != std::string::npos) { + content = this->streamingContent.substr(this->streamingPosition, contentEnd - this->streamingPosition); + } else { + content = this->streamingContent.substr(this->streamingPosition); + } + this->streamingPosition += content.size(); + if (!content.empty()) { + return wrapDeltaContent(content); + } + } + if (this->currentState == State::AfterToolCall) { + this->currentState = State::Content; + } + } + + if (finishReason != ov::genai::GenerationFinishReason::NONE) { + if ((this->currentState == State::ToolCallParameters || this->currentState == State::ToolCallEnded) && !this->toolCall.arguments.empty()) { + return wrapDeltaArgs(this->toolCall.arguments, toolCallIndex); + } + + if (this->currentState == State::Content && this->streamingPosition < this->streamingContent.size()) { + auto content = this->streamingContent.substr(this->streamingPosition); + this->streamingPosition += content.size(); + + return wrapDeltaContent(content); + } + } + + return std::nullopt; +} + +bool Gemma4ToolParser::parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall) { + size_t argsPos = toolStr.find(TOOL_ARGS_START_INDICATOR); + if (argsPos != std::string::npos) { + std::string toolNameWithPrefix = toolStr.substr(0, argsPos); + if (toolNameWithPrefix.find(TOOL_CALL_NAME_PREFIX) != 0) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool name does not start with expected prefix '{}'. Tool string: {}", TOOL_CALL_NAME_PREFIX, toolStr); + return false; + } + std::string toolName = toolNameWithPrefix.substr(TOOL_CALL_NAME_PREFIX.length()); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); + + int argsStrLen = toolStr.length() - argsPos - TOOL_ARGS_START_INDICATOR.length() - TOOL_ARGS_END_INDICATOR.length(); + std::string argsStr = toolStr.substr(argsPos + TOOL_ARGS_START_INDICATOR.length(), argsStrLen); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed args string: {}", argsStr); + std::vector> arguments = parseArguments(argsStr); + + toolCall.name = toolName; + rapidjson::Document argsDoc(rapidjson::kObjectType); + rapidjson::StringBuffer sb; + rapidjson::Writer argsWriter(sb); + argsWriter.StartObject(); + for (const std::pair& argument : arguments) { + argsWriter.Key(argument.first.c_str()); + writeArgumentToWriter(argument.second, argsWriter); + } + argsWriter.EndObject(); + toolCall.arguments = sb.GetString(); + toolCall.id = generateRandomId(); + return true; + } + return false; +} + +void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + std::vector tools; + std::vector> toolCallPositions; + size_t pos = 0; + + while (pos != std::string::npos) { + size_t start, end; + auto it = std::find(generatedTokens.begin() + pos, generatedTokens.end(), botTokenId); + if (it != generatedTokens.end()) { + start = std::distance(generatedTokens.begin(), it); + } else { + break; + } + auto itArgs = std::find(generatedTokens.begin() + start, generatedTokens.end(), eotTokenId); + if (itArgs != generatedTokens.end()) { + end = std::distance(generatedTokens.begin(), itArgs); + } else { + break; + } + + std::string toolCallStr = tokenizer.decode(std::vector(generatedTokens.begin() + start, generatedTokens.begin() + end + 1), ov::AnyMap{ov::genai::skip_special_tokens(false)}); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool list string: {}", toolCallStr); + + while (!toolCallStr.empty()) { + size_t toolEndPos = findInStringRespectingSpecialChars(toolCallStr, TOOL_ARGS_END_INDICATOR, 0); + std::string singleTool; + if (toolEndPos != std::string::npos) { + singleTool = toolCallStr.substr(0, toolEndPos + TOOL_ARGS_END_INDICATOR.length()); + if (toolEndPos + TOOL_ARGS_END_INDICATOR.length() < toolCallStr.length()) { + toolCallStr = toolCallStr.substr(toolEndPos + TOOL_ARGS_END_INDICATOR.length() + TOOL_SEPARATOR_STR.length()); + } else { + toolCallStr.clear(); + } + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed single tool string {}", singleTool); + } + + if (!singleTool.empty()) { + tools.push_back(singleTool); + } + } + + pos = end; + toolCallPositions.emplace_back(start, end); + } + + for (const std::string& tool : tools) { + ToolCall toolCall; + auto wasToolCallParsed = parseSingleToolCall(tool, toolCall); + if (wasToolCallParsed) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool call - name: {}, args: {}", toolCall.name, toolCall.arguments); + parsedOutput.toolCalls.push_back(toolCall); + } else { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse tool call from string: {}", tool); + } + } +s + std::vector contentWithoutToolCalls = generatedTokens; + for (auto it = toolCallPositions.rbegin(); it != toolCallPositions.rend(); ++it) { + contentWithoutToolCalls.erase(contentWithoutToolCalls.begin() + it->first, contentWithoutToolCalls.begin() + it->second + 1); + } + parsedOutput.content = tokenizer.decode(contentWithoutToolCalls, ov::AnyMap{ov::genai::skip_special_tokens(true)}); +} +} // namespace ovms \ No newline at end of file diff --git a/src/llm/io_processing/gemma4/tool_parser.hpp b/src/llm/io_processing/gemma4/tool_parser.hpp new file mode 100644 index 0000000000..2961c7c561 --- /dev/null +++ b/src/llm/io_processing/gemma4/tool_parser.hpp @@ -0,0 +1,93 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include "src/llm/io_processing/base_output_parser.hpp" + +namespace ovms { +class Gemma4ToolParser : public BaseOutputParser { +protected: + static const std::string TOOL_CALL_START_TAG; + static const std::string TOOL_CALL_END_TAG; + static const std::string TOOL_RESPONSE_TAG; + + static const std::string TOOL_ARGS_START_INDICATOR; + static const std::string TOOL_ARGS_END_INDICATOR; + static const std::string TOOL_SEPARATOR_STR; + + static const int64_t botTokenId; + + enum class State { + Content, // Content -> ToolCallStarted (on TOOL_CALL_START_TAG) + ToolCallStarted, // ToolCallStarted -> ToolCallParameters (on TOOL_ARGS_START_INDICATOR, emits name) + ToolCallParameters, // ToolCallParameters -> ToolCallEnded (on TOOL_ARGS_END_INDICATOR, emits args) + ToolCallEnded, // ToolCallEnded -> ToolCallStarted (on separator) | AfterToolCall (on end tag/list end) + AfterToolCall // AfterToolCall -> Content + }; + +public: + Gemma4ToolParser() = delete; + explicit Gemma4ToolParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags = {TOOL_CALL_START_TAG}; + return parsingStartTags; + } + + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector beginningOnlyTags = {}; + return beginningOnlyTags; + } + + const std::string& getParsingEndTag() const override { + return TOOL_CALL_END_TAG; + } + + bool requiresStreamingWithSpecialTokens() const override { + return true; //to be checked if it's actually required + } + + static std::string normalizeArgStr(const std::string& arg); + static std::string parseArrayParameter(std::string argumentStr); + static std::string parseObjectParameter(std::string argumentStr); + +private: + void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer); + + std::pair parseSingleArgument(const std::string& argumentStr); + std::vector> parseArguments(const std::string& argumentsStr); + + bool parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall); + bool parseNewContent(); + bool parseInContentState(); + bool parseInToolCallState(); + bool parseToolCallParametersState(); + bool parseInToolCallEndedState(); + + rapidjson::Document wrapDeltaContent(const std::string& content); + rapidjson::Document wrapDeltaArgs(const std::string& argsStr, int toolCallIndex); + + std::string streamingContent; + size_t streamingPosition{0}; + State currentState{State::Content}; + ToolCall toolCall; + int toolCallIndex{-1}; +}; +} // namespace ovms \ No newline at end of file From a53a9bfe1ee740b92896a7a7378388af0f2d6109 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 20 Apr 2026 11:25:50 +0200 Subject: [PATCH 02/33] adding tests --- src/llm/io_processing/gemma4/tool_parser.cpp | 9 +- src/llm/io_processing/gemma4/tool_parser.hpp | 2 +- .../gemma4_output_parser_test.cpp | 801 ++++++++++++++++++ 3 files changed, 807 insertions(+), 5 deletions(-) create mode 100644 src/test/llm/output_parsers/gemma4_output_parser_test.cpp diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index a2498bd5a9..47ceab6cd7 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -34,7 +34,7 @@ const std::string Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR = "<\">"; const std::string Gemma4ToolParser::TOOL_SEPARATOR_STR = ","; const int64_t Gemma4ToolParser::botTokenId = 10; -const int64_t Gemma4ToolParser::eotTokenId = 11; //to be changed +const int64_t Gemma4ToolParser::eotTokenId = 11; //to be changed std::string Gemma4ToolParser::parseArrayParameter(std::string argumentStr) { int quoteDepth = 0; @@ -152,7 +152,7 @@ void Gemma4ToolParser::writeArgumentToWriter(const std::string& arg, rapidjson:: writeArgumentOfAnyType(argumentDoc, writer); } -std::pair Gemma4ToolParser::parseSingleArgument(const std::string& argumentStr) { +std::pair Gemma4ToolParser::parseSingleArgument(const std::string& argumentStr) { std::pair argument; size_t equalPos = argumentStr.find(':'); @@ -457,8 +457,9 @@ void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector contentWithoutToolCalls = generatedTokens; + s + std::vector + contentWithoutToolCalls = generatedTokens; for (auto it = toolCallPositions.rbegin(); it != toolCallPositions.rend(); ++it) { contentWithoutToolCalls.erase(contentWithoutToolCalls.begin() + it->first, contentWithoutToolCalls.begin() + it->second + 1); } diff --git a/src/llm/io_processing/gemma4/tool_parser.hpp b/src/llm/io_processing/gemma4/tool_parser.hpp index 2961c7c561..5b7b34e05d 100644 --- a/src/llm/io_processing/gemma4/tool_parser.hpp +++ b/src/llm/io_processing/gemma4/tool_parser.hpp @@ -61,7 +61,7 @@ class Gemma4ToolParser : public BaseOutputParser { } bool requiresStreamingWithSpecialTokens() const override { - return true; //to be checked if it's actually required + return true; //to be checked if it's actually required } static std::string normalizeArgStr(const std::string& arg); diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp new file mode 100644 index 0000000000..7c99ab79b9 --- /dev/null +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -0,0 +1,801 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../llm/io_processing/base_output_parser.hpp" +#include "../../../llm/io_processing/output_parser.hpp" +#include "../../platform_utils.hpp" + +using namespace ovms; + +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\google\\gemma-4-31B-it"; +#else +// Hardcoded for usage in docker container +const std::string tokenizerPath = "/ovms/src/test/llm_testing/google/gemma-4-31B-it"; +#endif + +static std::unique_ptr gemma4Tokenizer; +static const ToolsSchemas_t& EMPTY_TOOLS_SCHEMA = {}; // not used in gemma4 + +class Gemma4OutputParserTest : public ::testing::Test { +protected: + std::unique_ptr outputParserWithRegularToolParsing; + + static void SetUpTestSuite() { + try { + gemma4Tokenizer = std::make_unique(tokenizerPath); + } catch (const std::exception& e) { + FAIL() << "Failed to initialize gemma4 tokenizer: " << e.what(); + } catch (...) { + FAIL() << "Failed to initialize gemma4 tokenizer due to unknown error."; + } + } + + static void TearDownTestSuite() { + gemma4Tokenizer.reset(); + } + + void SetUp() override { + // For Gemma4 model there is only tool parser available + outputParserWithRegularToolParsing = std::make_unique(*gemma4Tokenizer, "gemma4", "", EMPTY_TOOLS_SCHEMA); + } + + void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { + if (!expectedDelta.has_value() && !doc.has_value()) { + return; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; + } else { + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk; + } + } +}; + +TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { + std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { + std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + std::string testInput = input; + auto generatedTensor = gemma4Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); + EXPECT_EQ(parsedOutput.content, testInput); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithObjectArguments) { + std::string inputWithProperClosure = "<|tool_call>call:dummy{config:{'name':'astro_config','value':99}}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "dummy"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArguments) { + std::string inputWithProperClosure = "<|tool_call>call:test1{arg1:<\">data1,data2<\">}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "test1"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"data1,data2\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithListOfStringsAsArgument) { + std::string inputWithProperClosure = "<|tool_call>call:generate_DNA_sequence{length:100,preferences:['G','C']}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "generate_DNA_sequence"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"length\":100,\"preferences\":[\"G\",\"C\"]}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(Gemma4OutputParserTest, ParserToolCallWithBooleanArgument) { + std::string inputWithProperClosure = "<|tool_call>call:check_status{flag:true}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "check_status"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"flag\":true}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(Gemma4OutputParserTest, ParseTwoToolCallsAtOnce) { + std::string inputWithProperClosure = "<|tool_call>call:dummy1{config:{'name':'astro_config','value':99}},call:dummy2{config:{'name':'second_config','value':199}}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 2); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "dummy1"); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "dummy2"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"config\":{\"name\":\"second_config\",\"value\":199}}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); // ID should be generated + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithArrayArguments) { + std::string inputWithProperClosure = "<|tool_call>call:sort{array:[42,17,89,5,33],order:<\">descending<\">}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "sort"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringWithSingleQuotesArguments) { + std::string inputWithProperClosure = "<|tool_call>call:sort{array:[42,17,89,5,33],order:'descending'}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "sort"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { + std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}" + "<|tool_call>call:another_tool{param1:<\">data<\">,param2:true}" + "<|tool_call>call:third_tool{key:<\">value<\">}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 3); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); + auto firstToolCallId = parsedOutput.toolCalls[0].id; + + EXPECT_EQ(parsedOutput.toolCalls[1].name, "another_tool"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"param1\":\"data\",\"param2\":true}"); + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); + auto secondToolCallId = parsedOutput.toolCalls[1].id; + EXPECT_NE(firstToolCallId, secondToolCallId); + + EXPECT_EQ(parsedOutput.toolCalls[2].name, "third_tool"); + EXPECT_EQ(parsedOutput.toolCalls[2].arguments, "{\"key\":\"value\"}"); + EXPECT_EQ(parsedOutput.toolCalls[2].id.empty(), false); + auto thirdToolCallId = parsedOutput.toolCalls[2].id; + EXPECT_NE(firstToolCallId, thirdToolCallId); + EXPECT_NE(secondToolCallId, thirdToolCallId); + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCallsWithContentInBetween) { + std::string inputWithProperClosure = "Before tool calls content. " + "<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}" + "This is some content between tool calls." + "<|tool_call>call:another_tool{param1:<\">data<\">,param2:true}" + " This is some content between second and third tool call. " + "<|tool_call>call:third_tool{key:<\">value<\">}" + "After tool calls content."; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, "Before tool calls content. This is some content between tool calls. This is some content between second and third tool call. After tool calls content."); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 3); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); + auto firstToolCallId = parsedOutput.toolCalls[0].id; + + EXPECT_EQ(parsedOutput.toolCalls[1].name, "another_tool"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"param1\":\"data\",\"param2\":true}"); + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); + auto secondToolCallId = parsedOutput.toolCalls[1].id; + EXPECT_NE(firstToolCallId, secondToolCallId); + + EXPECT_EQ(parsedOutput.toolCalls[2].name, "third_tool"); + EXPECT_EQ(parsedOutput.toolCalls[2].arguments, "{\"key\":\"value\"}"); + EXPECT_EQ(parsedOutput.toolCalls[2].id.empty(), false); + auto thirdToolCallId = parsedOutput.toolCalls[2].id; + EXPECT_NE(firstToolCallId, thirdToolCallId); + EXPECT_NE(secondToolCallId, thirdToolCallId); + } +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyArguments) { + // Tool call with empty braces (no arguments) + std::string input = "<|tool_call>call:no_args_tool{}"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "no_args_tool"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { + std::string input = "This is a regular model response without tool calls."; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + EXPECT_EQ(parsedOutput.reasoning, ""); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { + std::string input = "This is a content part and next will be a tool call.\n\n<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + +TEST_F(Gemma4OutputParserTest, HolisticStreaming) { + std::vector>> chunkToDeltaVec{ + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"call:", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{array", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, + {":[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"42", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 17", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 89", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 5", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 33", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"order", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":<\">", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<\">,", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"call:d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"dummy"}}]}})"}, + {":{", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"}}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings + std::string expected = expectedDelta.value(); + std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + // Compare everything except the id value + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk (ignoring id value): " << chunk; + } else { + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; + } + } else { + std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; + std::string docStr = doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "std::nullopt"; + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk + << "\nexpectedDelta: " << expectedStr + << "\ndoc: " << docStr; + } + } +} + +TEST_F(Gemma4OutputParserTest, StreamingWithBiggerChunks) { + std::vector>> chunkToDeltaVec{ + {"SOME_CONTENT", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"SOME_CONTENT"}})"}, + {"MORE_CONTENT<|tool_call>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"MORE_CONTENT"}})"}, + {"call:sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{array:", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, + {"[42, 17, 89, 5, 33],order:<\">descending<\">", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings + std::string expected = expectedDelta.value(); + std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + // Compare everything except the id value + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk (ignoring id value): " << chunk; + } else { + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; + } + } else { + std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; + std::string docStr = doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "std::nullopt"; + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk + << "\nexpectedDelta: " << expectedStr + << "\ndoc: " << docStr; + } + } +} + +TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { + std::vector>> chunkToDeltaVec{ + // Tool call phase + // Starting first tool. Collecting chunk until full name is received. Don't return until then. + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"call:sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{array", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, + {":[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"42", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 17", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 89", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 5", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 33", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"order", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":<\">", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<\">}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Some ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Some "}})"}, + {"content ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"content "}})"}, + {"between ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"between "}})"}, + {"tool ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tool "}})"}, + {"calls.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"calls."}})"}, + {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"call:d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"dummy"}}]}})"}, + {":{", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"}}", ov ::genai ::GenerationFinishReason ::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, + {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"call:solve", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{e", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"solve"}}]}})"}, + {"quation", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":<\">", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"2", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"*", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"x", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"+", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"5)", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" =", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 13", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<\">}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"equation\":\"2*(x+5) = 13\"}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"And some content after second tool call", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"And some content after second tool call"}})"}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings + std::string expected = expectedDelta.value(); + std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + // Compare everything except the id value + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk (ignoring id value): " << chunk; + } else { + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; + } + } else { + std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; + std::string docStr = doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "std::nullopt"; + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk + << "\nexpectedDelta: " << expectedStr + << "\ndoc: " << docStr; + } + } +} + +TEST_F(Gemma4OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { + std::vector>> chunkToDeltaVec{ + // Tool parser is available, but tools are not in the request so every chunk is just a regular content + {"<|tool_call>", "{\"delta\":{\"content\":\"<|tool_call>\"}}"}, + {"call:super", "{\"delta\":{\"content\":\"call:super\"}}"}, + {"_tool_number_two", "{\"delta\":{\"content\":\"_tool_number_two\"}}"}, + {"{arg1", "{\"delta\":{\"content\":\"{arg1\"}}"}, + {":<\">", "{\"delta\":{\"content\":\":<\\\">\"}}"}, + {"val{{{ue1", "{\"delta\":{\"content\":\"val{{{ue1\"}}"}, + {"<\">}", "{\"delta\":{\"content\":\"<\\\">}\"}}"}, + {"", "{\"delta\":{\"content\":\"\"}}"}, + }; + + for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { + // Second argument is false as we simulate the case where tools have not been provided in the request + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, false, ov::genai::GenerationFinishReason::NONE); + assertChunkEqual(doc, expectedDelta, chunk); + } +} + +// Malformed tool calls + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingParentheses) { + std::string input = "<|tool_call>call:broken_tool"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingClosingParenthesis) { + std::string input = "<|tool_call>call:broken_tool{arg1:<\">value1<\">"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithArgumentMissingEquals) { + // Argument without ':' sign - parseSingleArgument sets value as empty + std::string input = "<|tool_call>call:broken{malformed_arg}"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + // The tool call is parsed but the argument value will be empty and invalid + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); +} + +// Tests with special characters +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingComparison) { + std::string input = R"x(<|tool_call>call:search{query:<">price >= 100, (sale)<">,limit:5})x"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "search"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"x({"query":"price >= 100, (sale)","limit":5})x"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesAndBrackets) { + std::string input = R"(<|tool_call>call:format{template:<">Hello {name}, items: [a, b, c]<">,count:3})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "format"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"template":"Hello {name}, items: [a, b, c]","count":3})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecialCharacters) { + std::string impl = "import package\nimport package2\n\ndef func(a, b):\n\td={\"python\": \"dict\"}\n\tl = [\"list \\\"with escaped text\\\"\", 123, []]\n\treturn f\"formatted {a} and {b}\""; + std::string input = R"(<|tool_call>call:execute{code:<">)" + impl + R"(<">})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"code":"import package\nimport package2\n\ndef func(a, b):\n\td={\"python\": \"dict\"}\n\tl = [\"list \\\"with escaped text\\\"\", 123, []]\n\treturn f\"formatted {a} and {b}\""})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingEscapedQuotes) { + std::string input = R"x(<|tool_call>call:execute{code:<">print(\"hello world\")<">,verbose:true})x"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"x({"code":"print(\"hello world\")","verbose":true})x"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingApostrophes) { + std::string input = R"(<|tool_call>call:log{message:<">it's a test, isn't it?<">,level:<">warn<">})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "log"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"message":"it's a test, isn't it?","level":"warn"})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBackslashes) { + std::string input = R"(<|tool_call>call:read_file{path:<">C:\Users\test\file.txt<">,encoding:<">utf-8<">})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "read_file"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"path":"C:\\Users\\test\\file.txt","encoding":"utf-8"})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsContainingQuotes) { + std::string input = R"(<|tool_call>call:save{lines:['it's the wonderful day','My name's Jan','That's Johns' car.']})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"lines":["it's the wonderful day","My name's Jan","That's Johns' car."]})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStringsContainingQuotes) { + std::string input = R"(<|tool_call>call:save{obj:{'name':'it's the wonderful day','greeting':'Hello, my name's Jan','note':'That's Johns' car.'}})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"obj":{"name":"it's the wonderful day","greeting":"Hello, my name's Jan","note":"That's Johns' car."}})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJSON) { + std::string input = R"(<|tool_call>call:send{payload:<">{'key': 'value', 'count': 42}<">,endpoint:<">api<">})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "send"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"payload":"{'key': 'value', 'count': 42}","endpoint":"api"})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyStringArgument) { + std::string input = R"(<|tool_call>call:create{name:<"><">,value:0})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "create"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"name":"","value":0})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { + std::string input = R"(<|tool_call>call:translate{text:<">zażółć gęślą jaźń<">,lang:<">pl<">})"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"text":"zażółć gęślą jaźń","lang":"pl"})"); +} \ No newline at end of file From 66cb9aca6b7ec51bddb69315fe9e66347e5bbcc2 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 20 Apr 2026 11:52:07 +0200 Subject: [PATCH 03/33] fixed build --- src/llm/BUILD | 16 +++++++++++++ src/llm/io_processing/gemma4/tool_parser.cpp | 24 ++++++------------- src/llm/io_processing/gemma4/tool_parser.hpp | 4 +++- src/llm/io_processing/utils.cpp | 1 - src/llm/io_processing/utils.hpp | 3 +++ .../gemma4_output_parser_test.cpp | 6 ++--- 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/llm/BUILD b/src/llm/BUILD index 6be387c954..4587e24402 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -197,6 +197,21 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "io_processing_gemma4_tool_parser", + hdrs = ["io_processing/gemma4/tool_parser.hpp"], + srcs = ["io_processing/gemma4/tool_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + "//src:libovmsstring_utils", + ":io_processing_utils", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) ovms_cc_library( # TODO split further so we don't have to recompile everything when changing one parser ... name = "output_parsers", @@ -234,6 +249,7 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_base_output_parser", ":io_processing_qwen3coder_tool_parser", ":io_processing_lfm2_tool_parser", + ":io_processing_gemma4_tool_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", ], diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 47ceab6cd7..33d9cc3da3 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -33,8 +33,8 @@ const std::string Gemma4ToolParser::TOOL_ARGS_END_INDICATOR = "}"; const std::string Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR = "<\">"; const std::string Gemma4ToolParser::TOOL_SEPARATOR_STR = ","; -const int64_t Gemma4ToolParser::botTokenId = 10; -const int64_t Gemma4ToolParser::eotTokenId = 11; //to be changed +const int64_t Gemma4ToolParser::botTokenId = 48; +const int64_t Gemma4ToolParser::eotTokenId = 49; std::string Gemma4ToolParser::parseArrayParameter(std::string argumentStr) { int quoteDepth = 0; @@ -146,7 +146,7 @@ void Gemma4ToolParser::writeArgumentToWriter(const std::string& arg, rapidjson:: // std::string normalized = normalizeArgStr(arg); to be fitted to actual normalization with corner cases handled rapidjson::Document doc; - doc.Parse(normalized.c_str()); + doc.Parse(arg.c_str()); rapidjson::Value& argumentDoc = doc; writeArgumentOfAnyType(argumentDoc, writer); @@ -216,13 +216,8 @@ bool Gemma4ToolParser::parseInContentState() { } bool Gemma4ToolParser::parseInToolCallState() { - size_t toolListStartPos = this->streamingContent.find(TOOL_LIST_START_INDICATOR, this->streamingPosition); size_t argsPos = this->streamingContent.find(TOOL_ARGS_START_INDICATOR, this->streamingPosition); - if (toolListStartPos != std::string::npos) { - this->streamingPosition = toolListStartPos + TOOL_LIST_START_INDICATOR.length(); - } - if (argsPos == std::string::npos) { return false; } @@ -264,13 +259,10 @@ bool Gemma4ToolParser::parseToolCallParametersState() { } bool Gemma4ToolParser::parseInToolCallEndedState() { - size_t pos = this->streamingContent.find(TOOL_LIST_END_INDICATOR, this->streamingPosition); size_t toolSeparatorPos = this->streamingContent.find(TOOL_SEPARATOR_STR, this->streamingPosition); size_t toolCallEndTagPos = this->streamingContent.find(TOOL_CALL_END_TAG, this->streamingPosition); SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Current state: ToolCallEnded. Streaming content from current position: {}", this->streamingContent.substr(this->streamingPosition)); - if (pos == std::string::npos && toolSeparatorPos == std::string::npos && toolCallEndTagPos == std::string::npos) { - return false; - } else if (toolSeparatorPos != std::string::npos && toolSeparatorPos < pos) { + if (toolSeparatorPos != std::string::npos && toolSeparatorPos < toolCallEndTagPos) { this->streamingPosition = toolSeparatorPos + TOOL_SEPARATOR_STR.length(); this->currentState = State::ToolCallStarted; SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected separator between tool calls at position: {}, expecting another tool call to start", toolSeparatorPos); @@ -279,9 +271,9 @@ bool Gemma4ToolParser::parseInToolCallEndedState() { this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); this->currentState = State::AfterToolCall; } else { - this->streamingPosition = pos + TOOL_LIST_END_INDICATOR.length(); + this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); this->currentState = State::AfterToolCall; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool list at position: {}, returning to content state", pos); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}, returning to content state", toolCallEndTagPos); } return true; } @@ -457,9 +449,7 @@ void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector - contentWithoutToolCalls = generatedTokens; + std::vector contentWithoutToolCalls = generatedTokens; for (auto it = toolCallPositions.rbegin(); it != toolCallPositions.rend(); ++it) { contentWithoutToolCalls.erase(contentWithoutToolCalls.begin() + it->first, contentWithoutToolCalls.begin() + it->second + 1); } diff --git a/src/llm/io_processing/gemma4/tool_parser.hpp b/src/llm/io_processing/gemma4/tool_parser.hpp index 5b7b34e05d..6b360a23fb 100644 --- a/src/llm/io_processing/gemma4/tool_parser.hpp +++ b/src/llm/io_processing/gemma4/tool_parser.hpp @@ -23,13 +23,15 @@ class Gemma4ToolParser : public BaseOutputParser { protected: static const std::string TOOL_CALL_START_TAG; static const std::string TOOL_CALL_END_TAG; - static const std::string TOOL_RESPONSE_TAG; + static const std::string TOOL_CALL_NAME_PREFIX; static const std::string TOOL_ARGS_START_INDICATOR; static const std::string TOOL_ARGS_END_INDICATOR; + static const std::string TOOL_ARGS_STRING_INDICATOR; static const std::string TOOL_SEPARATOR_STR; static const int64_t botTokenId; + static const int64_t eotTokenId; enum class State { Content, // Content -> ToolCallStarted (on TOOL_CALL_START_TAG) diff --git a/src/llm/io_processing/utils.cpp b/src/llm/io_processing/utils.cpp index e26ca376b4..0204657c2f 100644 --- a/src/llm/io_processing/utils.cpp +++ b/src/llm/io_processing/utils.cpp @@ -89,5 +89,4 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str } return std::string::npos; } - } // namespace ovms diff --git a/src/llm/io_processing/utils.hpp b/src/llm/io_processing/utils.hpp index 1a956f6fee..79c7358af9 100644 --- a/src/llm/io_processing/utils.hpp +++ b/src/llm/io_processing/utils.hpp @@ -28,4 +28,7 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str void writeArgumentOfAnyType(const rapidjson::Value& arg, rapidjson::Writer& writer); // Generates random alphanumeric string of length 9 for tool call ID std::string generateRandomId(); + +size_t findInStringRespectingSpecialChars(const std::string& str, const std::string& target, size_t startPos); +void writeArgumentOfAnyType(const rapidjson::Value& arg, rapidjson::Writer& writer); } // namespace ovms diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 7c99ab79b9..cd93cbce95 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2025 Intel Corporation +// Copyright 2026 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,10 +29,10 @@ using namespace ovms; #ifdef _WIN32 -const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\google\\gemma-4-31B-it"; +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\google\\gemma-4-26B-A4B-it"; #else // Hardcoded for usage in docker container -const std::string tokenizerPath = "/ovms/src/test/llm_testing/google/gemma-4-31B-it"; +const std::string tokenizerPath = "/ovms/src/test/llm_testing/google/gemma-4-26B-A4B-it"; #endif static std::unique_ptr gemma4Tokenizer; From 7a9a4c9c517904240a8c78f9382a7e9888fb84d1 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 20 Apr 2026 12:13:45 +0200 Subject: [PATCH 04/33] save --- src/llm/io_processing/output_parser.cpp | 6 ++++++ src/llm/io_processing/utils.cpp | 26 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index c6d9fe8a67..f0d2d3c479 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -29,7 +29,11 @@ #include "qwen3coder/qwen3coder_tool_parser.hpp" #include "devstral/tool_parser.hpp" #include "gptoss/reasoning_parser.hpp" +<<<<<<< HEAD #include "lfm2/lfm2_tool_parser.hpp" +======= +#include "gemma4/tool_parser.hpp" +>>>>>>> 0c55d5ad (save) namespace ovms { OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { @@ -184,6 +188,8 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (toolParserName == "lfm2") { toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "gemma4") { + toolParser = std::make_unique(tokenizer); } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: " + toolParserName); } diff --git a/src/llm/io_processing/utils.cpp b/src/llm/io_processing/utils.cpp index 0204657c2f..104e446739 100644 --- a/src/llm/io_processing/utils.cpp +++ b/src/llm/io_processing/utils.cpp @@ -35,6 +35,32 @@ std::string generateRandomId() { } return id; } +void writeArgumentOfAnyType(const rapidjson::Value& arg, rapidjson::Writer& writer) { + if (arg.IsString()) { + writer.String(arg.GetString()); + } else if (arg.IsInt64()) { + writer.Int64(arg.GetInt64()); + } else if (arg.IsDouble()) { + writer.Double(arg.GetDouble()); + } else if (arg.IsBool()) { + writer.Bool(arg.GetBool()); + } else if (arg.IsArray()) { + writer.StartArray(); + for (auto& elem : arg.GetArray()) { + writeArgumentOfAnyType(elem, writer); + } + writer.EndArray(); + } else if (arg.IsObject()) { + writer.StartObject(); + for (auto it = arg.MemberBegin(); it != arg.MemberEnd(); ++it) { + writer.Key(it->name.GetString()); + writeArgumentOfAnyType(it->value, writer); + } + writer.EndObject(); + } else { + writer.String(""); + } +} void writeArgumentOfAnyType(const rapidjson::Value& arg, rapidjson::Writer& writer) { if (arg.IsString()) { From 461c65b8adbc57f5b5a9b52c73619d385e9ff1e1 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Tue, 21 Apr 2026 11:51:52 +0200 Subject: [PATCH 05/33] save --- prepare_llm_models.sh | 16 ++++++++++- src/llm/io_processing/gemma4/tool_parser.cpp | 29 +++++++++++++------- windows_prepare_llm_models.bat | 2 ++ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index 6bc2a861ed..4f0acf4e3c 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -38,7 +38,11 @@ PHI4_MODEL="microsoft/Phi-4-mini-instruct" MISTRAL_MODEL="mistralai/Mistral-7B-Instruct-v0.3" GPT_OSS_MODEL="openai/gpt-oss-20b" DEVSTRAL_MODEL="unsloth/Devstral-Small-2507" +<<<<<<< HEAD LFM2_MODEL="LiquidAI/LFM2-2.6B" +======= +GEMMA4_MODEL="google/gemma-4-26B-A4B-it" +>>>>>>> 73a55b89 (save) if [ "$(python3 -c 'import sys; print(sys.version_info[1])')" -le "8" ]; then echo "Prepare models with python > 3.8."; exit 1 ; fi @@ -228,4 +232,14 @@ fi if [ ! -f "$1/$LFM2_MODEL/$TOKENIZER_FILE" ]; then echo "[ERROR] Models file $1/$LFM2_MODEL/$TOKENIZER_FILE does not exist." exit 1 -fi \ No newline at end of file +fi +if [ -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then + echo "Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." +else + mkdir -p $1/$GEMMA4_MODEL + convert_tokenizer $GEMMA4_MODEL --with_detokenizer -o $1/$GEMMA4_MODEL +fi +if [ ! -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then + echo "[ERROR] Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE does not exist." + exit 1 +fi diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 33d9cc3da3..54c4e92f5c 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -109,10 +109,10 @@ std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an array, normalized quotes for JSON parsing. Modified string: {}", normalized); } - if ((first == '\'' && last == '\'')) { - normalized[0] = '"'; - normalized[normalized.size() - 1] = '"'; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is enclosed in quotes, replaced outer quotes with double quotes for JSON parsing. Modified string: {}", normalized); + if (normalized.substr(0, TOOL_ARGS_STRING_INDICATOR.size()) == TOOL_ARGS_STRING_INDICATOR && + normalized.substr(normalized.size() - TOOL_ARGS_STRING_INDICATOR.size(), TOOL_ARGS_STRING_INDICATOR.size()) == TOOL_ARGS_STRING_INDICATOR) { + normalized = "\"" + normalized.substr(TOOL_ARGS_STRING_INDICATOR.size(), normalized.size() - 2 * TOOL_ARGS_STRING_INDICATOR.size()) + "\""; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is enclosed in string indicators, removed them for JSON parsing. Modified string: {}", normalized); } rapidjson::Document tempDoc; @@ -143,10 +143,10 @@ std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { } void Gemma4ToolParser::writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer) { - // std::string normalized = normalizeArgStr(arg); to be fitted to actual normalization with corner cases handled + std::string normalized = normalizeArgStr(arg); rapidjson::Document doc; - doc.Parse(arg.c_str()); + doc.Parse(normalized.c_str()); rapidjson::Value& argumentDoc = doc; writeArgumentOfAnyType(argumentDoc, writer); @@ -158,7 +158,13 @@ std::pair Gemma4ToolParser::parseSingleArgument(const size_t equalPos = argumentStr.find(':'); if (equalPos != std::string::npos) { argument.first = argumentStr.substr(0, equalPos); - argument.second = argumentStr.substr(equalPos + 1); + std::string value = argumentStr.substr(equalPos + 1); + size_t argsStringIndicatorPos = value.find(TOOL_ARGS_STRING_INDICATOR); + size_t argsStringIndicatorEndPos = value.rfind(TOOL_ARGS_STRING_INDICATOR); + if(argsStringIndicatorPos != std::string::npos && argsStringIndicatorEndPos != std::string::npos && argsStringIndicatorEndPos > argsStringIndicatorPos) { + value = value.substr(0, argsStringIndicatorPos) + "\"" + value.substr(argsStringIndicatorPos + TOOL_ARGS_STRING_INDICATOR.length(), argsStringIndicatorEndPos - argsStringIndicatorPos - TOOL_ARGS_STRING_INDICATOR.length()) + "\""; + } + argument.second = value; SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument - name: {}, value: {}", argument.first, argument.second); } else { argument.first = argumentStr; @@ -181,7 +187,7 @@ std::vector> Gemma4ToolParser::parseArgument SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more commas found, adding remaining argument string: {}", remainingStr); break; } - auto argStr = argumentsStr.substr(argPos, commaPos - argPos); + std::string argStr = argumentsStr.substr(argPos, commaPos - argPos); args.push_back(argStr); SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument string: {}", argStr); argPos = commaPos + TOOL_SEPARATOR_STR.length(); @@ -414,11 +420,11 @@ void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector(generatedTokens.begin() + start, generatedTokens.begin() + end + 1), ov::AnyMap{ov::genai::skip_special_tokens(false)}); + std::string toolCallStr = tokenizer.decode(std::vector(generatedTokens.begin() + start + 1, generatedTokens.begin() + end + 1), ov::AnyMap{ov::genai::skip_special_tokens(false)}); SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool list string: {}", toolCallStr); while (!toolCallStr.empty()) { - size_t toolEndPos = findInStringRespectingSpecialChars(toolCallStr, TOOL_ARGS_END_INDICATOR, 0); + size_t toolEndPos = toolCallStr.find(TOOL_ARGS_END_INDICATOR); std::string singleTool; if (toolEndPos != std::string::npos) { singleTool = toolCallStr.substr(0, toolEndPos + TOOL_ARGS_END_INDICATOR.length()); @@ -428,6 +434,9 @@ void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector Date: Thu, 23 Apr 2026 08:01:23 +0200 Subject: [PATCH 06/33] save --- src/llm/io_processing/gemma4/tool_parser.cpp | 111 +++++++++++------- src/llm/io_processing/gemma4/tool_parser.hpp | 4 +- .../gemma4_output_parser_test.cpp | 66 +++++------ 3 files changed, 102 insertions(+), 79 deletions(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 54c4e92f5c..8b46695491 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -30,8 +30,8 @@ const std::string Gemma4ToolParser::TOOL_CALL_NAME_PREFIX = "call:"; const std::string Gemma4ToolParser::TOOL_ARGS_START_INDICATOR = "{"; const std::string Gemma4ToolParser::TOOL_ARGS_END_INDICATOR = "}"; -const std::string Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR = "<\">"; -const std::string Gemma4ToolParser::TOOL_SEPARATOR_STR = ","; +const std::string Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR = "<|\"|>"; +const std::string Gemma4ToolParser::TOOL_ARGS_SEPARATOR_STR = ","; const int64_t Gemma4ToolParser::botTokenId = 48; const int64_t Gemma4ToolParser::eotTokenId = 49; @@ -60,27 +60,49 @@ std::string Gemma4ToolParser::parseArrayParameter(std::string argumentStr) { } std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { - int quoteDepth = 0; + size_t pos = 1; + std::vector> keyValuePairs; - for (size_t i = 1; i < argumentStr.size() - 1; ++i) { - if (argumentStr[i] != '\'') { - continue; + while (pos != std::string::npos) { + std::string key, value; + bool isStringValue = false; + size_t keyEndPos = argumentStr.find(':', pos); + if (keyEndPos == std::string::npos) { + break; } - - bool isLastElement = (i == argumentStr.size() - 2); - bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; - bool isFollowedByColon = !isLastElement && argumentStr[i + 1] == ':'; - - if (quoteDepth == 0) { - argumentStr[i] = '"'; - quoteDepth++; - } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement || isFollowedByColon)) { - argumentStr[i] = '"'; - quoteDepth--; + key = argumentStr.substr(pos, keyEndPos - pos); + size_t valueStartPos = keyEndPos + 1; + size_t valueEndPos; + if ( argumentStr.substr(keyEndPos + 2, TOOL_ARGS_STRING_INDICATOR.size()) == TOOL_ARGS_STRING_INDICATOR) { + valueStartPos = keyEndPos + 2 + TOOL_ARGS_STRING_INDICATOR.size(); + valueEndPos = argumentStr.find(TOOL_ARGS_STRING_INDICATOR, valueStartPos); + isStringValue = true; + } else { + valueEndPos = argumentStr.find(',', valueStartPos); + } + + if (valueEndPos == std::string::npos) { + valueEndPos = argumentStr.size() - 1; } + value = argumentStr.substr(valueStartPos, valueEndPos - valueStartPos); + if (isStringValue) { + value = "\"" + value + "\""; + } + keyValuePairs.emplace_back(key, value); + pos = (valueEndPos == argumentStr.size() - 1) ? std::string::npos : valueEndPos + 1; } - return argumentStr; + if (keyValuePairs.empty()) { + return argumentStr; + } + + std::string parsedObject = "{"; + for (const auto& [key, value] : keyValuePairs) { + parsedObject += "\"" + key + "\":" + value + ","; + } + parsedObject.back() = '}'; + return parsedObject; + } std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { @@ -101,7 +123,7 @@ std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { const char last = normalized.back(); if (first == '{' && last == '}') { normalized = parseObjectParameter(normalized); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument contains is an object, replaced single quotes with double quotes for JSON parsing. Modified string: {}", normalized); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument contains is an object, changed it to correct JSON format. Modified string: {}", normalized); } if (first == '[' && last == ']') { @@ -155,15 +177,10 @@ void Gemma4ToolParser::writeArgumentToWriter(const std::string& arg, rapidjson:: std::pair Gemma4ToolParser::parseSingleArgument(const std::string& argumentStr) { std::pair argument; - size_t equalPos = argumentStr.find(':'); - if (equalPos != std::string::npos) { - argument.first = argumentStr.substr(0, equalPos); - std::string value = argumentStr.substr(equalPos + 1); - size_t argsStringIndicatorPos = value.find(TOOL_ARGS_STRING_INDICATOR); - size_t argsStringIndicatorEndPos = value.rfind(TOOL_ARGS_STRING_INDICATOR); - if(argsStringIndicatorPos != std::string::npos && argsStringIndicatorEndPos != std::string::npos && argsStringIndicatorEndPos > argsStringIndicatorPos) { - value = value.substr(0, argsStringIndicatorPos) + "\"" + value.substr(argsStringIndicatorPos + TOOL_ARGS_STRING_INDICATOR.length(), argsStringIndicatorEndPos - argsStringIndicatorPos - TOOL_ARGS_STRING_INDICATOR.length()) + "\""; - } + size_t colonPos = argumentStr.find(':'); + if (colonPos != std::string::npos) { + argument.first = argumentStr.substr(0, colonPos); + std::string value = argumentStr.substr(colonPos + 1); argument.second = value; SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument - name: {}, value: {}", argument.first, argument.second); } else { @@ -180,7 +197,7 @@ std::vector> Gemma4ToolParser::parseArgument size_t argPos = 0; while (argPos < argumentsStr.length()) { - size_t commaPos = findInStringRespectingSpecialChars(argumentsStr, TOOL_SEPARATOR_STR, argPos); + size_t commaPos = findInStringRespectingSpecialChars(argumentsStr, TOOL_ARGS_SEPARATOR_STR, argPos); if (commaPos == std::string::npos) { auto remainingStr = argumentsStr.substr(argPos); args.push_back(remainingStr); @@ -190,7 +207,7 @@ std::vector> Gemma4ToolParser::parseArgument std::string argStr = argumentsStr.substr(argPos, commaPos - argPos); args.push_back(argStr); SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument string: {}", argStr); - argPos = commaPos + TOOL_SEPARATOR_STR.length(); + argPos = commaPos + TOOL_ARGS_SEPARATOR_STR.length(); } for (const std::string& arg : args) { @@ -201,12 +218,6 @@ std::vector> Gemma4ToolParser::parseArgument bool Gemma4ToolParser::parseInContentState() { size_t toolCallStartTagPos = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); - size_t toolCallEndTagPos = this->streamingContent.find(TOOL_CALL_END_TAG, this->streamingPosition); - if (toolCallEndTagPos != std::string::npos && toolCallStartTagPos == std::string::npos) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); - this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); - return false; - } if (toolCallStartTagPos != std::string::npos) { if (toolCallStartTagPos > this->streamingPosition) { SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Content found before tool call start tag at position: {}", toolCallStartTagPos); @@ -223,12 +234,18 @@ bool Gemma4ToolParser::parseInContentState() { bool Gemma4ToolParser::parseInToolCallState() { size_t argsPos = this->streamingContent.find(TOOL_ARGS_START_INDICATOR, this->streamingPosition); - if (argsPos == std::string::npos) { return false; } - std::string toolName = this->streamingContent.substr(this->streamingPosition, argsPos - this->streamingPosition); + size_t toolNameStart = this->streamingContent.find(TOOL_CALL_NAME_PREFIX, this->streamingPosition); + if (toolNameStart != std::string::npos && toolNameStart < argsPos) { + toolNameStart += TOOL_CALL_NAME_PREFIX.length(); + } else { + toolNameStart = this->streamingPosition; + } + + std::string toolName = this->streamingContent.substr(toolNameStart, argsPos - toolNameStart); this->toolCall = ToolCall{generateRandomId(), toolName, ""}; SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); this->streamingPosition = argsPos + TOOL_ARGS_START_INDICATOR.length(); @@ -265,13 +282,13 @@ bool Gemma4ToolParser::parseToolCallParametersState() { } bool Gemma4ToolParser::parseInToolCallEndedState() { - size_t toolSeparatorPos = this->streamingContent.find(TOOL_SEPARATOR_STR, this->streamingPosition); + size_t nextToolCallPos = this->streamingContent.find(TOOL_CALL_NAME_PREFIX, this->streamingPosition); size_t toolCallEndTagPos = this->streamingContent.find(TOOL_CALL_END_TAG, this->streamingPosition); SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Current state: ToolCallEnded. Streaming content from current position: {}", this->streamingContent.substr(this->streamingPosition)); - if (toolSeparatorPos != std::string::npos && toolSeparatorPos < toolCallEndTagPos) { - this->streamingPosition = toolSeparatorPos + TOOL_SEPARATOR_STR.length(); + if (nextToolCallPos != std::string::npos && nextToolCallPos < toolCallEndTagPos) { + this->streamingPosition = nextToolCallPos; this->currentState = State::ToolCallStarted; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected separator between tool calls at position: {}, expecting another tool call to start", toolSeparatorPos); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected next tool call at position: {}", nextToolCallPos); } else if (toolCallEndTagPos != std::string::npos) { SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); @@ -424,12 +441,18 @@ void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector ToolCallStarted (on TOOL_CALL_START_TAG) ToolCallStarted, // ToolCallStarted -> ToolCallParameters (on TOOL_ARGS_START_INDICATOR, emits name) ToolCallParameters, // ToolCallParameters -> ToolCallEnded (on TOOL_ARGS_END_INDICATOR, emits args) - ToolCallEnded, // ToolCallEnded -> ToolCallStarted (on separator) | AfterToolCall (on end tag/list end) + ToolCallEnded, // ToolCallEnded -> ToolCallStarted (on TOOL_CALL_NAME_PREFIX) | AfterToolCall (on end tag) AfterToolCall // AfterToolCall -> Content }; diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index cd93cbce95..da46f7e0e7 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -79,7 +79,7 @@ class Gemma4OutputParserTest : public ::testing::Test { }; TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { - std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}"; + std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -98,7 +98,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { } TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { - std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}"; + std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -114,7 +114,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { } TEST_F(Gemma4OutputParserTest, ParseToolCallWithObjectArguments) { - std::string inputWithProperClosure = "<|tool_call>call:dummy{config:{'name':'astro_config','value':99}}"; + std::string inputWithProperClosure = "<|tool_call>call:dummy{config:{name:<|\"|>astro_config<|\"|>,value:99}}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -133,7 +133,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithObjectArguments) { } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArguments) { - std::string inputWithProperClosure = "<|tool_call>call:test1{arg1:<\">data1,data2<\">}"; + std::string inputWithProperClosure = "<|tool_call>call:test1{arg1:<|\"|>data1,data2<|\"|>}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -190,7 +190,7 @@ TEST_F(Gemma4OutputParserTest, ParserToolCallWithBooleanArgument) { } TEST_F(Gemma4OutputParserTest, ParseTwoToolCallsAtOnce) { - std::string inputWithProperClosure = "<|tool_call>call:dummy1{config:{'name':'astro_config','value':99}},call:dummy2{config:{'name':'second_config','value':199}}"; + std::string inputWithProperClosure = "<|tool_call>call:dummy1{config:{name:<|\"|>astro_config<|\"|>,value:99}}call:dummy2{config:{name:<|\"|>second_config<|\"|>,value:199}}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -212,7 +212,7 @@ TEST_F(Gemma4OutputParserTest, ParseTwoToolCallsAtOnce) { } TEST_F(Gemma4OutputParserTest, ParseToolCallWithArrayArguments) { - std::string inputWithProperClosure = "<|tool_call>call:sort{array:[42,17,89,5,33],order:<\">descending<\">}"; + std::string inputWithProperClosure = "<|tool_call>call:sort{array:[42,17,89,5,33],order:<|\"|>descending<|\"|>}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -231,7 +231,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithArrayArguments) { } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringWithSingleQuotesArguments) { - std::string inputWithProperClosure = "<|tool_call>call:sort{array:[42,17,89,5,33],order:'descending'}"; + std::string inputWithProperClosure = "<|tool_call>call:sort{array:[42,17,89,5,33],order:<|\"|>descending<|\"|>}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -250,9 +250,9 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringWithSingleQuotesArguments) } TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { - std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}" - "<|tool_call>call:another_tool{param1:<\">data<\">,param2:true}" - "<|tool_call>call:third_tool{key:<\">value<\">}"; + std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}" + "<|tool_call>call:another_tool{param1:<|\"|>data<|\"|>,param2:true}" + "<|tool_call>call:third_tool{key:<|\"|>value<|\"|>}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -285,11 +285,11 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCallsWithContentInBetween) { std::string inputWithProperClosure = "Before tool calls content. " - "<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}" + "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}" "This is some content between tool calls." - "<|tool_call>call:another_tool{param1:<\">data<\">,param2:true}" + "<|tool_call>call:another_tool{param1:<|\"|>data<|\"|>,param2:true}" " This is some content between second and third tool call. " - "<|tool_call>call:third_tool{key:<\">value<\">}" + "<|tool_call>call:third_tool{key:<|\"|>value<|\"|>}" "After tool calls content."; std::vector inputs = {inputWithProperClosure}; @@ -342,7 +342,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { } TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { - std::string input = "This is a content part and next will be a tool call.\n\n<|tool_call>call:example_tool{arg1:<\">value1<\">,arg2:42}"; + std::string input = "This is a content part and next will be a tool call.\n\n<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -374,10 +374,10 @@ TEST_F(Gemma4OutputParserTest, HolisticStreaming) { {" 33", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"order", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {":<\">", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<\">,", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, {"call:d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"{config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"dummy"}}]}})"}, @@ -454,7 +454,7 @@ TEST_F(Gemma4OutputParserTest, StreamingWithBiggerChunks) { {"MORE_CONTENT<|tool_call>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"MORE_CONTENT"}})"}, {"call:sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"{array:", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, - {"[42, 17, 89, 5, 33],order:<\">descending<\">", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[42, 17, 89, 5, 33],order:<|\"|>descending<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, @@ -531,10 +531,10 @@ TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { {" 33", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"order", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {":<\">", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<\">}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"<|\"|>}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"Some ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Some "}})"}, {"content ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"content "}})"}, @@ -563,7 +563,7 @@ TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { {"call:solve", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"{e", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"solve"}}]}})"}, {"quation", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {":<\">", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"2", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"*", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"(", ov::genai::GenerationFinishReason::NONE, std::nullopt}, @@ -572,7 +572,7 @@ TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { {"5)", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {" =", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {" 13", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<\">}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"equation\":\"2*(x+5) = 13\"}"}}]}})"}, + {"<|\"|>}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"equation\":\"2*(x+5) = 13\"}"}}]}})"}, {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"And some content after second tool call", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"And some content after second tool call"}})"}, }; @@ -635,9 +635,9 @@ TEST_F(Gemma4OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { {"call:super", "{\"delta\":{\"content\":\"call:super\"}}"}, {"_tool_number_two", "{\"delta\":{\"content\":\"_tool_number_two\"}}"}, {"{arg1", "{\"delta\":{\"content\":\"{arg1\"}}"}, - {":<\">", "{\"delta\":{\"content\":\":<\\\">\"}}"}, + {":<|\"|>", "{\"delta\":{\"content\":\":<\\\">\"}}"}, {"val{{{ue1", "{\"delta\":{\"content\":\"val{{{ue1\"}}"}, - {"<\">}", "{\"delta\":{\"content\":\"<\\\">}\"}}"}, + {"<|\"|>}", "{\"delta\":{\"content\":\"<\\\">}\"}}"}, {"", "{\"delta\":{\"content\":\"\"}}"}, }; @@ -659,7 +659,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingParentheses) { } TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingClosingParenthesis) { - std::string input = "<|tool_call>call:broken_tool{arg1:<\">value1<\">"; + std::string input = "<|tool_call>call:broken_tool{arg1:<|\"|>value1<|\"|>"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -679,7 +679,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithArgumentMissingEquals) { // Tests with special characters TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingComparison) { - std::string input = R"x(<|tool_call>call:search{query:<">price >= 100, (sale)<">,limit:5})x"; + std::string input = R"x(<|tool_call>call:search{query:<|"|>price >= 100, (sale)<|"|>,limit:5})x"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -690,7 +690,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingCompari } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesAndBrackets) { - std::string input = R"(<|tool_call>call:format{template:<">Hello {name}, items: [a, b, c]<">,count:3})"; + std::string input = R"(<|tool_call>call:format{template:<|"|>Hello {name}, items: [a, b, c]<|"|>,count:3})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -702,7 +702,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesA TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecialCharacters) { std::string impl = "import package\nimport package2\n\ndef func(a, b):\n\td={\"python\": \"dict\"}\n\tl = [\"list \\\"with escaped text\\\"\", 123, []]\n\treturn f\"formatted {a} and {b}\""; - std::string input = R"(<|tool_call>call:execute{code:<">)" + impl + R"(<">})"; + std::string input = R"(<|tool_call>call:execute{code:<|"|>)" + impl + R"(<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -713,7 +713,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecial } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingEscapedQuotes) { - std::string input = R"x(<|tool_call>call:execute{code:<">print(\"hello world\")<">,verbose:true})x"; + std::string input = R"x(<|tool_call>call:execute{code:<|"|>print(\"hello world\")<|"|>,verbose:true})x"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -724,7 +724,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingEscaped } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingApostrophes) { - std::string input = R"(<|tool_call>call:log{message:<">it's a test, isn't it?<">,level:<">warn<">})"; + std::string input = R"(<|tool_call>call:log{message:<|"|>it's a test, isn't it?<|"|>,level:<|"|>warn<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -735,7 +735,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingApostro } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBackslashes) { - std::string input = R"(<|tool_call>call:read_file{path:<">C:\Users\test\file.txt<">,encoding:<">utf-8<">})"; + std::string input = R"(<|tool_call>call:read_file{path:<|"|>C:\Users\test\file.txt<|"|>,encoding:<|"|>utf-8<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -768,7 +768,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStrings } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJSON) { - std::string input = R"(<|tool_call>call:send{payload:<">{'key': 'value', 'count': 42}<">,endpoint:<">api<">})"; + std::string input = R"(<|tool_call>call:send{payload:<|"|>{'key': 'value', 'count': 42}<|"|>,endpoint:<|"|>api<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -779,7 +779,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJ } TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyStringArgument) { - std::string input = R"(<|tool_call>call:create{name:<"><">,value:0})"; + std::string input = R"(<|tool_call>call:create{name:<|"|><|"|>,value:0})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -790,7 +790,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyStringArgument) { } TEST_F(Gemma4OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { - std::string input = R"(<|tool_call>call:translate{text:<">zażółć gęślą jaźń<">,lang:<">pl<">})"; + std::string input = R"(<|tool_call>call:translate{text:<|"|>zażółć gęślą jaźń<|"|>,lang:<|"|>pl<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); From bcaf3722dc5240d81f4efe27b90df708f9947268 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 27 Apr 2026 13:51:10 +0200 Subject: [PATCH 07/33] fix object parsing --- src/llm/io_processing/gemma4/tool_parser.cpp | 12 +++++++++--- .../llm/output_parsers/gemma4_output_parser_test.cpp | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 8b46695491..278910b911 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -73,8 +73,8 @@ std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { key = argumentStr.substr(pos, keyEndPos - pos); size_t valueStartPos = keyEndPos + 1; size_t valueEndPos; - if ( argumentStr.substr(keyEndPos + 2, TOOL_ARGS_STRING_INDICATOR.size()) == TOOL_ARGS_STRING_INDICATOR) { - valueStartPos = keyEndPos + 2 + TOOL_ARGS_STRING_INDICATOR.size(); + if (argumentStr.substr(valueStartPos, TOOL_ARGS_STRING_INDICATOR.size()) == TOOL_ARGS_STRING_INDICATOR) { + valueStartPos = valueStartPos + TOOL_ARGS_STRING_INDICATOR.size(); valueEndPos = argumentStr.find(TOOL_ARGS_STRING_INDICATOR, valueStartPos); isStringValue = true; } else { @@ -89,7 +89,13 @@ std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { value = "\"" + value + "\""; } keyValuePairs.emplace_back(key, value); - pos = (valueEndPos == argumentStr.size() - 1) ? std::string::npos : valueEndPos + 1; + if (valueEndPos == argumentStr.size() - 1) { + break; + } else if (isStringValue) { + pos = valueEndPos + TOOL_ARGS_STRING_INDICATOR.size() + 1; + } else { + pos = valueEndPos + 1; + } } if (keyValuePairs.empty()) { diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index da46f7e0e7..19a5284bb4 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -190,7 +190,7 @@ TEST_F(Gemma4OutputParserTest, ParserToolCallWithBooleanArgument) { } TEST_F(Gemma4OutputParserTest, ParseTwoToolCallsAtOnce) { - std::string inputWithProperClosure = "<|tool_call>call:dummy1{config:{name:<|\"|>astro_config<|\"|>,value:99}}call:dummy2{config:{name:<|\"|>second_config<|\"|>,value:199}}"; + std::string inputWithProperClosure = "<|tool_call>call:dummy1{config:{name:<|\"|>astro_config<|\"|>,value:99}}call:dummy2{config:{value:199,name:<|\"|>second_config<|\"|>}}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -205,7 +205,7 @@ TEST_F(Gemma4OutputParserTest, ParseTwoToolCallsAtOnce) { EXPECT_EQ(parsedOutput.toolCalls[1].name, "dummy2"); // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); - EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"config\":{\"name\":\"second_config\",\"value\":199}}"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"config\":{\"value\":199,\"name\":\"second_config\"}}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); // ID should be generated } From 5a7e40ad933811ac348dc9fe430eed6657117c92 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Tue, 28 Apr 2026 10:58:37 +0200 Subject: [PATCH 08/33] streaming and tests fixes --- src/llm/io_processing/gemma4/tool_parser.cpp | 6 ++- src/llm/io_processing/utils.cpp | 8 ++-- .../gemma4_output_parser_test.cpp | 39 +++++++++---------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 278910b911..4695f93077 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -152,7 +152,7 @@ std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { size_t errorOffset = tempDoc.GetErrorOffset(); SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse argument string as JSON. Argument string: {}, Error: {} Offset: {}", normalized, errorMessage, errorOffset); - if (first == '\"' && last == '\"') { + if (normalized.front() == '\"' && normalized.back() == '\"') { normalized = normalized.substr(1, normalized.size() - 2); } finalValue.SetString(normalized.c_str(), static_cast(normalized.size()), tempDoc.GetAllocator()); @@ -261,8 +261,12 @@ bool Gemma4ToolParser::parseInToolCallState() { } bool Gemma4ToolParser::parseToolCallParametersState() { + if(this->streamingContent.back() == TOOL_ARGS_END_INDICATOR.back()) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool arguments end indicator found at the end of streaming content, attempting to parse arguments: {}", this->streamingContent.substr(this->streamingPosition)); + } size_t pos = findInStringRespectingSpecialChars(this->streamingContent, TOOL_ARGS_END_INDICATOR, this->streamingPosition); if (pos == std::string::npos) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool arguments end indicator not found in streaming content starting from position: {}", this->streamingPosition); return false; } std::string argumentsStr = this->streamingContent.substr(this->streamingPosition, pos - this->streamingPosition); diff --git a/src/llm/io_processing/utils.cpp b/src/llm/io_processing/utils.cpp index 104e446739..e48ef42c99 100644 --- a/src/llm/io_processing/utils.cpp +++ b/src/llm/io_processing/utils.cpp @@ -96,6 +96,11 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str int singleQuoteDepth = 0; for (size_t i = startPos; i < str.length(); ++i) { + if (bracketDepth == 0 && braceDepth == 0 && quoteDepth == 0 && singleQuoteDepth == 0 && + str.compare(i, target.length(), target) == 0) { + return i; + } + if (str[i] == '{') { braceDepth++; } else if (str[i] == '}') { @@ -108,9 +113,6 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str quoteDepth = 1 - quoteDepth; } else if (str[i] == '\'' && (i == 0 || str[i - 1] != '\\')) { singleQuoteDepth = 1 - singleQuoteDepth; - } else if (bracketDepth == 0 && braceDepth == 0 && quoteDepth == 0 && singleQuoteDepth == 0 && - str.compare(i, target.length(), target) == 0) { - return i; } } return std::string::npos; diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 19a5284bb4..187b32a1b5 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -103,7 +103,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { std::string testInput = input; - auto generatedTensor = gemma4Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(testInput, ov::genai::add_special_tokens(true)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); EXPECT_EQ(parsedOutput.content, testInput); @@ -377,21 +377,21 @@ TEST_F(Gemma4OutputParserTest, HolisticStreaming) { {":<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<|\"|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, {"call:d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"{config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"dummy"}}]}})"}, {":{", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"}}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, @@ -546,16 +546,15 @@ TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"{config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"dummy"}}]}})"}, {":{", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"}}", ov ::genai ::GenerationFinishReason ::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, @@ -635,9 +634,9 @@ TEST_F(Gemma4OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { {"call:super", "{\"delta\":{\"content\":\"call:super\"}}"}, {"_tool_number_two", "{\"delta\":{\"content\":\"_tool_number_two\"}}"}, {"{arg1", "{\"delta\":{\"content\":\"{arg1\"}}"}, - {":<|\"|>", "{\"delta\":{\"content\":\":<\\\">\"}}"}, + {":<|\"|>", "{\"delta\":{\"content\":\":<|\\\"|>\"}}"}, {"val{{{ue1", "{\"delta\":{\"content\":\"val{{{ue1\"}}"}, - {"<|\"|>}", "{\"delta\":{\"content\":\"<\\\">}\"}}"}, + {"<|\"|>}", "{\"delta\":{\"content\":\"<|\\\"|>}\"}}"}, {"", "{\"delta\":{\"content\":\"\"}}"}, }; @@ -746,7 +745,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBacksla } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsContainingQuotes) { - std::string input = R"(<|tool_call>call:save{lines:['it's the wonderful day','My name's Jan','That's Johns' car.']})"; + std::string input = R"(<|tool_call>call:save{lines:[<|"|>it's the wonderful day<|"|>,<|"|>My name's Jan<|"|>,<|"|>That's Johns' car.<|"|>]})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); @@ -757,7 +756,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsC } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStringsContainingQuotes) { - std::string input = R"(<|tool_call>call:save{obj:{'name':'it's the wonderful day','greeting':'Hello, my name's Jan','note':'That's Johns' car.'}})"; + std::string input = R"(<|tool_call>call:save{obj:{name:<|"|>it's the wonderful day<|"|>,greeting:<|"|>Hello, my name's Jan<|"|>,note:<|"|>That's Johns' car.<|"|>}})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); From 9d473fcb2d28592d3dead6487bcab0fd575e0f65 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Tue, 28 Apr 2026 13:28:59 +0200 Subject: [PATCH 09/33] enabling all tests --- src/llm/io_processing/gemma4/tool_parser.cpp | 40 +++++++++++-------- .../gemma4_output_parser_test.cpp | 12 +++--- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 4695f93077..da54e841e4 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -37,26 +37,34 @@ const int64_t Gemma4ToolParser::botTokenId = 48; const int64_t Gemma4ToolParser::eotTokenId = 49; std::string Gemma4ToolParser::parseArrayParameter(std::string argumentStr) { - int quoteDepth = 0; - - for (size_t i = 1; i < argumentStr.size() - 1; ++i) { - if (argumentStr[i] != '\'') { - continue; + size_t pos = 1; + std::string parsedArguments = "["; + + while (pos != std::string::npos) { + size_t stringStartPos = argumentStr.find(TOOL_ARGS_STRING_INDICATOR, pos); + if (stringStartPos == std::string::npos) { + break; + } + stringStartPos += TOOL_ARGS_STRING_INDICATOR.size(); + size_t stringEndPos = argumentStr.find(TOOL_ARGS_STRING_INDICATOR, stringStartPos); + if (stringEndPos == std::string::npos) { + break; } - bool isLastElement = (i == argumentStr.size() - 2); - bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; - - if (quoteDepth == 0) { - argumentStr[i] = '"'; - quoteDepth++; - } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement)) { - argumentStr[i] = '"'; - quoteDepth--; + std::string originalStr = argumentStr.substr(stringStartPos, stringEndPos - stringStartPos); + size_t quotePos = 0; + while ((quotePos = originalStr.find('\"', quotePos)) != std::string::npos) { + originalStr.insert(quotePos, "\\"); + quotePos += 2; } + parsedArguments += "\"" + originalStr + "\","; + + pos = stringEndPos + TOOL_ARGS_STRING_INDICATOR.size() + 1; } - return argumentStr; + parsedArguments.back() = ']'; + + return parsedArguments; } std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { @@ -132,7 +140,7 @@ std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument contains is an object, changed it to correct JSON format. Modified string: {}", normalized); } - if (first == '[' && last == ']') { + if (first == '[' && last == ']' && normalized.find(TOOL_ARGS_STRING_INDICATOR) != std::string::npos) { normalized = parseArrayParameter(normalized); SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an array, normalized quotes for JSON parsing. Modified string: {}", normalized); } diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 187b32a1b5..2b0ae4c7ea 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -99,14 +99,14 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; + std::string inputWithoutSpecialTokens = "call:example_tool{arg1:value1,arg2:42}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - std::string testInput = input; - auto generatedTensor = gemma4Tokenizer->encode(testInput, ov::genai::add_special_tokens(true)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(true)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); - EXPECT_EQ(parsedOutput.content, testInput); + EXPECT_EQ(parsedOutput.content, inputWithoutSpecialTokens); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); @@ -152,7 +152,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArguments) { } TEST_F(Gemma4OutputParserTest, ParseToolCallWithListOfStringsAsArgument) { - std::string inputWithProperClosure = "<|tool_call>call:generate_DNA_sequence{length:100,preferences:['G','C']}"; + std::string inputWithProperClosure = "<|tool_call>call:generate_DNA_sequence{length:100,preferences:[<|\"|>G<|\"|>,<|\"|>C<|\"|>]}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -745,14 +745,14 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBacksla } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsContainingQuotes) { - std::string input = R"(<|tool_call>call:save{lines:[<|"|>it's the wonderful day<|"|>,<|"|>My name's Jan<|"|>,<|"|>That's Johns' car.<|"|>]})"; + std::string input = R"(<|tool_call>call:save{lines:[<|"|>it's the wonderful day<|"|>,<|"|>He said: "My name's John"<|"|>,<|"|>That's Johns' car.<|"|>]})"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"lines":["it's the wonderful day","My name's Jan","That's Johns' car."]})"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"lines":["it's the wonderful day","He said: \"My name's John\"","That's Johns' car."]})"); } TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStringsContainingQuotes) { From 047cd5f02844b416c95aed99713cff165bf5c60a Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Wed, 6 May 2026 10:43:10 +0200 Subject: [PATCH 10/33] streaming workaround for unary --- .../visual_language_model/legacy/servable.cpp | 37 ++++++++++++++----- src/test/http_openai_handler_test.cpp | 3 +- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index 6297745360..f2ca400332 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -179,23 +179,40 @@ absl::Status VisualLanguageModelLegacyServable::readCompleteExecutionResults(std absl::Status VisualLanguageModelLegacyServable::prepareCompleteResponse(std::shared_ptr& executionContext) { auto legacyExecutionContext = std::static_pointer_cast(executionContext); - if (legacyExecutionContext->payload.client->isDisconnected()) { - return absl::CancelledError(); + + // temporary workaround to use streaming logic in unary + // to be fixed after require_special_tokens flag implemented + std::string completeText; + auto generationStatus = legacyExecutionContext->finished.wait_for(std::chrono::nanoseconds::zero()); + + while (generationStatus != std::future_status::ready) { + if (legacyExecutionContext->payload.client->isDisconnected()) { + return absl::CancelledError(); + } + std::unique_lock lock(legacyExecutionContext->mutex); + while (executionContext->lastStreamerCallbackOutput.size() == 0 && generationStatus != std::future_status::ready) { + legacyExecutionContext->executionInProgress.wait_for(lock, std::chrono::milliseconds(10)); + generationStatus = legacyExecutionContext->finished.wait_for(std::chrono::nanoseconds::zero()); + } + completeText += executionContext->lastStreamerCallbackOutput; + executionContext->lastStreamerCallbackOutput = ""; + generationStatus = legacyExecutionContext->finished.wait_for(std::chrono::nanoseconds::zero()); } - // TODO(mzegla): Usage of streaming flow here due to GenAI generate limitations. - // This diverges from the general flow - we should have unified systematic approach. + if (!legacyExecutionContext->success) { + return absl::InvalidArgumentError("Request processing failed, check its correctness."); + } executionContext->textStreamer->end(); - - std::string completeText; { - std::lock_guard lock(legacyExecutionContext->mutex); - completeText = std::move(executionContext->lastStreamerCallbackOutput); - executionContext->lastStreamerCallbackOutput.clear(); + std::unique_lock lock(legacyExecutionContext->mutex); + completeText += executionContext->lastStreamerCallbackOutput; + executionContext->lastStreamerCallbackOutput = ""; } - executionContext->response = executionContext->apiHandler->serializeUnaryResponse(legacyExecutionContext->results, completeText); + executionContext->apiHandler->setPromptTokensUsage(legacyExecutionContext->results.perf_metrics.get_num_input_tokens()); + executionContext->apiHandler->setCompletionTokensUsage(legacyExecutionContext->results.perf_metrics.get_num_generated_tokens()); + executionContext->response = executionContext->apiHandler->serializeUnaryResponse(legacyExecutionContext->results, completeText); SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Complete unary response: {}", executionContext->response); return absl::OkStatus(); } diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index c3a40cba3c..289beffb63 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -3090,7 +3090,8 @@ TEST_F(HttpOpenAIHandlerParsingTest, SerializeUnaryResponseVLMDecodedResultsWith doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer, "hermes3", ""); ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - + + std::string toolCallContent = "I will call a tool.{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Paris\"}}"; ov::genai::VLMDecodedResults results; std::string vlmText = "I will call a tool.{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Paris\"}}"; From e92a335f77b628ecbda82a1774834c82d6ca1325 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Wed, 6 May 2026 10:53:02 +0200 Subject: [PATCH 11/33] style --- spelling-whitelist.txt | 1 + src/llm/io_processing/gemma4/tool_parser.cpp | 9 ++++----- src/llm/io_processing/utils.cpp | 2 +- src/llm/visual_language_model/legacy/servable.cpp | 2 +- src/test/http_openai_handler_test.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index bd12dae11c..85763a1742 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -29,3 +29,4 @@ demos/vlm_npu/README.md:157: mane ==> main, many, maine demos/vlm_npu/README.md:218: mane ==> main, many, maine demos/integration_with_OpenWebUI/README.md:423: Buildin ==> Building, Build in src/test/llm/output_parsers/lfm2_output_parser_test.cpp +src/test/llm/output_parsers/gemma4_output_parser_test.cpp diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index da54e841e4..4cb56d0ba5 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -39,7 +39,7 @@ const int64_t Gemma4ToolParser::eotTokenId = 49; std::string Gemma4ToolParser::parseArrayParameter(std::string argumentStr) { size_t pos = 1; std::string parsedArguments = "["; - + while (pos != std::string::npos) { size_t stringStartPos = argumentStr.find(TOOL_ARGS_STRING_INDICATOR, pos); if (stringStartPos == std::string::npos) { @@ -86,9 +86,9 @@ std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { valueEndPos = argumentStr.find(TOOL_ARGS_STRING_INDICATOR, valueStartPos); isStringValue = true; } else { - valueEndPos = argumentStr.find(',', valueStartPos); + valueEndPos = argumentStr.find(',', valueStartPos); } - + if (valueEndPos == std::string::npos) { valueEndPos = argumentStr.size() - 1; } @@ -116,7 +116,6 @@ std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { } parsedObject.back() = '}'; return parsedObject; - } std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { @@ -269,7 +268,7 @@ bool Gemma4ToolParser::parseInToolCallState() { } bool Gemma4ToolParser::parseToolCallParametersState() { - if(this->streamingContent.back() == TOOL_ARGS_END_INDICATOR.back()) { + if (this->streamingContent.back() == TOOL_ARGS_END_INDICATOR.back()) { SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool arguments end indicator found at the end of streaming content, attempting to parse arguments: {}", this->streamingContent.substr(this->streamingPosition)); } size_t pos = findInStringRespectingSpecialChars(this->streamingContent, TOOL_ARGS_END_INDICATOR, this->streamingPosition); diff --git a/src/llm/io_processing/utils.cpp b/src/llm/io_processing/utils.cpp index e48ef42c99..bc41094762 100644 --- a/src/llm/io_processing/utils.cpp +++ b/src/llm/io_processing/utils.cpp @@ -100,7 +100,7 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str str.compare(i, target.length(), target) == 0) { return i; } - + if (str[i] == '{') { braceDepth++; } else if (str[i] == '}') { diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index f2ca400332..c6b4ce88b6 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -181,7 +181,7 @@ absl::Status VisualLanguageModelLegacyServable::prepareCompleteResponse(std::sha auto legacyExecutionContext = std::static_pointer_cast(executionContext); // temporary workaround to use streaming logic in unary - // to be fixed after require_special_tokens flag implemented + // to be fixed after require_special_tokens flag implemented std::string completeText; auto generationStatus = legacyExecutionContext->finished.wait_for(std::chrono::nanoseconds::zero()); diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 289beffb63..219c8955d7 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -3090,7 +3090,7 @@ TEST_F(HttpOpenAIHandlerParsingTest, SerializeUnaryResponseVLMDecodedResultsWith doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer, "hermes3", ""); ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - + std::string toolCallContent = "I will call a tool.{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Paris\"}}"; ov::genai::VLMDecodedResults results; std::string vlmText = From 80def303d0dd2415cadfad87ac5adbbea47a6bb2 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Wed, 6 May 2026 10:57:22 +0200 Subject: [PATCH 12/33] cpplint --- src/llm/io_processing/gemma4/tool_parser.cpp | 2 +- src/llm/io_processing/gemma4/tool_parser.hpp | 5 +++-- src/test/llm/output_parsers/gemma4_output_parser_test.cpp | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 4cb56d0ba5..9388bd82ce 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -504,4 +504,4 @@ void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector #include +#include #include "src/llm/io_processing/base_output_parser.hpp" namespace ovms { @@ -63,7 +64,7 @@ class Gemma4ToolParser : public BaseOutputParser { } bool requiresStreamingWithSpecialTokens() const override { - return true; //to be checked if it's actually required + return true; } static std::string normalizeArgStr(const std::string& arg); @@ -92,4 +93,4 @@ class Gemma4ToolParser : public BaseOutputParser { ToolCall toolCall; int toolCallIndex{-1}; }; -} // namespace ovms \ No newline at end of file +} // namespace ovms diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 2b0ae4c7ea..a1025c0bc3 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -797,4 +797,4 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"text":"zażółć gęślą jaźń","lang":"pl"})"); -} \ No newline at end of file +} From 16f32941aaa7f1fe0a814e4eea9bb51a93f0a1bb Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Wed, 6 May 2026 14:41:40 +0200 Subject: [PATCH 13/33] review changes --- src/llm/io_processing/gemma4/tool_parser.cpp | 2 +- src/llm/io_processing/gemma4/tool_parser.hpp | 2 +- .../gemma4_output_parser_test.cpp | 106 ++++++------------ 3 files changed, 37 insertions(+), 73 deletions(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 9388bd82ce..53385c334d 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -36,7 +36,7 @@ const std::string Gemma4ToolParser::TOOL_ARGS_SEPARATOR_STR = ","; const int64_t Gemma4ToolParser::botTokenId = 48; const int64_t Gemma4ToolParser::eotTokenId = 49; -std::string Gemma4ToolParser::parseArrayParameter(std::string argumentStr) { +std::string Gemma4ToolParser::parseArrayParameter(const std::string& argumentStr) { size_t pos = 1; std::string parsedArguments = "["; diff --git a/src/llm/io_processing/gemma4/tool_parser.hpp b/src/llm/io_processing/gemma4/tool_parser.hpp index 493aedc410..aebe7c5fa3 100644 --- a/src/llm/io_processing/gemma4/tool_parser.hpp +++ b/src/llm/io_processing/gemma4/tool_parser.hpp @@ -68,7 +68,7 @@ class Gemma4ToolParser : public BaseOutputParser { } static std::string normalizeArgStr(const std::string& arg); - static std::string parseArrayParameter(std::string argumentStr); + static std::string parseArrayParameter(const std::string& argumentStr); static std::string parseObjectParameter(std::string argumentStr); private: diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index a1025c0bc3..5ed31005e5 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -83,7 +83,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -91,9 +91,8 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } } @@ -103,7 +102,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(true)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); EXPECT_EQ(parsedOutput.content, inputWithoutSpecialTokens); @@ -118,7 +117,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithObjectArguments) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -126,9 +125,8 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithObjectArguments) { ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "dummy"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } } @@ -137,7 +135,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArguments) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -145,9 +143,8 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArguments) { ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "test1"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"data1,data2\"}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } } @@ -156,7 +153,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithListOfStringsAsArgument) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -164,9 +161,8 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithListOfStringsAsArgument) { ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "generate_DNA_sequence"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"length\":100,\"preferences\":[\"G\",\"C\"]}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } } @@ -175,7 +171,7 @@ TEST_F(Gemma4OutputParserTest, ParserToolCallWithBooleanArgument) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -183,9 +179,8 @@ TEST_F(Gemma4OutputParserTest, ParserToolCallWithBooleanArgument) { ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "check_status"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"flag\":true}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } } @@ -194,7 +189,7 @@ TEST_F(Gemma4OutputParserTest, ParseTwoToolCallsAtOnce) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -203,11 +198,10 @@ TEST_F(Gemma4OutputParserTest, ParseTwoToolCallsAtOnce) { ASSERT_EQ(parsedOutput.toolCalls.size(), 2); EXPECT_EQ(parsedOutput.toolCalls[0].name, "dummy1"); EXPECT_EQ(parsedOutput.toolCalls[1].name, "dummy2"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"config\":{\"value\":199,\"name\":\"second_config\"}}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated - EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); } } @@ -216,26 +210,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithArrayArguments) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; - std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - EXPECT_EQ(parsedOutput.content, ""); - EXPECT_EQ(parsedOutput.reasoning, ""); - - ASSERT_EQ(parsedOutput.toolCalls.size(), 1); - EXPECT_EQ(parsedOutput.toolCalls[0].name, "sort"); - // Parser removes whitespaces, so we expect arguments value to be without spaces - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated - } -} - -TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringWithSingleQuotesArguments) { - std::string inputWithProperClosure = "<|tool_call>call:sort{array:[42,17,89,5,33],order:<|\"|>descending<|\"|>}"; - - std::vector inputs = {inputWithProperClosure}; - for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -243,9 +218,8 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringWithSingleQuotesArguments) ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "sort"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } } @@ -256,7 +230,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -294,7 +268,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCallsWithContentI std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, "Before tool calls content. This is some content between tool calls. This is some content between second and third tool call. After tool calls content."); @@ -324,7 +298,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCallsWithContentI TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyArguments) { // Tool call with empty braces (no arguments) std::string input = "<|tool_call>call:no_args_tool{}"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -333,7 +307,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyArguments) { TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); @@ -343,7 +317,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { std::string input = "This is a content part and next will be a tool call.\n\n<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); @@ -470,7 +444,6 @@ TEST_F(Gemma4OutputParserTest, StreamingWithBiggerChunks) { rapidjson::Writer writer(buffer); doc->Accept(writer); std::string docStr = buffer.GetString(); - // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; auto docIdPos = docStr.find(idKey); @@ -486,7 +459,6 @@ TEST_F(Gemma4OutputParserTest, StreamingWithBiggerChunks) { std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; - // Compare everything except the id value std::string docStrNoId = docStr; std::string expectedNoId = expected; docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); @@ -513,8 +485,6 @@ TEST_F(Gemma4OutputParserTest, StreamingWithBiggerChunks) { TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { std::vector>> chunkToDeltaVec{ - // Tool call phase - // Starting first tool. Collecting chunk until full name is received. Don't return until then. {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"call:sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, @@ -586,7 +556,6 @@ TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { rapidjson::Writer writer(buffer); doc->Accept(writer); std::string docStr = buffer.GetString(); - // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; auto docIdPos = docStr.find(idKey); @@ -629,7 +598,6 @@ TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { TEST_F(Gemma4OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { std::vector>> chunkToDeltaVec{ - // Tool parser is available, but tools are not in the request so every chunk is just a regular content {"<|tool_call>", "{\"delta\":{\"content\":\"<|tool_call>\"}}"}, {"call:super", "{\"delta\":{\"content\":\"call:super\"}}"}, {"_tool_number_two", "{\"delta\":{\"content\":\"_tool_number_two\"}}"}, @@ -641,7 +609,6 @@ TEST_F(Gemma4OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { }; for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { - // Second argument is false as we simulate the case where tools have not been provided in the request std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, false, ov::genai::GenerationFinishReason::NONE); assertChunkEqual(doc, expectedDelta, chunk); } @@ -651,7 +618,7 @@ TEST_F(Gemma4OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingParentheses) { std::string input = "<|tool_call>call:broken_tool"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); @@ -659,27 +626,24 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingParentheses) { TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingClosingParenthesis) { std::string input = "<|tool_call>call:broken_tool{arg1:<|\"|>value1<|\"|>"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } TEST_F(Gemma4OutputParserTest, ParseToolCallWithArgumentMissingEquals) { - // Argument without ':' sign - parseSingleArgument sets value as empty std::string input = "<|tool_call>call:broken{malformed_arg}"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - // The tool call is parsed but the argument value will be empty and invalid ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); } -// Tests with special characters TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingComparison) { std::string input = R"x(<|tool_call>call:search{query:<|"|>price >= 100, (sale)<|"|>,limit:5})x"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -690,7 +654,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingCompari TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesAndBrackets) { std::string input = R"(<|tool_call>call:format{template:<|"|>Hello {name}, items: [a, b, c]<|"|>,count:3})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -702,7 +666,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesA TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecialCharacters) { std::string impl = "import package\nimport package2\n\ndef func(a, b):\n\td={\"python\": \"dict\"}\n\tl = [\"list \\\"with escaped text\\\"\", 123, []]\n\treturn f\"formatted {a} and {b}\""; std::string input = R"(<|tool_call>call:execute{code:<|"|>)" + impl + R"(<|"|>})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -713,7 +677,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecial TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingEscapedQuotes) { std::string input = R"x(<|tool_call>call:execute{code:<|"|>print(\"hello world\")<|"|>,verbose:true})x"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -724,7 +688,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingEscaped TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingApostrophes) { std::string input = R"(<|tool_call>call:log{message:<|"|>it's a test, isn't it?<|"|>,level:<|"|>warn<|"|>})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -735,7 +699,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingApostro TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBackslashes) { std::string input = R"(<|tool_call>call:read_file{path:<|"|>C:\Users\test\file.txt<|"|>,encoding:<|"|>utf-8<|"|>})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -746,7 +710,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBacksla TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsContainingQuotes) { std::string input = R"(<|tool_call>call:save{lines:[<|"|>it's the wonderful day<|"|>,<|"|>He said: "My name's John"<|"|>,<|"|>That's Johns' car.<|"|>]})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -757,7 +721,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsC TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStringsContainingQuotes) { std::string input = R"(<|tool_call>call:save{obj:{name:<|"|>it's the wonderful day<|"|>,greeting:<|"|>Hello, my name's Jan<|"|>,note:<|"|>That's Johns' car.<|"|>}})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -768,7 +732,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStrings TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJSON) { std::string input = R"(<|tool_call>call:send{payload:<|"|>{'key': 'value', 'count': 42}<|"|>,endpoint:<|"|>api<|"|>})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -779,7 +743,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJ TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyStringArgument) { std::string input = R"(<|tool_call>call:create{name:<|"|><|"|>,value:0})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); @@ -790,7 +754,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyStringArgument) { TEST_F(Gemma4OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { std::string input = R"(<|tool_call>call:translate{text:<|"|>zażółć gęślą jaźń<|"|>,lang:<|"|>pl<|"|>})"; - auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); EXPECT_EQ(parsedOutput.content, ""); From 8a1b51bbdcac640dc8e2d8603adc3d1133372d3a Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Thu, 7 May 2026 08:19:41 +0200 Subject: [PATCH 14/33] simplify vlm unary wa, change model for tests --- prepare_llm_models.sh | 5 +--- .../visual_language_model/legacy/servable.cpp | 30 +++---------------- .../gemma4_output_parser_test.cpp | 4 +-- windows_prepare_llm_models.bat | 2 +- 4 files changed, 8 insertions(+), 33 deletions(-) diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index 4f0acf4e3c..b74c4db9a3 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -38,11 +38,8 @@ PHI4_MODEL="microsoft/Phi-4-mini-instruct" MISTRAL_MODEL="mistralai/Mistral-7B-Instruct-v0.3" GPT_OSS_MODEL="openai/gpt-oss-20b" DEVSTRAL_MODEL="unsloth/Devstral-Small-2507" -<<<<<<< HEAD LFM2_MODEL="LiquidAI/LFM2-2.6B" -======= -GEMMA4_MODEL="google/gemma-4-26B-A4B-it" ->>>>>>> 73a55b89 (save) +GEMMA4_MODEL="OpenVINO/gemma-4-E4B-it-int4-ov" if [ "$(python3 -c 'import sys; print(sys.version_info[1])')" -le "8" ]; then echo "Prepare models with python > 3.8."; exit 1 ; fi diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index c6b4ce88b6..f267b17294 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -179,35 +179,13 @@ absl::Status VisualLanguageModelLegacyServable::readCompleteExecutionResults(std absl::Status VisualLanguageModelLegacyServable::prepareCompleteResponse(std::shared_ptr& executionContext) { auto legacyExecutionContext = std::static_pointer_cast(executionContext); + executionContext->textStreamer->end(); - // temporary workaround to use streaming logic in unary - // to be fixed after require_special_tokens flag implemented std::string completeText; - auto generationStatus = legacyExecutionContext->finished.wait_for(std::chrono::nanoseconds::zero()); - - while (generationStatus != std::future_status::ready) { - if (legacyExecutionContext->payload.client->isDisconnected()) { - return absl::CancelledError(); - } - std::unique_lock lock(legacyExecutionContext->mutex); - while (executionContext->lastStreamerCallbackOutput.size() == 0 && generationStatus != std::future_status::ready) { - legacyExecutionContext->executionInProgress.wait_for(lock, std::chrono::milliseconds(10)); - generationStatus = legacyExecutionContext->finished.wait_for(std::chrono::nanoseconds::zero()); - } - completeText += executionContext->lastStreamerCallbackOutput; - executionContext->lastStreamerCallbackOutput = ""; - generationStatus = legacyExecutionContext->finished.wait_for(std::chrono::nanoseconds::zero()); - } - - if (!legacyExecutionContext->success) { - return absl::InvalidArgumentError("Request processing failed, check its correctness."); - } - - executionContext->textStreamer->end(); { - std::unique_lock lock(legacyExecutionContext->mutex); - completeText += executionContext->lastStreamerCallbackOutput; - executionContext->lastStreamerCallbackOutput = ""; + std::lock_guard lock(legacyExecutionContext->mutex); + completeText = std::move(executionContext->lastStreamerCallbackOutput); + executionContext->lastStreamerCallbackOutput.clear(); } executionContext->apiHandler->setPromptTokensUsage(legacyExecutionContext->results.perf_metrics.get_num_input_tokens()); diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 5ed31005e5..092fe508c3 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -29,10 +29,10 @@ using namespace ovms; #ifdef _WIN32 -const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\google\\gemma-4-26B-A4B-it"; +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\OpenVINO\\gemma-4-E4B-it-int4-ov"; #else // Hardcoded for usage in docker container -const std::string tokenizerPath = "/ovms/src/test/llm_testing/google/gemma-4-26B-A4B-it"; +const std::string tokenizerPath = "/ovms/src/test/llm_testing/OpenVINO/gemma-4-E4B-it-int4-ov"; #endif static std::unique_ptr gemma4Tokenizer; diff --git a/windows_prepare_llm_models.bat b/windows_prepare_llm_models.bat index 66bc898272..156159f2c8 100644 --- a/windows_prepare_llm_models.bat +++ b/windows_prepare_llm_models.bat @@ -45,7 +45,7 @@ set "MISTRAL_MODEL=mistralai/Mistral-7B-Instruct-v0.3" set "GPTOSS_MODEL=openai/gpt-oss-20b" set "DEVSTRAL_MODEL=unsloth/Devstral-Small-2507" set "LFM2_MODEL=LiquidAI/LFM2-2.6B" -set "GEMMA4_MODEL=google/gemma-4-26B-A4B-it" +set "GEMMA4_MODEL=OpenVINO/gemma-4-E4B-it-int4-ov" echo Downloading LLM testing models to directory %~1 set "PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cpu https://storage.openvinotoolkit.org/simple/wheels/nightly" From 0417f8b52efb9a1379356808812842fdc1e811ee Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Thu, 7 May 2026 10:56:51 +0200 Subject: [PATCH 15/33] changing exports --- prepare_llm_models.sh | 3 +-- windows_prepare_llm_models.bat | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index b74c4db9a3..a27426dbbb 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -233,8 +233,7 @@ fi if [ -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then echo "Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." else - mkdir -p $1/$GEMMA4_MODEL - convert_tokenizer $GEMMA4_MODEL --with_detokenizer -o $1/$GEMMA4_MODEL + hf download "$GEMMA4_MODEL" --local-dir $1/$GEMMA4_MODEL fi if [ ! -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then echo "[ERROR] Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE does not exist." diff --git a/windows_prepare_llm_models.bat b/windows_prepare_llm_models.bat index 156159f2c8..f5e7fdefd8 100644 --- a/windows_prepare_llm_models.bat +++ b/windows_prepare_llm_models.bat @@ -86,7 +86,7 @@ call :download_tokenizer "%MISTRAL_MODEL%" "%~1\%MISTRAL_MODEL%" call :download_tokenizer "%GPTOSS_MODEL%" "%~1\%GPTOSS_MODEL%" call :download_tokenizer "%DEVSTRAL_MODEL%" "%~1\%DEVSTRAL_MODEL%" call :download_tokenizer "%LFM2_MODEL%" "%~1\%LFM2_MODEL%" -call :download_tokenizer "%GEMMA4_MODEL%" "%~1\%GEMMA4_MODEL%" +call :download_openvino "%GEMMA4_MODEL%" "%~1" exit /b 0 @@ -127,7 +127,9 @@ if not exist "%repository%\%model%\openvino_tokenizer.bin" ( echo Downloading model to %repository%\%model% directory. hf download "%model%" --local-dir "%repository%\%model%" :: WA to use newer tokenizer model format which supports padding. - convert_tokenizer "%~3" --with_detokenizer -o "%~2\%~1" + if not "%~3"=="" ( + convert_tokenizer "%~3" --with_detokenizer -o "%~2\%~1" + ) ) else ( echo Models file %repository%\%model%\openvino_tokenizer.bin exists. Skipping downloading models. ) From 0258121ffbbab5828caf2a30c0873fba743a7b03 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Thu, 7 May 2026 11:14:41 +0200 Subject: [PATCH 16/33] downloading only tokenizer --- prepare_llm_models.sh | 2 +- windows_prepare_llm_models.bat | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index a27426dbbb..010104c721 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -233,7 +233,7 @@ fi if [ -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then echo "Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." else - hf download "$GEMMA4_MODEL" --local-dir $1/$GEMMA4_MODEL + hf download "$GEMMA4_MODEL" --local-dir $1/$GEMMA4_MODEL --include *tokenizer* --exclude *detokenizer* fi if [ ! -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then echo "[ERROR] Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE does not exist." diff --git a/windows_prepare_llm_models.bat b/windows_prepare_llm_models.bat index f5e7fdefd8..67f44a2214 100644 --- a/windows_prepare_llm_models.bat +++ b/windows_prepare_llm_models.bat @@ -86,7 +86,7 @@ call :download_tokenizer "%MISTRAL_MODEL%" "%~1\%MISTRAL_MODEL%" call :download_tokenizer "%GPTOSS_MODEL%" "%~1\%GPTOSS_MODEL%" call :download_tokenizer "%DEVSTRAL_MODEL%" "%~1\%DEVSTRAL_MODEL%" call :download_tokenizer "%LFM2_MODEL%" "%~1\%LFM2_MODEL%" -call :download_openvino "%GEMMA4_MODEL%" "%~1" +call :download_openvino_tokenizer "%GEMMA4_MODEL%" "%~1" exit /b 0 @@ -127,14 +127,24 @@ if not exist "%repository%\%model%\openvino_tokenizer.bin" ( echo Downloading model to %repository%\%model% directory. hf download "%model%" --local-dir "%repository%\%model%" :: WA to use newer tokenizer model format which supports padding. - if not "%~3"=="" ( - convert_tokenizer "%~3" --with_detokenizer -o "%~2\%~1" - ) + convert_tokenizer "%~3" --with_detokenizer -o "%~2\%~1" ) else ( echo Models file %repository%\%model%\openvino_tokenizer.bin exists. Skipping downloading models. ) exit /b 0 +:download_openvino_tokenizer +set "model=%~1" +set "repository=%~2" + +if not exist "%repository%\%model%\openvino_tokenizer.bin" ( + echo Downloading tokenizer and detokenizer for %model% model to %repository%\%model% directory. + mkdir "%repository%\%model%" + hf download "%model%" --local-dir "%repository%\%model%" --include *tokenizer* --exclude *detokenizer* +) else ( + echo Models file %repository%\%model%\openvino_tokenizer.bin exists. Skipping downloading models. +) + :: Helper subroutine to download tokenizers :download_tokenizer set "model=%~1" From a0ca4387e145e1f30b293cea1f995fa342d45d22 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 7 May 2026 16:41:39 +0200 Subject: [PATCH 17/33] gemma4 reasoning parser --- docs/parameters.md | 4 +-- src/llm/BUILD | 1 + .../io_processing/gemma4/reasoning_parser.hpp | 29 +++++++++++++++++++ src/llm/io_processing/output_parser.cpp | 3 ++ .../io_processing/qwen3/reasoning_parser.hpp | 25 ++++++++++++---- 5 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 src/llm/io_processing/gemma4/reasoning_parser.hpp diff --git a/docs/parameters.md b/docs/parameters.md index 56ef12414c..507d991d11 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -138,8 +138,8 @@ Task specific parameters for different tasks (text generation/image generation/e | `--max_prompt_len` | `integer` | Sets NPU specific property for maximum number of tokens in the prompt. | | `--kv_cache_precision` | `string` | Reduced kv cache precision to `u8` lowers the cache size consumption. Accepted values: `u8` or empty (default). | | `--model_distribution_policy` | `string` | TENSOR_PARALLEL distributes tensor to multiple sockets/devices and processes it in parallel. PIPELINE_PARALLEL distributes different tensors to process by each device. Accepted values: `TENSOR_PARALLEL`, `PIPELINE_PARALLEL` or empty (default). | -| `--reasoning_parser` | `string` | Type of parser to use for reasoning content extraction from model output. Currently supported: [qwen3, gptoss] | -| `--tool_parser` | `string` | Type of parser to use for tool calls extraction from model output. Currently supported: [llama3, phi4, hermes3, mistral, qwen3coder, gptoss, devstral, lfm2] | +| `--reasoning_parser` | `string` | Type of parser to use for reasoning content extraction from model output. Currently supported: [qwen3, gptoss, gemma4] | +| `--tool_parser` | `string` | Type of parser to use for tool calls extraction from model output. Currently supported: [llama3, phi4, hermes3, mistral, qwen3coder, gptoss, devstral, lfm2, gemma4] | | `--enable_tool_guided_generation` | `bool` | Enables enforcing tool schema during generation. Requires setting response parser. Default: false. | ### Image generation diff --git a/src/llm/BUILD b/src/llm/BUILD index 4587e24402..2eb3374417 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -222,6 +222,7 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w "io_processing/devstral/tool_parser.hpp", "io_processing/mistral/tool_parser.hpp", "io_processing/qwen3/reasoning_parser.hpp", + "io_processing/gemma4/reasoning_parser.hpp", "io_processing/gptoss/reasoning_parser.hpp", "io_processing/gptoss/tool_parser.hpp", "io_processing/gptoss/harmony.hpp", diff --git a/src/llm/io_processing/gemma4/reasoning_parser.hpp b/src/llm/io_processing/gemma4/reasoning_parser.hpp new file mode 100644 index 0000000000..e3aba30796 --- /dev/null +++ b/src/llm/io_processing/gemma4/reasoning_parser.hpp @@ -0,0 +1,29 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include "../qwen3/reasoning_parser.hpp" + +namespace ovms { +class Gemma4ReasoningParser : public Qwen3ReasoningParser { +public: + Gemma4ReasoningParser() = delete; + explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer) : + Qwen3ReasoningParser(tokenizer, "<|channel>thought\n", "", true) {} +}; +} // namespace ovms diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index f0d2d3c479..bacdde61de 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -28,6 +28,7 @@ #include "qwen3/reasoning_parser.hpp" #include "qwen3coder/qwen3coder_tool_parser.hpp" #include "devstral/tool_parser.hpp" +#include "gemma4/reasoning_parser.hpp" #include "gptoss/reasoning_parser.hpp" <<<<<<< HEAD #include "lfm2/lfm2_tool_parser.hpp" @@ -196,6 +197,8 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to if (reasoningParserName == "qwen3") { reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "gemma4") { + reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); } else if (!reasoningParserName.empty()) { diff --git a/src/llm/io_processing/qwen3/reasoning_parser.hpp b/src/llm/io_processing/qwen3/reasoning_parser.hpp index 6254e874e5..d7ccb9d76a 100644 --- a/src/llm/io_processing/qwen3/reasoning_parser.hpp +++ b/src/llm/io_processing/qwen3/reasoning_parser.hpp @@ -28,26 +28,41 @@ namespace ovms { class Qwen3ReasoningParser : public BaseOutputParser { protected: // Tags used to identify the reasoning segment in the content - const std::string parsingStartTag = ""; - const std::string parsingEndTag = ""; + const std::string parsingStartTag; + const std::string parsingEndTag; + const bool specialTokensRequired; + const std::vector parsingStartTags; + const std::vector specialParsingStartTags; + + Qwen3ReasoningParser(ov::genai::Tokenizer& tokenizer, + const std::string& startTag, + const std::string& endTag, + bool requiresSpecialTokens) : + BaseOutputParser(tokenizer), + parsingStartTag(startTag), + parsingEndTag(endTag), + specialTokensRequired(requiresSpecialTokens), + parsingStartTags{startTag}, + specialParsingStartTags{} {} public: Qwen3ReasoningParser() = delete; explicit Qwen3ReasoningParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} + Qwen3ReasoningParser(tokenizer, "", "", false) {} void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; std::optional parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override; const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags{this->parsingStartTag}; return parsingStartTags; } const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; return specialParsingStartTags; } const std::string& getParsingEndTag() const override { return parsingEndTag; } + bool requiresStreamingWithSpecialTokens() const override { + return specialTokensRequired; + } }; } // namespace ovms From 01af9add6fef0e8febb6861f143129d42042f809 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 09:10:35 +0200 Subject: [PATCH 18/33] gemma thinking save --- prepare_llm_models.sh | 2 +- src/llm/BUILD | 1 + .../io_processing/gemma4/reasoning_parser.cpp | 45 +++++++++++++++++++ .../io_processing/gemma4/reasoning_parser.hpp | 1 + .../gemma4_output_parser_test.cpp | 20 ++++++++- windows_prepare_llm_models.bat | 2 +- 6 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 src/llm/io_processing/gemma4/reasoning_parser.cpp diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index 010104c721..7c51e50eb1 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -233,7 +233,7 @@ fi if [ -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then echo "Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." else - hf download "$GEMMA4_MODEL" --local-dir $1/$GEMMA4_MODEL --include *tokenizer* --exclude *detokenizer* + hf download "$GEMMA4_MODEL" --local-dir $1/$GEMMA4_MODEL --include *tokenizer* fi if [ ! -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then echo "[ERROR] Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE does not exist." diff --git a/src/llm/BUILD b/src/llm/BUILD index 2eb3374417..67d1071893 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -235,6 +235,7 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w "io_processing/devstral/tool_parser.cpp", "io_processing/mistral/tool_parser.cpp", "io_processing/qwen3/reasoning_parser.cpp", + "io_processing/gemma4/reasoning_parser.cpp", "io_processing/gptoss/reasoning_parser.cpp", "io_processing/gptoss/tool_parser.cpp", "io_processing/gptoss/harmony.cpp", diff --git a/src/llm/io_processing/gemma4/reasoning_parser.cpp b/src/llm/io_processing/gemma4/reasoning_parser.cpp new file mode 100644 index 0000000000..1091ef55b3 --- /dev/null +++ b/src/llm/io_processing/gemma4/reasoning_parser.cpp @@ -0,0 +1,45 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "../../../logging.hpp" +#include "reasoning_parser.hpp" +#include "../utils.hpp" + +namespace ovms { +void Gemma4ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + std::string contentWithSpecialTokens = tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false)); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsing reasoning with Gemma4ReasoningParser. Content with special tokens: {}", contentWithSpecialTokens); + std::string startReasoningTag = getParsingStartTags()[0]; + std::string endReasoningTag = getParsingEndTag(); + size_t startPos = contentWithSpecialTokens.find(startReasoningTag); + size_t endPos = contentWithSpecialTokens.find(endReasoningTag); + + if (startPos != std::string::npos && endPos != std::string::npos && startPos < endPos) { + size_t reasoningStart = startPos + startReasoningTag.length(); + std::string reasoningText = contentWithSpecialTokens.substr(reasoningStart, endPos - reasoningStart); + parsedOutput.reasoning = reasoningText; + // Remove reasoning from content + std::string contentWithoutReasoning = contentWithSpecialTokens.substr(0, startPos) + contentWithSpecialTokens.substr(endPos + endReasoningTag.length()); + parsedOutput.content = contentWithoutReasoning; + } +} +} // namespace ovms diff --git a/src/llm/io_processing/gemma4/reasoning_parser.hpp b/src/llm/io_processing/gemma4/reasoning_parser.hpp index e3aba30796..d00bd99f59 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.hpp @@ -25,5 +25,6 @@ class Gemma4ReasoningParser : public Qwen3ReasoningParser { Gemma4ReasoningParser() = delete; explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer) : Qwen3ReasoningParser(tokenizer, "<|channel>thought\n", "", true) {} + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; }; } // namespace ovms diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 092fe508c3..7deb45393a 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -58,7 +58,7 @@ class Gemma4OutputParserTest : public ::testing::Test { void SetUp() override { // For Gemma4 model there is only tool parser available - outputParserWithRegularToolParsing = std::make_unique(*gemma4Tokenizer, "gemma4", "", EMPTY_TOOLS_SCHEMA); + outputParserWithRegularToolParsing = std::make_unique(*gemma4Tokenizer, "gemma4", "gemma4", EMPTY_TOOLS_SCHEMA); } void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { @@ -96,6 +96,24 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { } } +TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCallAndReasoning) { + std::string inputWithProperClosure = "<|channel>thought\nSome reasoning content<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "Some reasoning content"); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); + } +} + TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; std::string inputWithoutSpecialTokens = "call:example_tool{arg1:value1,arg2:42}"; diff --git a/windows_prepare_llm_models.bat b/windows_prepare_llm_models.bat index 67f44a2214..c49fdc1f0f 100644 --- a/windows_prepare_llm_models.bat +++ b/windows_prepare_llm_models.bat @@ -140,7 +140,7 @@ set "repository=%~2" if not exist "%repository%\%model%\openvino_tokenizer.bin" ( echo Downloading tokenizer and detokenizer for %model% model to %repository%\%model% directory. mkdir "%repository%\%model%" - hf download "%model%" --local-dir "%repository%\%model%" --include *tokenizer* --exclude *detokenizer* + hf download "%model%" --local-dir "%repository%\%model%" --include *tokenizer* ) else ( echo Models file %repository%\%model%\openvino_tokenizer.bin exists. Skipping downloading models. ) From d5c6f0eac2000880321de6daae647aef5fd68e97 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 11:15:37 +0200 Subject: [PATCH 19/33] reasoning fix --- src/llm/BUILD | 26 ++++++++++++++----- .../io_processing/gemma4/reasoning_parser.cpp | 14 ++++------ .../io_processing/gemma4/reasoning_parser.hpp | 9 ++++++- src/llm/io_processing/gemma4/tool_parser.cpp | 8 ++++++ src/llm/io_processing/gemma4/tool_parser.hpp | 2 ++ .../io_processing/qwen3/reasoning_parser.hpp | 25 ++++-------------- .../gemma4_output_parser_test.cpp | 15 +++++++++++ 7 files changed, 63 insertions(+), 36 deletions(-) diff --git a/src/llm/BUILD b/src/llm/BUILD index 67d1071893..d3d8fbab95 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -199,8 +199,25 @@ ovms_cc_library( ) ovms_cc_library( name = "io_processing_gemma4_tool_parser", - hdrs = ["io_processing/gemma4/tool_parser.hpp"], - srcs = ["io_processing/gemma4/tool_parser.cpp"], + hdrs = ["io_processing/gemma4/tool_parser.hpp", "io_processing/gemma4/reasoning_parser.hpp"], + srcs = ["io_processing/gemma4/tool_parser.cpp", "io_processing/gemma4/reasoning_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + "//src:libovmsstring_utils", + ":io_processing_utils", + ":io_processing_base_output_parser", + ":io_processing_qwen3_reasoning_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "io_processing_qwen3_reasoning_parser", + hdrs = ["io_processing/qwen3/reasoning_parser.hpp"], + srcs = ["io_processing/qwen3/reasoning_parser.cpp"], deps = [ "@com_github_tencent_rapidjson//:rapidjson", "//src/port:rapidjson_document", @@ -221,8 +238,6 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w "io_processing/phi4/tool_parser.hpp", "io_processing/devstral/tool_parser.hpp", "io_processing/mistral/tool_parser.hpp", - "io_processing/qwen3/reasoning_parser.hpp", - "io_processing/gemma4/reasoning_parser.hpp", "io_processing/gptoss/reasoning_parser.hpp", "io_processing/gptoss/tool_parser.hpp", "io_processing/gptoss/harmony.hpp", @@ -234,8 +249,6 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w "io_processing/phi4/tool_parser.cpp", "io_processing/devstral/tool_parser.cpp", "io_processing/mistral/tool_parser.cpp", - "io_processing/qwen3/reasoning_parser.cpp", - "io_processing/gemma4/reasoning_parser.cpp", "io_processing/gptoss/reasoning_parser.cpp", "io_processing/gptoss/tool_parser.cpp", "io_processing/gptoss/harmony.cpp", @@ -252,6 +265,7 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_qwen3coder_tool_parser", ":io_processing_lfm2_tool_parser", ":io_processing_gemma4_tool_parser", + ":io_processing_qwen3_reasoning_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", ], diff --git a/src/llm/io_processing/gemma4/reasoning_parser.cpp b/src/llm/io_processing/gemma4/reasoning_parser.cpp index 1091ef55b3..5d74b6abd6 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.cpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.cpp @@ -26,19 +26,15 @@ namespace ovms { void Gemma4ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - std::string contentWithSpecialTokens = tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false)); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsing reasoning with Gemma4ReasoningParser. Content with special tokens: {}", contentWithSpecialTokens); - std::string startReasoningTag = getParsingStartTags()[0]; - std::string endReasoningTag = getParsingEndTag(); - size_t startPos = contentWithSpecialTokens.find(startReasoningTag); - size_t endPos = contentWithSpecialTokens.find(endReasoningTag); + size_t startPos = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningTokenId) - generatedTokens.begin(); + size_t endPos = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId) - generatedTokens.begin(); if (startPos != std::string::npos && endPos != std::string::npos && startPos < endPos) { - size_t reasoningStart = startPos + startReasoningTag.length(); - std::string reasoningText = contentWithSpecialTokens.substr(reasoningStart, endPos - reasoningStart); + size_t reasoningStart = startPos + 3; // deleting "<|channel>thought\n" + std::string reasoningText = tokenizer.decode(std::vector(generatedTokens.begin() + reasoningStart, generatedTokens.begin() + endPos), ov::genai::skip_special_tokens(true)); parsedOutput.reasoning = reasoningText; // Remove reasoning from content - std::string contentWithoutReasoning = contentWithSpecialTokens.substr(0, startPos) + contentWithSpecialTokens.substr(endPos + endReasoningTag.length()); + std::string contentWithoutReasoning = tokenizer.decode(std::vector(generatedTokens.begin() + endPos + 1, generatedTokens.end()), ov::genai::skip_special_tokens(true)); parsedOutput.content = contentWithoutReasoning; } } diff --git a/src/llm/io_processing/gemma4/reasoning_parser.hpp b/src/llm/io_processing/gemma4/reasoning_parser.hpp index d00bd99f59..22025085fe 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.hpp @@ -21,10 +21,17 @@ namespace ovms { class Gemma4ReasoningParser : public Qwen3ReasoningParser { +protected: + const int64_t reasoningTokenId = 100; + const int64_t reasoningEndTokenId = 101; public: Gemma4ReasoningParser() = delete; explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer) : - Qwen3ReasoningParser(tokenizer, "<|channel>thought\n", "", true) {} + Qwen3ReasoningParser(tokenizer) {} void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + + bool requiresStreamingWithSpecialTokens() const override { + return true; + } }; } // namespace ovms diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 53385c334d..0bec0b0d5e 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -36,6 +36,9 @@ const std::string Gemma4ToolParser::TOOL_ARGS_SEPARATOR_STR = ","; const int64_t Gemma4ToolParser::botTokenId = 48; const int64_t Gemma4ToolParser::eotTokenId = 49; +const int64_t Gemma4ToolParser::reasoningTokenId = 100; +const int64_t Gemma4ToolParser::reasoningEndTokenId = 101; + std::string Gemma4ToolParser::parseArrayParameter(const std::string& argumentStr) { size_t pos = 1; std::string parsedArguments = "["; @@ -502,6 +505,11 @@ void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vectorfirst, contentWithoutToolCalls.begin() + it->second + 1); } + + auto reasoningEnd = std::find(contentWithoutToolCalls.begin(), contentWithoutToolCalls.end(), reasoningEndTokenId); + if (reasoningEnd != contentWithoutToolCalls.end()) { + contentWithoutToolCalls.erase(contentWithoutToolCalls.begin(), reasoningEnd + 1); + } parsedOutput.content = tokenizer.decode(contentWithoutToolCalls, ov::AnyMap{ov::genai::skip_special_tokens(true)}); } } // namespace ovms diff --git a/src/llm/io_processing/gemma4/tool_parser.hpp b/src/llm/io_processing/gemma4/tool_parser.hpp index aebe7c5fa3..917fd67e3a 100644 --- a/src/llm/io_processing/gemma4/tool_parser.hpp +++ b/src/llm/io_processing/gemma4/tool_parser.hpp @@ -33,6 +33,8 @@ class Gemma4ToolParser : public BaseOutputParser { static const int64_t botTokenId; static const int64_t eotTokenId; + static const int64_t reasoningTokenId; + static const int64_t reasoningEndTokenId; enum class State { Content, // Content -> ToolCallStarted (on TOOL_CALL_START_TAG) diff --git a/src/llm/io_processing/qwen3/reasoning_parser.hpp b/src/llm/io_processing/qwen3/reasoning_parser.hpp index d7ccb9d76a..6254e874e5 100644 --- a/src/llm/io_processing/qwen3/reasoning_parser.hpp +++ b/src/llm/io_processing/qwen3/reasoning_parser.hpp @@ -28,41 +28,26 @@ namespace ovms { class Qwen3ReasoningParser : public BaseOutputParser { protected: // Tags used to identify the reasoning segment in the content - const std::string parsingStartTag; - const std::string parsingEndTag; - const bool specialTokensRequired; - const std::vector parsingStartTags; - const std::vector specialParsingStartTags; - - Qwen3ReasoningParser(ov::genai::Tokenizer& tokenizer, - const std::string& startTag, - const std::string& endTag, - bool requiresSpecialTokens) : - BaseOutputParser(tokenizer), - parsingStartTag(startTag), - parsingEndTag(endTag), - specialTokensRequired(requiresSpecialTokens), - parsingStartTags{startTag}, - specialParsingStartTags{} {} + const std::string parsingStartTag = ""; + const std::string parsingEndTag = ""; public: Qwen3ReasoningParser() = delete; explicit Qwen3ReasoningParser(ov::genai::Tokenizer& tokenizer) : - Qwen3ReasoningParser(tokenizer, "", "", false) {} + BaseOutputParser(tokenizer) {} void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; std::optional parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override; const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{this->parsingStartTag}; return parsingStartTags; } const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; return specialParsingStartTags; } const std::string& getParsingEndTag() const override { return parsingEndTag; } - bool requiresStreamingWithSpecialTokens() const override { - return specialTokensRequired; - } }; } // namespace ovms diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 7deb45393a..066d47d269 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -114,6 +114,21 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCallAndReasoning } } +TEST_F(Gemma4OutputParserTest, ParseReasoningWithoutToolCall) { + std::string inputWithProperClosure = "<|channel>thought\nSome reasoning contentSOME CONTENT WITHOUT TOOL CALL"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, "SOME CONTENT WITHOUT TOOL CALL"); + EXPECT_EQ(parsedOutput.reasoning, "Some reasoning content"); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + } +} + TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; std::string inputWithoutSpecialTokens = "call:example_tool{arg1:value1,arg2:42}"; From 9fc630642772fabb3ab5c836ba3c0536f190975c Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 11:19:07 +0200 Subject: [PATCH 20/33] clang-format --- src/llm/io_processing/gemma4/reasoning_parser.cpp | 2 +- src/llm/io_processing/gemma4/reasoning_parser.hpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/llm/io_processing/gemma4/reasoning_parser.cpp b/src/llm/io_processing/gemma4/reasoning_parser.cpp index 5d74b6abd6..f404a1ad55 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.cpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.cpp @@ -30,7 +30,7 @@ void Gemma4ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector< size_t endPos = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId) - generatedTokens.begin(); if (startPos != std::string::npos && endPos != std::string::npos && startPos < endPos) { - size_t reasoningStart = startPos + 3; // deleting "<|channel>thought\n" + size_t reasoningStart = startPos + 3; // deleting "<|channel>thought\n" std::string reasoningText = tokenizer.decode(std::vector(generatedTokens.begin() + reasoningStart, generatedTokens.begin() + endPos), ov::genai::skip_special_tokens(true)); parsedOutput.reasoning = reasoningText; // Remove reasoning from content diff --git a/src/llm/io_processing/gemma4/reasoning_parser.hpp b/src/llm/io_processing/gemma4/reasoning_parser.hpp index 22025085fe..063cf2913b 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.hpp @@ -24,6 +24,7 @@ class Gemma4ReasoningParser : public Qwen3ReasoningParser { protected: const int64_t reasoningTokenId = 100; const int64_t reasoningEndTokenId = 101; + public: Gemma4ReasoningParser() = delete; explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer) : From 2aab3983ce95e41405eb75c719627f39e8b37af9 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 11:23:20 +0200 Subject: [PATCH 21/33] cpplint --- src/llm/io_processing/gemma4/reasoning_parser.hpp | 1 + src/llm/visual_language_model/legacy/servable.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/llm/io_processing/gemma4/reasoning_parser.hpp b/src/llm/io_processing/gemma4/reasoning_parser.hpp index 063cf2913b..f692df45af 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include "../qwen3/reasoning_parser.hpp" diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index f267b17294..20c97068d1 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include "../../../logging.hpp" #include "../../../status.hpp" From 1b304f4d8d5b378c18a137fb218da3e24b4db540 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 11:41:43 +0200 Subject: [PATCH 22/33] clang format --- src/llm/io_processing/output_parser.cpp | 500 ++++++++++++------------ 1 file changed, 250 insertions(+), 250 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index bacdde61de..fbad23d797 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -36,221 +36,221 @@ #include "gemma4/tool_parser.hpp" >>>>>>> 0c55d5ad (save) -namespace ovms { -OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { - if (tag.empty()) { - return TagLookupStatus::NOT_FOUND; - } - if (tag.size() > buffer.size()) { - /* + namespace ovms { + OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { + if (tag.empty()) { + return TagLookupStatus::NOT_FOUND; + } + if (tag.size() > buffer.size()) { + /* If the tag is longer than the buffer, we check if the buffer and tag overlap (either partially or fully for exact match) They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE otherwise we return NOT_FOUND */ - if (stringsOverlap(buffer, tag)) { - return TagLookupStatus::FOUND_INCOMPLETE; - } else { - return TagLookupStatus::NOT_FOUND; - } - } else if (tag.size() < buffer.size()) { - /* + if (stringsOverlap(buffer, tag)) { + return TagLookupStatus::FOUND_INCOMPLETE; + } else { + return TagLookupStatus::NOT_FOUND; + } + } else if (tag.size() < buffer.size()) { + /* If the tag is shorter than the buffer, we check: a) if the tag is a substring of the buffer (tag is fully matched) b) if the buffer and tag overlap (part of the tag is matched) in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE otherwise we return NOT_FOUND */ - if (buffer.find(tag) != std::string::npos) { - return TagLookupStatus::FOUND_COMPLETE; - } else if (stringsOverlap(buffer, tag)) { - return TagLookupStatus::FOUND_INCOMPLETE; + if (buffer.find(tag) != std::string::npos) { + return TagLookupStatus::FOUND_COMPLETE; + } else if (stringsOverlap(buffer, tag)) { + return TagLookupStatus::FOUND_INCOMPLETE; + } else { + return TagLookupStatus::NOT_FOUND; + } } else { - return TagLookupStatus::NOT_FOUND; - } - } else { - /* + /* If the tag and buffer are of the same length, we check: a) if they are equal (tag is fully matched) b) if they overlap (part of the tag is matched) in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE otherwise we return NOT_FOUND */ - if (buffer == tag) { - return TagLookupStatus::FOUND_COMPLETE; - } else if (stringsOverlap(buffer, tag)) { - return TagLookupStatus::FOUND_INCOMPLETE; - } else { - return TagLookupStatus::NOT_FOUND; + if (buffer == tag) { + return TagLookupStatus::FOUND_COMPLETE; + } else if (stringsOverlap(buffer, tag)) { + return TagLookupStatus::FOUND_INCOMPLETE; + } else { + return TagLookupStatus::NOT_FOUND; + } } } -} -OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTags(const std::vector& tags) const { - // We look for multiple tags and return the status in the following priority: FOUND COMPLETE > FOUND_INCOMPLETE > NOT_FOUND - TagLookupStatus finalTagLookupStatus = TagLookupStatus::NOT_FOUND; - for (const auto& tag : tags) { - auto tagLookupStatus = lookupTag(tag); - if (tagLookupStatus == TagLookupStatus::FOUND_COMPLETE) { - return TagLookupStatus::FOUND_COMPLETE; - } - if (tagLookupStatus == TagLookupStatus::FOUND_INCOMPLETE) { - finalTagLookupStatus = TagLookupStatus::FOUND_INCOMPLETE; + OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTags(const std::vector& tags) const { + // We look for multiple tags and return the status in the following priority: FOUND COMPLETE > FOUND_INCOMPLETE > NOT_FOUND + TagLookupStatus finalTagLookupStatus = TagLookupStatus::NOT_FOUND; + for (const auto& tag : tags) { + auto tagLookupStatus = lookupTag(tag); + if (tagLookupStatus == TagLookupStatus::FOUND_COMPLETE) { + return TagLookupStatus::FOUND_COMPLETE; + } + if (tagLookupStatus == TagLookupStatus::FOUND_INCOMPLETE) { + finalTagLookupStatus = TagLookupStatus::FOUND_INCOMPLETE; + } } + return finalTagLookupStatus; } - return finalTagLookupStatus; -} -void OutputParser::StreamOutputCache::add(const std::string& chunk) { - buffer += chunk; -} + void OutputParser::StreamOutputCache::add(const std::string& chunk) { + buffer += chunk; + } -void OutputParser::StreamOutputCache::clear() { - buffer.clear(); -} + void OutputParser::StreamOutputCache::clear() { + buffer.clear(); + } -const std::string& OutputParser::StreamOutputCache::getBuffer() const { - return buffer; -} + const std::string& OutputParser::StreamOutputCache::getBuffer() const { + return buffer; + } -rapidjson::Document OutputParser::parseContentChunk(ProcessingPhase newPhase) { - std::string chunkContent = streamOutputCache.getBuffer(); - if (toolParser != nullptr) { - auto& specialTagsToErase = toolParser->getSpecialTagsToErase(); - for (const auto& tag : specialTagsToErase) { - size_t pos = 0; - while ((pos = chunkContent.find(tag, pos)) != std::string::npos) { - chunkContent.erase(pos, tag.length()); + rapidjson::Document OutputParser::parseContentChunk(ProcessingPhase newPhase) { + std::string chunkContent = streamOutputCache.getBuffer(); + if (toolParser != nullptr) { + auto& specialTagsToErase = toolParser->getSpecialTagsToErase(); + for (const auto& tag : specialTagsToErase) { + size_t pos = 0; + while ((pos = chunkContent.find(tag, pos)) != std::string::npos) { + chunkContent.erase(pos, tag.length()); + } } } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("content"); + writer.String(chunkContent.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + streamOutputCache.clear(); + processingPhase = newPhase; + return doc; } - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("content"); - writer.String(chunkContent.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); - streamOutputCache.clear(); - processingPhase = newPhase; - return doc; -} -std::optional OutputParser::parseToolCallChunk(ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { - if (!toolParser) { - throw std::runtime_error("Tool parser is not available, cannot parse tool call chunk"); - } - std::optional result; - try { - result = toolParser->parseChunk(streamOutputCache.getBuffer(), finishReason); - } catch (...) { + std::optional OutputParser::parseToolCallChunk(ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { + if (!toolParser) { + throw std::runtime_error("Tool parser is not available, cannot parse tool call chunk"); + } + std::optional result; + try { + result = toolParser->parseChunk(streamOutputCache.getBuffer(), finishReason); + } catch (...) { + streamOutputCache.clear(); + throw; + } streamOutputCache.clear(); - throw; + processingPhase = newPhase; + return result; } - streamOutputCache.clear(); - processingPhase = newPhase; - return result; -} -std::optional OutputParser::parseReasoningChunk(ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { - if (!reasoningParser) { - throw std::runtime_error("Reasoning parser is not available, cannot parse reasoning chunk"); - } - std::optional result; - try { - result = reasoningParser->parseChunk(streamOutputCache.getBuffer(), finishReason); - } catch (...) { + std::optional OutputParser::parseReasoningChunk(ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { + if (!reasoningParser) { + throw std::runtime_error("Reasoning parser is not available, cannot parse reasoning chunk"); + } + std::optional result; + try { + result = reasoningParser->parseChunk(streamOutputCache.getBuffer(), finishReason); + } catch (...) { + streamOutputCache.clear(); + throw; + } streamOutputCache.clear(); - throw; + processingPhase = newPhase; + return result; } - streamOutputCache.clear(); - processingPhase = newPhase; - return result; -} -OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string toolParserName, const std::string reasoningParserName, const ToolsSchemas_t& toolNameSchemaMap) : - tokenizer(tokenizer) { - if (toolParserName == "llama3") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "hermes3") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "phi4") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "mistral") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "gptoss") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "qwen3coder") { - toolParser = std::make_unique(tokenizer, toolNameSchemaMap); - } else if (toolParserName == "devstral") { - toolParser = std::make_unique(tokenizer, toolNameSchemaMap); - } else if (toolParserName == "lfm2") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "gemma4") { - toolParser = std::make_unique(tokenizer); - } else if (!toolParserName.empty()) { - throw std::runtime_error("Unsupported tool parser: " + toolParserName); - } + OutputParser::OutputParser(ov::genai::Tokenizer & tokenizer, const std::string toolParserName, const std::string reasoningParserName, const ToolsSchemas_t& toolNameSchemaMap) : + tokenizer(tokenizer) { + if (toolParserName == "llama3") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "hermes3") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "phi4") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "mistral") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "gptoss") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "qwen3coder") { + toolParser = std::make_unique(tokenizer, toolNameSchemaMap); + } else if (toolParserName == "devstral") { + toolParser = std::make_unique(tokenizer, toolNameSchemaMap); + } else if (toolParserName == "lfm2") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "gemma4") { + toolParser = std::make_unique(tokenizer); + } else if (!toolParserName.empty()) { + throw std::runtime_error("Unsupported tool parser: " + toolParserName); + } - if (reasoningParserName == "qwen3") { - reasoningParser = std::make_unique(tokenizer); - } else if (reasoningParserName == "gemma4") { - reasoningParser = std::make_unique(tokenizer); - } else if (reasoningParserName == "gptoss") { - reasoningParser = std::make_unique(tokenizer); - } else if (!reasoningParserName.empty()) { - throw std::runtime_error("Unsupported reasoning parser: " + reasoningParserName); - } + if (reasoningParserName == "qwen3") { + reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "gemma4") { + reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "gptoss") { + reasoningParser = std::make_unique(tokenizer); + } else if (!reasoningParserName.empty()) { + throw std::runtime_error("Unsupported reasoning parser: " + reasoningParserName); + } - if (toolParser && reasoningParser) { - if (toolParser->requiresStreamingWithSpecialTokens() != reasoningParser->requiresStreamingWithSpecialTokens()) { - throw std::runtime_error("Cannot use tool parser " + toolParserName + " with reasoning parser " + reasoningParserName + - " as they have different requirements for special tokens in streaming mode"); + if (toolParser && reasoningParser) { + if (toolParser->requiresStreamingWithSpecialTokens() != reasoningParser->requiresStreamingWithSpecialTokens()) { + throw std::runtime_error("Cannot use tool parser " + toolParserName + " with reasoning parser " + reasoningParserName + + " as they have different requirements for special tokens in streaming mode"); + } } } -} -bool OutputParser::isToolParserAvailable() const { - return toolParser != nullptr; -} + bool OutputParser::isToolParserAvailable() const { + return toolParser != nullptr; + } -bool OutputParser::isReasoningParserAvailable() const { - return reasoningParser != nullptr; -} + bool OutputParser::isReasoningParserAvailable() const { + return reasoningParser != nullptr; + } -std::string OutputParser::getToolParserStartTag() const { - if (toolParser) { - return toolParser->getParsingStartTags()[0]; - } else { - throw std::runtime_error("Tool parser is not available, cannot get start tag"); + std::string OutputParser::getToolParserStartTag() const { + if (toolParser) { + return toolParser->getParsingStartTags()[0]; + } else { + throw std::runtime_error("Tool parser is not available, cannot get start tag"); + } } -} -ParsedOutput OutputParser::parse(const std::vector& generatedTokens, const bool toolsAvailable) { - // Model output is processed by the chain of parsers. Each parser extracts relevant part of the output and fills the ParsedOutput structure. - // At the beginning, the content field of ParsedOutput is already filled with decoded content from generatedTokens. - // When parser extracts relevant information, it should remove it from the content field, so we don't duplicate it in the final output. + ParsedOutput OutputParser::parse(const std::vector& generatedTokens, const bool toolsAvailable) { + // Model output is processed by the chain of parsers. Each parser extracts relevant part of the output and fills the ParsedOutput structure. + // At the beginning, the content field of ParsedOutput is already filled with decoded content from generatedTokens. + // When parser extracts relevant information, it should remove it from the content field, so we don't duplicate it in the final output. - if (spdlog::default_logger_raw()->level() == spdlog::level::trace) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); - } - ParsedOutput parsedOutput; - parsedOutput.content = tokenizer.decode(generatedTokens); - if (reasoningParser) { - reasoningParser->parse(parsedOutput, generatedTokens); - } - // We run tool parser only if the parser is available and tools have been provided in the request. - if (toolParser && toolsAvailable) { - toolParser->parse(parsedOutput, generatedTokens); + if (spdlog::default_logger_raw()->level() == spdlog::level::trace) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); + } + ParsedOutput parsedOutput; + parsedOutput.content = tokenizer.decode(generatedTokens); + if (reasoningParser) { + reasoningParser->parse(parsedOutput, generatedTokens); + } + // We run tool parser only if the parser is available and tools have been provided in the request. + if (toolParser && toolsAvailable) { + toolParser->parse(parsedOutput, generatedTokens); + } + return parsedOutput; } - return parsedOutput; -} -std::optional OutputParser::parseChunk(const std::string& chunkResponse, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason) { - /* + std::optional OutputParser::parseChunk(const std::string& chunkResponse, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason) { + /* Using appropriate parser based on the current processing phase Call to this method should return either result from parserContentChunk, parseToolCallChunk, parseReasoningChunk when we can determine the phase or std::nullopt when we are waiting for more chunks to determine if we should switch phase or not. @@ -258,99 +258,99 @@ std::optional OutputParser::parseChunk(const std::string& c so only use those methods or return nullopt. */ - bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingStartTags().empty() && !reasoningParser->getParsingEndTag().empty(); - bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingStartTags().empty(); - bool applyToolParser = toolParserExistsAndSupportsStreaming && toolsAvailable; + bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingStartTags().empty() && !reasoningParser->getParsingEndTag().empty(); + bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingStartTags().empty(); + bool applyToolParser = toolParserExistsAndSupportsStreaming && toolsAvailable; - streamOutputCache.add(chunkResponse); + streamOutputCache.add(chunkResponse); - if (processingPhase == UNKNOWN) { - // If we are in the UNKNOWN phase, we need to determine if we should switch to CONTENT, REASONING, or TOOL_CALLS phase. - TagLookupStatus anyStartTagStatus = TagLookupStatus::NOT_FOUND; - if (reasoningParserExistsAndSupportsStreaming) { - // Check if reasoning start tag has been received - TagLookupStatus reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingStartTags()); - if (reasoningStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If reasoning start tag is not found, check if any of the special start tags are found - reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getSpecialParsingStartTags()); + if (processingPhase == UNKNOWN) { + // If we are in the UNKNOWN phase, we need to determine if we should switch to CONTENT, REASONING, or TOOL_CALLS phase. + TagLookupStatus anyStartTagStatus = TagLookupStatus::NOT_FOUND; + if (reasoningParserExistsAndSupportsStreaming) { + // Check if reasoning start tag has been received + TagLookupStatus reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingStartTags()); + if (reasoningStartTagStatus == TagLookupStatus::NOT_FOUND) { + // If reasoning start tag is not found, check if any of the special start tags are found + reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getSpecialParsingStartTags()); + } + if (reasoningStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { + return parseReasoningChunk(finishReason); + } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing, so potential tool parser start tag is not missed + anyStartTagStatus = reasoningStartTagStatus; } - if (reasoningStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - return parseReasoningChunk(finishReason); - } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing, so potential tool parser start tag is not missed - anyStartTagStatus = reasoningStartTagStatus; - } - if (applyToolParser) { - // Check if tool call start tag has been received - TagLookupStatus toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); - if (toolCallStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If tool call start tag is not found, check if any of the special start tags are found - toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getSpecialParsingStartTags()); + if (applyToolParser) { + // Check if tool call start tag has been received + TagLookupStatus toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); + if (toolCallStartTagStatus == TagLookupStatus::NOT_FOUND) { + // If tool call start tag is not found, check if any of the special start tags are found + toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getSpecialParsingStartTags()); + } + if (toolCallStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { + return parseToolCallChunk(finishReason); + } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing + if (toolCallStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE) { + anyStartTagStatus = toolCallStartTagStatus; // We have at least one incomplete start tag + } } - if (toolCallStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - return parseToolCallChunk(finishReason); - } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing - if (toolCallStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE) { - anyStartTagStatus = toolCallStartTagStatus; // We have at least one incomplete start tag - } - } - if ((!reasoningParserExistsAndSupportsStreaming && !applyToolParser) || finishReason != ov::genai::GenerationFinishReason::NONE || anyStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If no special parsers are available, generation has finished or we have no start tags we just return content chunks and switch to CONTENT phase. + if ((!reasoningParserExistsAndSupportsStreaming && !applyToolParser) || finishReason != ov::genai::GenerationFinishReason::NONE || anyStartTagStatus == TagLookupStatus::NOT_FOUND) { + // If no special parsers are available, generation has finished or we have no start tags we just return content chunks and switch to CONTENT phase. + return parseContentChunk(); + } + // If we are here, it means we have incomplete start tag for either reasoning or tool parser, so we wait for more chunks + return std::nullopt; + } else if (processingPhase == REASONING) { + // If we are in the REASONING phase, we check if parsing end tag is found and if so, switch to UNKNOWN phase. + TagLookupStatus endTagStatus = streamOutputCache.lookupTag(reasoningParser->getParsingEndTag()); + if (endTagStatus == TagLookupStatus::FOUND_COMPLETE) { + // Switch back to UNKNOWN phase (we can have either CONTENT or TOOL_CALLS next) + return parseReasoningChunk(finishReason, UNKNOWN); + } else if (endTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; // Wait for more chunks to determine if end tag is complete + } + return parseReasoningChunk(finishReason); + } else if (processingPhase == CONTENT) { + // If we are in the CONTENT phase, we check if tool parser start tag is found and if so, switch to TOOL_CALLS phase. + // TOOL_CALLS is the only phase that can be processed after CONTENT. + if (applyToolParser) { + TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); + if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { + return parseToolCallChunk(finishReason); + } else if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; // Wait for more chunks to determine if end tag is complete + } + return parseContentChunk(); + } return parseContentChunk(); - } - // If we are here, it means we have incomplete start tag for either reasoning or tool parser, so we wait for more chunks - return std::nullopt; - } else if (processingPhase == REASONING) { - // If we are in the REASONING phase, we check if parsing end tag is found and if so, switch to UNKNOWN phase. - TagLookupStatus endTagStatus = streamOutputCache.lookupTag(reasoningParser->getParsingEndTag()); - if (endTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // Switch back to UNKNOWN phase (we can have either CONTENT or TOOL_CALLS next) - return parseReasoningChunk(finishReason, UNKNOWN); - } else if (endTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete - } - return parseReasoningChunk(finishReason); - } else if (processingPhase == CONTENT) { - // If we are in the CONTENT phase, we check if tool parser start tag is found and if so, switch to TOOL_CALLS phase. - // TOOL_CALLS is the only phase that can be processed after CONTENT. - if (applyToolParser) { + } else if (processingPhase == TOOL_CALLS_PROCESSING_TOOL) { + // Processing TOOL_CALLS is the last phase, so we always return the result of tool parser. + TagLookupStatus toolEndTagStatus = streamOutputCache.lookupTag(toolParser->getParsingEndTag()); + if (toolEndTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; // Wait for more chunks to determine if end tag is complete + } + if (toolEndTagStatus == TagLookupStatus::FOUND_COMPLETE) { + // If tool call has finished, we switch to waiting for next tool call as tool calls in the last phase, + // so we either get next tool call or finish processing. + return parseToolCallChunk(finishReason, TOOL_CALLS_WAITING_FOR_TOOL); + } + return parseToolCallChunk(finishReason); + } else if (processingPhase == TOOL_CALLS_WAITING_FOR_TOOL) { + // In this phase we are waiting for next tool call or finish of generation. + // If we get next tool call start tag, we switch to TOOL_CALLS phase, otherwise if generation finishes we switch to CONTENT phase to flush any remaining content. TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); + if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; // Wait for more chunks to determine if start tag is complete + } if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - return parseToolCallChunk(finishReason); - } else if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete + // If tool call has started, we switch back to processing tool phase. + return parseToolCallChunk(finishReason, TOOL_CALLS_PROCESSING_TOOL); } - return parseContentChunk(); - } - return parseContentChunk(); - } else if (processingPhase == TOOL_CALLS_PROCESSING_TOOL) { - // Processing TOOL_CALLS is the last phase, so we always return the result of tool parser. - TagLookupStatus toolEndTagStatus = streamOutputCache.lookupTag(toolParser->getParsingEndTag()); - if (toolEndTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete - } - if (toolEndTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // If tool call has finished, we switch to waiting for next tool call as tool calls in the last phase, - // so we either get next tool call or finish processing. - return parseToolCallChunk(finishReason, TOOL_CALLS_WAITING_FOR_TOOL); - } - return parseToolCallChunk(finishReason); - } else if (processingPhase == TOOL_CALLS_WAITING_FOR_TOOL) { - // In this phase we are waiting for next tool call or finish of generation. - // If we get next tool call start tag, we switch to TOOL_CALLS phase, otherwise if generation finishes we switch to CONTENT phase to flush any remaining content. - TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); - if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if start tag is complete - } - if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // If tool call has started, we switch back to processing tool phase. - return parseToolCallChunk(finishReason, TOOL_CALLS_PROCESSING_TOOL); + return parseToolCallChunk(finishReason); + } else { + SPDLOG_LOGGER_ERROR(llm_calculator_logger, "Unexpected processing phase: {}", static_cast(processingPhase)); + throw std::runtime_error("Unexpected error during stream output parsing"); } - return parseToolCallChunk(finishReason); - } else { - SPDLOG_LOGGER_ERROR(llm_calculator_logger, "Unexpected processing phase: {}", static_cast(processingPhase)); - throw std::runtime_error("Unexpected error during stream output parsing"); } -} } // namespace ovms From 23523495f077a6d888d4e91773f681c1b2aa8c67 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 11:45:01 +0200 Subject: [PATCH 23/33] styles --- src/llm/io_processing/output_parser.cpp | 543 +++++++++--------- .../visual_language_model/legacy/servable.cpp | 1 - 2 files changed, 270 insertions(+), 274 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index fbad23d797..e888b9f7ab 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -30,327 +30,324 @@ #include "devstral/tool_parser.hpp" #include "gemma4/reasoning_parser.hpp" #include "gptoss/reasoning_parser.hpp" -<<<<<<< HEAD #include "lfm2/lfm2_tool_parser.hpp" -======= #include "gemma4/tool_parser.hpp" ->>>>>>> 0c55d5ad (save) - namespace ovms { - OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { - if (tag.empty()) { +namespace ovms { +OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { + if (tag.empty()) { + return TagLookupStatus::NOT_FOUND; + } + if (tag.size() > buffer.size()) { + /* + If the tag is longer than the buffer, we check if the buffer and tag overlap (either partially or fully for exact match) + They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE + otherwise we return NOT_FOUND + */ + if (stringsOverlap(buffer, tag)) { + return TagLookupStatus::FOUND_INCOMPLETE; + } else { return TagLookupStatus::NOT_FOUND; } - if (tag.size() > buffer.size()) { - /* - If the tag is longer than the buffer, we check if the buffer and tag overlap (either partially or fully for exact match) - They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ - if (stringsOverlap(buffer, tag)) { - return TagLookupStatus::FOUND_INCOMPLETE; - } else { - return TagLookupStatus::NOT_FOUND; - } - } else if (tag.size() < buffer.size()) { - /* - If the tag is shorter than the buffer, we check: - a) if the tag is a substring of the buffer (tag is fully matched) - b) if the buffer and tag overlap (part of the tag is matched) - in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ - if (buffer.find(tag) != std::string::npos) { - return TagLookupStatus::FOUND_COMPLETE; - } else if (stringsOverlap(buffer, tag)) { - return TagLookupStatus::FOUND_INCOMPLETE; - } else { - return TagLookupStatus::NOT_FOUND; - } + } else if (tag.size() < buffer.size()) { + /* + If the tag is shorter than the buffer, we check: + a) if the tag is a substring of the buffer (tag is fully matched) + b) if the buffer and tag overlap (part of the tag is matched) + in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE + otherwise we return NOT_FOUND + */ + if (buffer.find(tag) != std::string::npos) { + return TagLookupStatus::FOUND_COMPLETE; + } else if (stringsOverlap(buffer, tag)) { + return TagLookupStatus::FOUND_INCOMPLETE; } else { - /* - If the tag and buffer are of the same length, we check: - a) if they are equal (tag is fully matched) - b) if they overlap (part of the tag is matched) - in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ - if (buffer == tag) { - return TagLookupStatus::FOUND_COMPLETE; - } else if (stringsOverlap(buffer, tag)) { - return TagLookupStatus::FOUND_INCOMPLETE; - } else { - return TagLookupStatus::NOT_FOUND; - } + return TagLookupStatus::NOT_FOUND; + } + } else { + /* + If the tag and buffer are of the same length, we check: + a) if they are equal (tag is fully matched) + b) if they overlap (part of the tag is matched) + in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE + otherwise we return NOT_FOUND + */ + if (buffer == tag) { + return TagLookupStatus::FOUND_COMPLETE; + } else if (stringsOverlap(buffer, tag)) { + return TagLookupStatus::FOUND_INCOMPLETE; + } else { + return TagLookupStatus::NOT_FOUND; } } +} - OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTags(const std::vector& tags) const { - // We look for multiple tags and return the status in the following priority: FOUND COMPLETE > FOUND_INCOMPLETE > NOT_FOUND - TagLookupStatus finalTagLookupStatus = TagLookupStatus::NOT_FOUND; - for (const auto& tag : tags) { - auto tagLookupStatus = lookupTag(tag); - if (tagLookupStatus == TagLookupStatus::FOUND_COMPLETE) { - return TagLookupStatus::FOUND_COMPLETE; - } - if (tagLookupStatus == TagLookupStatus::FOUND_INCOMPLETE) { - finalTagLookupStatus = TagLookupStatus::FOUND_INCOMPLETE; - } +OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTags(const std::vector& tags) const { + // We look for multiple tags and return the status in the following priority: FOUND COMPLETE > FOUND_INCOMPLETE > NOT_FOUND + TagLookupStatus finalTagLookupStatus = TagLookupStatus::NOT_FOUND; + for (const auto& tag : tags) { + auto tagLookupStatus = lookupTag(tag); + if (tagLookupStatus == TagLookupStatus::FOUND_COMPLETE) { + return TagLookupStatus::FOUND_COMPLETE; + } + if (tagLookupStatus == TagLookupStatus::FOUND_INCOMPLETE) { + finalTagLookupStatus = TagLookupStatus::FOUND_INCOMPLETE; } - return finalTagLookupStatus; } + return finalTagLookupStatus; +} - void OutputParser::StreamOutputCache::add(const std::string& chunk) { - buffer += chunk; - } +void OutputParser::StreamOutputCache::add(const std::string& chunk) { + buffer += chunk; +} - void OutputParser::StreamOutputCache::clear() { - buffer.clear(); - } +void OutputParser::StreamOutputCache::clear() { + buffer.clear(); +} - const std::string& OutputParser::StreamOutputCache::getBuffer() const { - return buffer; - } +const std::string& OutputParser::StreamOutputCache::getBuffer() const { + return buffer; +} - rapidjson::Document OutputParser::parseContentChunk(ProcessingPhase newPhase) { - std::string chunkContent = streamOutputCache.getBuffer(); - if (toolParser != nullptr) { - auto& specialTagsToErase = toolParser->getSpecialTagsToErase(); - for (const auto& tag : specialTagsToErase) { - size_t pos = 0; - while ((pos = chunkContent.find(tag, pos)) != std::string::npos) { - chunkContent.erase(pos, tag.length()); - } +rapidjson::Document OutputParser::parseContentChunk(ProcessingPhase newPhase) { + std::string chunkContent = streamOutputCache.getBuffer(); + if (toolParser != nullptr) { + auto& specialTagsToErase = toolParser->getSpecialTagsToErase(); + for (const auto& tag : specialTagsToErase) { + size_t pos = 0; + while ((pos = chunkContent.find(tag, pos)) != std::string::npos) { + chunkContent.erase(pos, tag.length()); } } - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("content"); - writer.String(chunkContent.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); - streamOutputCache.clear(); - processingPhase = newPhase; - return doc; } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("content"); + writer.String(chunkContent.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + streamOutputCache.clear(); + processingPhase = newPhase; + return doc; +} - std::optional OutputParser::parseToolCallChunk(ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { - if (!toolParser) { - throw std::runtime_error("Tool parser is not available, cannot parse tool call chunk"); - } - std::optional result; - try { - result = toolParser->parseChunk(streamOutputCache.getBuffer(), finishReason); - } catch (...) { - streamOutputCache.clear(); - throw; - } +std::optional OutputParser::parseToolCallChunk(ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { + if (!toolParser) { + throw std::runtime_error("Tool parser is not available, cannot parse tool call chunk"); + } + std::optional result; + try { + result = toolParser->parseChunk(streamOutputCache.getBuffer(), finishReason); + } catch (...) { streamOutputCache.clear(); - processingPhase = newPhase; - return result; + throw; } + streamOutputCache.clear(); + processingPhase = newPhase; + return result; +} - std::optional OutputParser::parseReasoningChunk(ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { - if (!reasoningParser) { - throw std::runtime_error("Reasoning parser is not available, cannot parse reasoning chunk"); - } - std::optional result; - try { - result = reasoningParser->parseChunk(streamOutputCache.getBuffer(), finishReason); - } catch (...) { - streamOutputCache.clear(); - throw; - } +std::optional OutputParser::parseReasoningChunk(ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { + if (!reasoningParser) { + throw std::runtime_error("Reasoning parser is not available, cannot parse reasoning chunk"); + } + std::optional result; + try { + result = reasoningParser->parseChunk(streamOutputCache.getBuffer(), finishReason); + } catch (...) { streamOutputCache.clear(); - processingPhase = newPhase; - return result; + throw; } + streamOutputCache.clear(); + processingPhase = newPhase; + return result; +} - OutputParser::OutputParser(ov::genai::Tokenizer & tokenizer, const std::string toolParserName, const std::string reasoningParserName, const ToolsSchemas_t& toolNameSchemaMap) : - tokenizer(tokenizer) { - if (toolParserName == "llama3") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "hermes3") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "phi4") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "mistral") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "gptoss") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "qwen3coder") { - toolParser = std::make_unique(tokenizer, toolNameSchemaMap); - } else if (toolParserName == "devstral") { - toolParser = std::make_unique(tokenizer, toolNameSchemaMap); - } else if (toolParserName == "lfm2") { - toolParser = std::make_unique(tokenizer); - } else if (toolParserName == "gemma4") { - toolParser = std::make_unique(tokenizer); - } else if (!toolParserName.empty()) { - throw std::runtime_error("Unsupported tool parser: " + toolParserName); - } +OutputParser::OutputParser(ov::genai::Tokenizer & tokenizer, const std::string toolParserName, const std::string reasoningParserName, const ToolsSchemas_t& toolNameSchemaMap) : + tokenizer(tokenizer) { + if (toolParserName == "llama3") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "hermes3") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "phi4") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "mistral") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "gptoss") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "qwen3coder") { + toolParser = std::make_unique(tokenizer, toolNameSchemaMap); + } else if (toolParserName == "devstral") { + toolParser = std::make_unique(tokenizer, toolNameSchemaMap); + } else if (toolParserName == "lfm2") { + toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "gemma4") { + toolParser = std::make_unique(tokenizer); + } else if (!toolParserName.empty()) { + throw std::runtime_error("Unsupported tool parser: " + toolParserName); + } - if (reasoningParserName == "qwen3") { - reasoningParser = std::make_unique(tokenizer); - } else if (reasoningParserName == "gemma4") { - reasoningParser = std::make_unique(tokenizer); - } else if (reasoningParserName == "gptoss") { - reasoningParser = std::make_unique(tokenizer); - } else if (!reasoningParserName.empty()) { - throw std::runtime_error("Unsupported reasoning parser: " + reasoningParserName); - } + if (reasoningParserName == "qwen3") { + reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "gemma4") { + reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "gptoss") { + reasoningParser = std::make_unique(tokenizer); + } else if (!reasoningParserName.empty()) { + throw std::runtime_error("Unsupported reasoning parser: " + reasoningParserName); + } - if (toolParser && reasoningParser) { - if (toolParser->requiresStreamingWithSpecialTokens() != reasoningParser->requiresStreamingWithSpecialTokens()) { - throw std::runtime_error("Cannot use tool parser " + toolParserName + " with reasoning parser " + reasoningParserName + - " as they have different requirements for special tokens in streaming mode"); - } + if (toolParser && reasoningParser) { + if (toolParser->requiresStreamingWithSpecialTokens() != reasoningParser->requiresStreamingWithSpecialTokens()) { + throw std::runtime_error("Cannot use tool parser " + toolParserName + " with reasoning parser " + reasoningParserName + + " as they have different requirements for special tokens in streaming mode"); } } +} - bool OutputParser::isToolParserAvailable() const { - return toolParser != nullptr; - } +bool OutputParser::isToolParserAvailable() const { + return toolParser != nullptr; +} - bool OutputParser::isReasoningParserAvailable() const { - return reasoningParser != nullptr; - } +bool OutputParser::isReasoningParserAvailable() const { + return reasoningParser != nullptr; +} - std::string OutputParser::getToolParserStartTag() const { - if (toolParser) { - return toolParser->getParsingStartTags()[0]; - } else { - throw std::runtime_error("Tool parser is not available, cannot get start tag"); - } +std::string OutputParser::getToolParserStartTag() const { + if (toolParser) { + return toolParser->getParsingStartTags()[0]; + } else { + throw std::runtime_error("Tool parser is not available, cannot get start tag"); } +} - ParsedOutput OutputParser::parse(const std::vector& generatedTokens, const bool toolsAvailable) { - // Model output is processed by the chain of parsers. Each parser extracts relevant part of the output and fills the ParsedOutput structure. - // At the beginning, the content field of ParsedOutput is already filled with decoded content from generatedTokens. - // When parser extracts relevant information, it should remove it from the content field, so we don't duplicate it in the final output. +ParsedOutput OutputParser::parse(const std::vector& generatedTokens, const bool toolsAvailable) { + // Model output is processed by the chain of parsers. Each parser extracts relevant part of the output and fills the ParsedOutput structure. + // At the beginning, the content field of ParsedOutput is already filled with decoded content from generatedTokens. + // When parser extracts relevant information, it should remove it from the content field, so we don't duplicate it in the final output. - if (spdlog::default_logger_raw()->level() == spdlog::level::trace) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); - } - ParsedOutput parsedOutput; - parsedOutput.content = tokenizer.decode(generatedTokens); - if (reasoningParser) { - reasoningParser->parse(parsedOutput, generatedTokens); - } - // We run tool parser only if the parser is available and tools have been provided in the request. - if (toolParser && toolsAvailable) { - toolParser->parse(parsedOutput, generatedTokens); - } - return parsedOutput; + if (spdlog::default_logger_raw()->level() == spdlog::level::trace) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); } + ParsedOutput parsedOutput; + parsedOutput.content = tokenizer.decode(generatedTokens); + if (reasoningParser) { + reasoningParser->parse(parsedOutput, generatedTokens); + } + // We run tool parser only if the parser is available and tools have been provided in the request. + if (toolParser && toolsAvailable) { + toolParser->parse(parsedOutput, generatedTokens); + } + return parsedOutput; +} - std::optional OutputParser::parseChunk(const std::string& chunkResponse, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason) { - /* - Using appropriate parser based on the current processing phase - Call to this method should return either result from parserContentChunk, parseToolCallChunk, parseReasoningChunk when we can determine the phase - or std::nullopt when we are waiting for more chunks to determine if we should switch phase or not. - Note that mentioned methods do not take chunk as argument, they read it from streamOutputCache and are responsible for clearing the cache, - so only use those methods or return nullopt. - */ - - bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingStartTags().empty() && !reasoningParser->getParsingEndTag().empty(); - bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingStartTags().empty(); - bool applyToolParser = toolParserExistsAndSupportsStreaming && toolsAvailable; +std::optional OutputParser::parseChunk(const std::string& chunkResponse, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason) { + /* +Using appropriate parser based on the current processing phase +Call to this method should return either result from parserContentChunk, parseToolCallChunk, parseReasoningChunk when we can determine the phase +or std::nullopt when we are waiting for more chunks to determine if we should switch phase or not. +Note that mentioned methods do not take chunk as argument, they read it from streamOutputCache and are responsible for clearing the cache, +so only use those methods or return nullopt. +*/ - streamOutputCache.add(chunkResponse); + bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingStartTags().empty() && !reasoningParser->getParsingEndTag().empty(); + bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingStartTags().empty(); + bool applyToolParser = toolParserExistsAndSupportsStreaming && toolsAvailable; - if (processingPhase == UNKNOWN) { - // If we are in the UNKNOWN phase, we need to determine if we should switch to CONTENT, REASONING, or TOOL_CALLS phase. - TagLookupStatus anyStartTagStatus = TagLookupStatus::NOT_FOUND; - if (reasoningParserExistsAndSupportsStreaming) { - // Check if reasoning start tag has been received - TagLookupStatus reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingStartTags()); - if (reasoningStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If reasoning start tag is not found, check if any of the special start tags are found - reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getSpecialParsingStartTags()); - } - if (reasoningStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - return parseReasoningChunk(finishReason); - } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing, so potential tool parser start tag is not missed - anyStartTagStatus = reasoningStartTagStatus; - } + streamOutputCache.add(chunkResponse); - if (applyToolParser) { - // Check if tool call start tag has been received - TagLookupStatus toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); - if (toolCallStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If tool call start tag is not found, check if any of the special start tags are found - toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getSpecialParsingStartTags()); - } - if (toolCallStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - return parseToolCallChunk(finishReason); - } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing - if (toolCallStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE) { - anyStartTagStatus = toolCallStartTagStatus; // We have at least one incomplete start tag - } + if (processingPhase == UNKNOWN) { + // If we are in the UNKNOWN phase, we need to determine if we should switch to CONTENT, REASONING, or TOOL_CALLS phase. + TagLookupStatus anyStartTagStatus = TagLookupStatus::NOT_FOUND; + if (reasoningParserExistsAndSupportsStreaming) { + // Check if reasoning start tag has been received + TagLookupStatus reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingStartTags()); + if (reasoningStartTagStatus == TagLookupStatus::NOT_FOUND) { + // If reasoning start tag is not found, check if any of the special start tags are found + reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getSpecialParsingStartTags()); } + if (reasoningStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { + return parseReasoningChunk(finishReason); + } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing, so potential tool parser start tag is not missed + anyStartTagStatus = reasoningStartTagStatus; + } - if ((!reasoningParserExistsAndSupportsStreaming && !applyToolParser) || finishReason != ov::genai::GenerationFinishReason::NONE || anyStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If no special parsers are available, generation has finished or we have no start tags we just return content chunks and switch to CONTENT phase. - return parseContentChunk(); - } - // If we are here, it means we have incomplete start tag for either reasoning or tool parser, so we wait for more chunks - return std::nullopt; - } else if (processingPhase == REASONING) { - // If we are in the REASONING phase, we check if parsing end tag is found and if so, switch to UNKNOWN phase. - TagLookupStatus endTagStatus = streamOutputCache.lookupTag(reasoningParser->getParsingEndTag()); - if (endTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // Switch back to UNKNOWN phase (we can have either CONTENT or TOOL_CALLS next) - return parseReasoningChunk(finishReason, UNKNOWN); - } else if (endTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete + if (applyToolParser) { + // Check if tool call start tag has been received + TagLookupStatus toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); + if (toolCallStartTagStatus == TagLookupStatus::NOT_FOUND) { + // If tool call start tag is not found, check if any of the special start tags are found + toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getSpecialParsingStartTags()); } - return parseReasoningChunk(finishReason); - } else if (processingPhase == CONTENT) { - // If we are in the CONTENT phase, we check if tool parser start tag is found and if so, switch to TOOL_CALLS phase. - // TOOL_CALLS is the only phase that can be processed after CONTENT. - if (applyToolParser) { - TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); - if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - return parseToolCallChunk(finishReason); - } else if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete - } - return parseContentChunk(); + if (toolCallStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { + return parseToolCallChunk(finishReason); + } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing + if (toolCallStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE) { + anyStartTagStatus = toolCallStartTagStatus; // We have at least one incomplete start tag } + } + + if ((!reasoningParserExistsAndSupportsStreaming && !applyToolParser) || finishReason != ov::genai::GenerationFinishReason::NONE || anyStartTagStatus == TagLookupStatus::NOT_FOUND) { + // If no special parsers are available, generation has finished or we have no start tags we just return content chunks and switch to CONTENT phase. return parseContentChunk(); - } else if (processingPhase == TOOL_CALLS_PROCESSING_TOOL) { - // Processing TOOL_CALLS is the last phase, so we always return the result of tool parser. - TagLookupStatus toolEndTagStatus = streamOutputCache.lookupTag(toolParser->getParsingEndTag()); - if (toolEndTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete - } - if (toolEndTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // If tool call has finished, we switch to waiting for next tool call as tool calls in the last phase, - // so we either get next tool call or finish processing. - return parseToolCallChunk(finishReason, TOOL_CALLS_WAITING_FOR_TOOL); - } - return parseToolCallChunk(finishReason); - } else if (processingPhase == TOOL_CALLS_WAITING_FOR_TOOL) { - // In this phase we are waiting for next tool call or finish of generation. - // If we get next tool call start tag, we switch to TOOL_CALLS phase, otherwise if generation finishes we switch to CONTENT phase to flush any remaining content. + } + // If we are here, it means we have incomplete start tag for either reasoning or tool parser, so we wait for more chunks + return std::nullopt; + } else if (processingPhase == REASONING) { + // If we are in the REASONING phase, we check if parsing end tag is found and if so, switch to UNKNOWN phase. + TagLookupStatus endTagStatus = streamOutputCache.lookupTag(reasoningParser->getParsingEndTag()); + if (endTagStatus == TagLookupStatus::FOUND_COMPLETE) { + // Switch back to UNKNOWN phase (we can have either CONTENT or TOOL_CALLS next) + return parseReasoningChunk(finishReason, UNKNOWN); + } else if (endTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; // Wait for more chunks to determine if end tag is complete + } + return parseReasoningChunk(finishReason); + } else if (processingPhase == CONTENT) { + // If we are in the CONTENT phase, we check if tool parser start tag is found and if so, switch to TOOL_CALLS phase. + // TOOL_CALLS is the only phase that can be processed after CONTENT. + if (applyToolParser) { TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); - if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if start tag is complete - } if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // If tool call has started, we switch back to processing tool phase. - return parseToolCallChunk(finishReason, TOOL_CALLS_PROCESSING_TOOL); + return parseToolCallChunk(finishReason); + } else if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; // Wait for more chunks to determine if end tag is complete } - return parseToolCallChunk(finishReason); - } else { - SPDLOG_LOGGER_ERROR(llm_calculator_logger, "Unexpected processing phase: {}", static_cast(processingPhase)); - throw std::runtime_error("Unexpected error during stream output parsing"); + return parseContentChunk(); + } + return parseContentChunk(); + } else if (processingPhase == TOOL_CALLS_PROCESSING_TOOL) { + // Processing TOOL_CALLS is the last phase, so we always return the result of tool parser. + TagLookupStatus toolEndTagStatus = streamOutputCache.lookupTag(toolParser->getParsingEndTag()); + if (toolEndTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; // Wait for more chunks to determine if end tag is complete + } + if (toolEndTagStatus == TagLookupStatus::FOUND_COMPLETE) { + // If tool call has finished, we switch to waiting for next tool call as tool calls in the last phase, + // so we either get next tool call or finish processing. + return parseToolCallChunk(finishReason, TOOL_CALLS_WAITING_FOR_TOOL); + } + return parseToolCallChunk(finishReason); + } else if (processingPhase == TOOL_CALLS_WAITING_FOR_TOOL) { + // In this phase we are waiting for next tool call or finish of generation. + // If we get next tool call start tag, we switch to TOOL_CALLS phase, otherwise if generation finishes we switch to CONTENT phase to flush any remaining content. + TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); + if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; // Wait for more chunks to determine if start tag is complete + } + if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { + // If tool call has started, we switch back to processing tool phase. + return parseToolCallChunk(finishReason, TOOL_CALLS_PROCESSING_TOOL); } + return parseToolCallChunk(finishReason); + } else { + SPDLOG_LOGGER_ERROR(llm_calculator_logger, "Unexpected processing phase: {}", static_cast(processingPhase)); + throw std::runtime_error("Unexpected error during stream output parsing"); } +} } // namespace ovms diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index 20c97068d1..f267b17294 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include "../../../logging.hpp" #include "../../../status.hpp" From c6f3835481201baf9d8da60480bc56b6716d32cb Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 11:48:12 +0200 Subject: [PATCH 24/33] styles --- src/llm/io_processing/output_parser.cpp | 4 ++-- src/llm/io_processing/utils.cpp | 26 ------------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index e888b9f7ab..d581423bdd 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -168,7 +168,7 @@ std::optional OutputParser::parseReasoningChunk(ov::genai:: return result; } -OutputParser::OutputParser(ov::genai::Tokenizer & tokenizer, const std::string toolParserName, const std::string reasoningParserName, const ToolsSchemas_t& toolNameSchemaMap) : +OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string toolParserName, const std::string reasoningParserName, const ToolsSchemas_t& toolNameSchemaMap) : tokenizer(tokenizer) { if (toolParserName == "llama3") { toolParser = std::make_unique(tokenizer); @@ -205,7 +205,7 @@ OutputParser::OutputParser(ov::genai::Tokenizer & tokenizer, const std::string t if (toolParser && reasoningParser) { if (toolParser->requiresStreamingWithSpecialTokens() != reasoningParser->requiresStreamingWithSpecialTokens()) { throw std::runtime_error("Cannot use tool parser " + toolParserName + " with reasoning parser " + reasoningParserName + - " as they have different requirements for special tokens in streaming mode"); + " as they have different requirements for special tokens in streaming mode"); } } } diff --git a/src/llm/io_processing/utils.cpp b/src/llm/io_processing/utils.cpp index bc41094762..f42bac08ae 100644 --- a/src/llm/io_processing/utils.cpp +++ b/src/llm/io_processing/utils.cpp @@ -35,32 +35,6 @@ std::string generateRandomId() { } return id; } -void writeArgumentOfAnyType(const rapidjson::Value& arg, rapidjson::Writer& writer) { - if (arg.IsString()) { - writer.String(arg.GetString()); - } else if (arg.IsInt64()) { - writer.Int64(arg.GetInt64()); - } else if (arg.IsDouble()) { - writer.Double(arg.GetDouble()); - } else if (arg.IsBool()) { - writer.Bool(arg.GetBool()); - } else if (arg.IsArray()) { - writer.StartArray(); - for (auto& elem : arg.GetArray()) { - writeArgumentOfAnyType(elem, writer); - } - writer.EndArray(); - } else if (arg.IsObject()) { - writer.StartObject(); - for (auto it = arg.MemberBegin(); it != arg.MemberEnd(); ++it) { - writer.Key(it->name.GetString()); - writeArgumentOfAnyType(it->value, writer); - } - writer.EndObject(); - } else { - writer.String(""); - } -} void writeArgumentOfAnyType(const rapidjson::Value& arg, rapidjson::Writer& writer) { if (arg.IsString()) { From f63d78ef5c3097c11c6b053829bd052fe457879f Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 14:06:03 +0200 Subject: [PATCH 25/33] streaming reasoning, review --- .../io_processing/gemma4/reasoning_parser.cpp | 38 +++++++++++++++++-- .../io_processing/gemma4/reasoning_parser.hpp | 19 +++++++++- src/llm/io_processing/gemma4/tool_parser.cpp | 2 + src/llm/io_processing/gemma4/tool_parser.hpp | 6 +++ src/llm/io_processing/output_parser.cpp | 32 ++++++++-------- .../visual_language_model/legacy/servable.cpp | 7 ++++ src/test/http_openai_handler_test.cpp | 1 - 7 files changed, 83 insertions(+), 22 deletions(-) diff --git a/src/llm/io_processing/gemma4/reasoning_parser.cpp b/src/llm/io_processing/gemma4/reasoning_parser.cpp index f404a1ad55..a666477191 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.cpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.cpp @@ -26,16 +26,48 @@ namespace ovms { void Gemma4ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - size_t startPos = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningTokenId) - generatedTokens.begin(); - size_t endPos = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId) - generatedTokens.begin(); + auto startPos = std::string::npos; + auto endPos = std::string::npos; + + auto startIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningTokenId); + auto endIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId); + + if (startIt != generatedTokens.end() && endIt != generatedTokens.end() && startIt < endIt) { + startPos = std::distance(generatedTokens.begin(), startIt); + endPos = std::distance(generatedTokens.begin(), endIt); + } if (startPos != std::string::npos && endPos != std::string::npos && startPos < endPos) { size_t reasoningStart = startPos + 3; // deleting "<|channel>thought\n" std::string reasoningText = tokenizer.decode(std::vector(generatedTokens.begin() + reasoningStart, generatedTokens.begin() + endPos), ov::genai::skip_special_tokens(true)); parsedOutput.reasoning = reasoningText; // Remove reasoning from content - std::string contentWithoutReasoning = tokenizer.decode(std::vector(generatedTokens.begin() + endPos + 1, generatedTokens.end()), ov::genai::skip_special_tokens(true)); + std::string contentWithoutReasoning = tokenizer.decode(std::vector(generatedTokens.begin() + endPos + 1, generatedTokens.end()), ov::genai::skip_special_tokens(true)); // content MUST never appear before reasoning parsedOutput.content = contentWithoutReasoning; } } +std::optional Gemma4ReasoningParser::parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) { + if (chunk.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for Gemma4ReasoningParser"); + return std::nullopt; + } + + if (chunk.find(getParsingStartTags()[0]) != std::string::npos || chunk.find(getParsingEndTag()) != std::string::npos) { + return std::nullopt; + } else { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("reasoning_content"); + writer.String(chunk.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + return doc; + } + return std::nullopt; +} } // namespace ovms diff --git a/src/llm/io_processing/gemma4/reasoning_parser.hpp b/src/llm/io_processing/gemma4/reasoning_parser.hpp index f692df45af..0ea1eef9ab 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.hpp @@ -23,17 +23,32 @@ namespace ovms { class Gemma4ReasoningParser : public Qwen3ReasoningParser { protected: - const int64_t reasoningTokenId = 100; - const int64_t reasoningEndTokenId = 101; + const int64_t reasoningTokenId = 100; // <|channel> + const int64_t reasoningEndTokenId = 101; // + const std::string parsingStartTag = "<|channel>thought\n"; + const std::string parsingEndTag = ""; public: Gemma4ReasoningParser() = delete; explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer) : Qwen3ReasoningParser(tokenizer) {} void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override; bool requiresStreamingWithSpecialTokens() const override { return true; } + + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{this->parsingStartTag}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return parsingEndTag; + } }; } // namespace ovms diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 0bec0b0d5e..18693f0ac8 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -33,6 +33,8 @@ const std::string Gemma4ToolParser::TOOL_ARGS_END_INDICATOR = "}"; const std::string Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR = "<|\"|>"; const std::string Gemma4ToolParser::TOOL_ARGS_SEPARATOR_STR = ","; +const std::string Gemma4ToolParser::TURN_END_TAG = ""; + const int64_t Gemma4ToolParser::botTokenId = 48; const int64_t Gemma4ToolParser::eotTokenId = 49; diff --git a/src/llm/io_processing/gemma4/tool_parser.hpp b/src/llm/io_processing/gemma4/tool_parser.hpp index 917fd67e3a..24075968f1 100644 --- a/src/llm/io_processing/gemma4/tool_parser.hpp +++ b/src/llm/io_processing/gemma4/tool_parser.hpp @@ -30,6 +30,7 @@ class Gemma4ToolParser : public BaseOutputParser { static const std::string TOOL_ARGS_END_INDICATOR; static const std::string TOOL_ARGS_STRING_INDICATOR; static const std::string TOOL_ARGS_SEPARATOR_STR; + static const std::string TURN_END_TAG; static const int64_t botTokenId; static const int64_t eotTokenId; @@ -56,6 +57,11 @@ class Gemma4ToolParser : public BaseOutputParser { return parsingStartTags; } + const std::vector& getSpecialTagsToErase() const override { + static const std::vector tagsToErase = {TURN_END_TAG}; + return tagsToErase; + } + const std::vector& getSpecialParsingStartTags() const override { static const std::vector beginningOnlyTags = {}; return beginningOnlyTags; diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index d581423bdd..b1c62c6a50 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -40,10 +40,10 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s } if (tag.size() > buffer.size()) { /* - If the tag is longer than the buffer, we check if the buffer and tag overlap (either partially or fully for exact match) - They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ + If the tag is longer than the buffer, we check if the buffer and tag overlap (either partially or fully for exact match) + They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE + otherwise we return NOT_FOUND + */ if (stringsOverlap(buffer, tag)) { return TagLookupStatus::FOUND_INCOMPLETE; } else { @@ -51,12 +51,12 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s } } else if (tag.size() < buffer.size()) { /* - If the tag is shorter than the buffer, we check: - a) if the tag is a substring of the buffer (tag is fully matched) - b) if the buffer and tag overlap (part of the tag is matched) - in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ + If the tag is shorter than the buffer, we check: + a) if the tag is a substring of the buffer (tag is fully matched) + b) if the buffer and tag overlap (part of the tag is matched) + in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE + otherwise we return NOT_FOUND + */ if (buffer.find(tag) != std::string::npos) { return TagLookupStatus::FOUND_COMPLETE; } else if (stringsOverlap(buffer, tag)) { @@ -66,12 +66,12 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s } } else { /* - If the tag and buffer are of the same length, we check: - a) if they are equal (tag is fully matched) - b) if they overlap (part of the tag is matched) - in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ + If the tag and buffer are of the same length, we check: + a) if they are equal (tag is fully matched) + b) if they overlap (part of the tag is matched) + in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE + otherwise we return NOT_FOUND + */ if (buffer == tag) { return TagLookupStatus::FOUND_COMPLETE; } else if (stringsOverlap(buffer, tag)) { diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index f267b17294..d00316c3e4 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -179,6 +179,13 @@ absl::Status VisualLanguageModelLegacyServable::readCompleteExecutionResults(std absl::Status VisualLanguageModelLegacyServable::prepareCompleteResponse(std::shared_ptr& executionContext) { auto legacyExecutionContext = std::static_pointer_cast(executionContext); + if (legacyExecutionContext->payload.client->isDisconnected()) { + return absl::CancelledError(); + } + + // TODO(mzegla): Usage of streaming flow here due to GenAI generate limitations. + // This diverges from the general flow - we should have unified systematic approach. + executionContext->textStreamer->end(); std::string completeText; diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 219c8955d7..c3a40cba3c 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -3091,7 +3091,6 @@ TEST_F(HttpOpenAIHandlerParsingTest, SerializeUnaryResponseVLMDecodedResultsWith ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - std::string toolCallContent = "I will call a tool.{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Paris\"}}"; ov::genai::VLMDecodedResults results; std::string vlmText = "I will call a tool.{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Paris\"}}"; From da43f5cdaa629829a9518a65d959eee7fbbbf69d Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 14:08:17 +0200 Subject: [PATCH 26/33] clang format --- src/llm/io_processing/gemma4/reasoning_parser.cpp | 2 +- src/llm/io_processing/gemma4/reasoning_parser.hpp | 9 +++++---- src/llm/visual_language_model/legacy/servable.cpp | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/llm/io_processing/gemma4/reasoning_parser.cpp b/src/llm/io_processing/gemma4/reasoning_parser.cpp index a666477191..3e1c4cb4f2 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.cpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.cpp @@ -42,7 +42,7 @@ void Gemma4ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector< std::string reasoningText = tokenizer.decode(std::vector(generatedTokens.begin() + reasoningStart, generatedTokens.begin() + endPos), ov::genai::skip_special_tokens(true)); parsedOutput.reasoning = reasoningText; // Remove reasoning from content - std::string contentWithoutReasoning = tokenizer.decode(std::vector(generatedTokens.begin() + endPos + 1, generatedTokens.end()), ov::genai::skip_special_tokens(true)); // content MUST never appear before reasoning + std::string contentWithoutReasoning = tokenizer.decode(std::vector(generatedTokens.begin() + endPos + 1, generatedTokens.end()), ov::genai::skip_special_tokens(true)); // content MUST never appear before reasoning parsedOutput.content = contentWithoutReasoning; } } diff --git a/src/llm/io_processing/gemma4/reasoning_parser.hpp b/src/llm/io_processing/gemma4/reasoning_parser.hpp index 0ea1eef9ab..eb363eb414 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.hpp @@ -23,11 +23,12 @@ namespace ovms { class Gemma4ReasoningParser : public Qwen3ReasoningParser { protected: - const int64_t reasoningTokenId = 100; // <|channel> - const int64_t reasoningEndTokenId = 101; // + const int64_t reasoningTokenId = 100; // <|channel> + const int64_t reasoningEndTokenId = 101; // const std::string parsingStartTag = "<|channel>thought\n"; const std::string parsingEndTag = ""; + public: Gemma4ReasoningParser() = delete; explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer) : @@ -38,8 +39,8 @@ class Gemma4ReasoningParser : public Qwen3ReasoningParser { bool requiresStreamingWithSpecialTokens() const override { return true; } - - const std::vector& getParsingStartTags() const override { + + const std::vector& getParsingStartTags() const override { static const std::vector parsingStartTags{this->parsingStartTag}; return parsingStartTags; } diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index d00316c3e4..6ca45b0b03 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -185,7 +185,7 @@ absl::Status VisualLanguageModelLegacyServable::prepareCompleteResponse(std::sha // TODO(mzegla): Usage of streaming flow here due to GenAI generate limitations. // This diverges from the general flow - we should have unified systematic approach. - + executionContext->textStreamer->end(); std::string completeText; From 6eab061fb2c70addaffa3c00451fdc1f07bd0810 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 14:09:57 +0200 Subject: [PATCH 27/33] cpplint --- src/llm/io_processing/gemma4/reasoning_parser.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llm/io_processing/gemma4/reasoning_parser.hpp b/src/llm/io_processing/gemma4/reasoning_parser.hpp index eb363eb414..f4e10bda96 100644 --- a/src/llm/io_processing/gemma4/reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/reasoning_parser.hpp @@ -17,6 +17,7 @@ #include #include +#include #include "../qwen3/reasoning_parser.hpp" From 8823f5a0c937e93bdaf2b76be6abc96565c190ac Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 14:14:15 +0200 Subject: [PATCH 28/33] review --- src/llm/io_processing/gemma4/tool_parser.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 18693f0ac8..c533defde5 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -35,11 +35,11 @@ const std::string Gemma4ToolParser::TOOL_ARGS_SEPARATOR_STR = ","; const std::string Gemma4ToolParser::TURN_END_TAG = ""; -const int64_t Gemma4ToolParser::botTokenId = 48; -const int64_t Gemma4ToolParser::eotTokenId = 49; +const int64_t Gemma4ToolParser::botTokenId = 48; // <|tool_call> +const int64_t Gemma4ToolParser::eotTokenId = 49; // -const int64_t Gemma4ToolParser::reasoningTokenId = 100; -const int64_t Gemma4ToolParser::reasoningEndTokenId = 101; +const int64_t Gemma4ToolParser::reasoningTokenId = 100; // <|channel> +const int64_t Gemma4ToolParser::reasoningEndTokenId = 101; // std::string Gemma4ToolParser::parseArrayParameter(const std::string& argumentStr) { size_t pos = 1; @@ -72,7 +72,7 @@ std::string Gemma4ToolParser::parseArrayParameter(const std::string& argumentStr return parsedArguments; } -std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { +std::string Gemma4ToolParser::parseObjectParameter(const std::string& argumentStr) { size_t pos = 1; std::vector> keyValuePairs; @@ -85,7 +85,7 @@ std::string Gemma4ToolParser::parseObjectParameter(std::string argumentStr) { } key = argumentStr.substr(pos, keyEndPos - pos); size_t valueStartPos = keyEndPos + 1; - size_t valueEndPos; + size_t valueEndPos = std::string::npos; if (argumentStr.substr(valueStartPos, TOOL_ARGS_STRING_INDICATOR.size()) == TOOL_ARGS_STRING_INDICATOR) { valueStartPos = valueStartPos + TOOL_ARGS_STRING_INDICATOR.size(); valueEndPos = argumentStr.find(TOOL_ARGS_STRING_INDICATOR, valueStartPos); @@ -445,7 +445,9 @@ void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector Date: Fri, 8 May 2026 14:17:39 +0200 Subject: [PATCH 29/33] missing changes --- src/llm/io_processing/gemma4/tool_parser.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.hpp b/src/llm/io_processing/gemma4/tool_parser.hpp index 24075968f1..c33d32b6c9 100644 --- a/src/llm/io_processing/gemma4/tool_parser.hpp +++ b/src/llm/io_processing/gemma4/tool_parser.hpp @@ -77,7 +77,7 @@ class Gemma4ToolParser : public BaseOutputParser { static std::string normalizeArgStr(const std::string& arg); static std::string parseArrayParameter(const std::string& argumentStr); - static std::string parseObjectParameter(std::string argumentStr); + static std::string parseObjectParameter(const std::string& argumentStr); private: void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer); From c5f20948ecf21a589b78600851c0b5cc5cbbddfa Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 15:30:57 +0200 Subject: [PATCH 30/33] fix multiturn args parsing --- .../gemma4/gemma4_reasoning_parser.cpp | 67 ++++++++++ .../gemma4/gemma4_reasoning_parser.hpp | 46 +++++++ src/llm/io_processing/gemma4/tool_parser.cpp | 8 +- src/llm/io_processing/output_parser.cpp | 20 +-- src/llm/io_processing/utils.cpp | 2 +- .../gemma4_output_parser_test.cpp | 120 +++++++++++++++++- 6 files changed, 245 insertions(+), 18 deletions(-) create mode 100644 src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp create mode 100644 src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp diff --git a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp new file mode 100644 index 0000000000..0c933d8657 --- /dev/null +++ b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp @@ -0,0 +1,67 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "../../../logging.hpp" +#include "gemma4_reasoning_parser.hpp" +#include "../utils.hpp" + +namespace ovms { +void Gemma4ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + std::string startReasoningTag = getParsingStartTags()[0]; + std::string endReasoningTag = getParsingEndTag(); + size_t startPos = parsedOutput.content.find(startReasoningTag); + size_t endPos = parsedOutput.content.find(endReasoningTag); + + if (startPos != std::string::npos && endPos != std::string::npos && startPos < endPos) { + size_t reasoningStart = startPos + startReasoningTag.length(); + std::string reasoningText = parsedOutput.content.substr(reasoningStart, endPos - reasoningStart); + parsedOutput.reasoning = reasoningText; + // Remove reasoning from content + parsedOutput.content.erase(startPos, endPos - startPos + endReasoningTag.length()); + } +} + +std::optional Gemma4ReasoningParser::parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) { + if (chunk.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for Gemma4ReasoningParser"); + return std::nullopt; + } + + if (chunk.find(getParsingStartTags()[0]) != std::string::npos || chunk.find(getParsingEndTag()) != std::string::npos) { + return std::nullopt; + } else { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("reasoning_content"); + writer.String(chunk.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + return doc; + } + return std::nullopt; +} +} // namespace ovms diff --git a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp new file mode 100644 index 0000000000..9a83ea969c --- /dev/null +++ b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include "../base_output_parser.hpp" + +namespace ovms { +class Gemma4ReasoningParser : public BaseOutputParser { +protected: + // Tags used to identify the reasoning segment in the content + std::string parsingStartTag = "<|channel>thought\n"; + std::string parsingEndTag = ""; + +public: + Gemma4ReasoningParser() = delete; + explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{this->parsingStartTag}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return parsingEndTag; + } +}; +} // namespace ovms diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index c533defde5..29dc4be648 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -35,11 +35,11 @@ const std::string Gemma4ToolParser::TOOL_ARGS_SEPARATOR_STR = ","; const std::string Gemma4ToolParser::TURN_END_TAG = ""; -const int64_t Gemma4ToolParser::botTokenId = 48; // <|tool_call> -const int64_t Gemma4ToolParser::eotTokenId = 49; // +const int64_t Gemma4ToolParser::botTokenId = 48; // <|tool_call> +const int64_t Gemma4ToolParser::eotTokenId = 49; // -const int64_t Gemma4ToolParser::reasoningTokenId = 100; // <|channel> -const int64_t Gemma4ToolParser::reasoningEndTokenId = 101; // +const int64_t Gemma4ToolParser::reasoningTokenId = 100; // <|channel> +const int64_t Gemma4ToolParser::reasoningEndTokenId = 101; // std::string Gemma4ToolParser::parseArrayParameter(const std::string& argumentStr) { size_t pos = 1; diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index b1c62c6a50..d0a99002cb 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -52,8 +52,8 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s } else if (tag.size() < buffer.size()) { /* If the tag is shorter than the buffer, we check: - a) if the tag is a substring of the buffer (tag is fully matched) - b) if the buffer and tag overlap (part of the tag is matched) + a) if the tag is a substring of the buffer (tag is fully matched) + b) if the buffer and tag overlap (part of the tag is matched) in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE otherwise we return NOT_FOUND */ @@ -67,8 +67,8 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s } else { /* If the tag and buffer are of the same length, we check: - a) if they are equal (tag is fully matched) - b) if they overlap (part of the tag is matched) + a) if they are equal (tag is fully matched) + b) if they overlap (part of the tag is matched) in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE otherwise we return NOT_FOUND */ @@ -248,12 +248,12 @@ ParsedOutput OutputParser::parse(const std::vector& generatedTokens, co std::optional OutputParser::parseChunk(const std::string& chunkResponse, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason) { /* -Using appropriate parser based on the current processing phase -Call to this method should return either result from parserContentChunk, parseToolCallChunk, parseReasoningChunk when we can determine the phase -or std::nullopt when we are waiting for more chunks to determine if we should switch phase or not. -Note that mentioned methods do not take chunk as argument, they read it from streamOutputCache and are responsible for clearing the cache, -so only use those methods or return nullopt. -*/ + Using appropriate parser based on the current processing phase + Call to this method should return either result from parserContentChunk, parseToolCallChunk, parseReasoningChunk when we can determine the phase + or std::nullopt when we are waiting for more chunks to determine if we should switch phase or not. + Note that mentioned methods do not take chunk as argument, they read it from streamOutputCache and are responsible for clearing the cache, + so only use those methods or return nullopt. + */ bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingStartTags().empty() && !reasoningParser->getParsingEndTag().empty(); bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingStartTags().empty(); diff --git a/src/llm/io_processing/utils.cpp b/src/llm/io_processing/utils.cpp index f42bac08ae..c58ed1ccf0 100644 --- a/src/llm/io_processing/utils.cpp +++ b/src/llm/io_processing/utils.cpp @@ -85,7 +85,7 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str bracketDepth--; } else if (str[i] == '"' && (i == 0 || str[i - 1] != '\\')) { quoteDepth = 1 - quoteDepth; - } else if (str[i] == '\'' && (i == 0 || str[i - 1] != '\\')) { + } else if (quoteDepth == 0 && str[i] == '\'' && (i == 0 || str[i - 1] != '\\')) { singleQuoteDepth = 1 - singleQuoteDepth; } } diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 066d47d269..17ab3d39ca 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -257,9 +257,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithArrayArguments) { } TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { - std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}" - "<|tool_call>call:another_tool{param1:<|\"|>data<|\"|>,param2:true}" - "<|tool_call>call:third_tool{key:<|\"|>value<|\"|>}"; + std::string inputWithProperClosure = "<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}call:another_tool{param1:<|\"|>data<|\"|>,param2:true}call:third_tool{key:<|\"|>value<|\"|>}"; std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { @@ -336,6 +334,105 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyArguments) { ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "no_args_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, ""); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithMultipleUtfChars) { + // Tool call with empty braces (no arguments) and content around + std::string input = R"(<|tool_call>call:post_tweet{content:<|"|>Check out the sorted report! 🚀 We've made improvements to the content. Tagging @currenttech and mentioning Julia for our insightful team. #currenttech #trend<|"|>,mentions:[<|"|>@currenttech<|"|>,<|"|>Julia<|"|>],tags:[<|"|>#currenttrend<|"|>]})"; + auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "post_tweet"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"content":"Check out the sorted report! 🚀 We've made improvements to the content. Tagging @currenttech and mentioning Julia for our insightful team. #currenttech #trend","mentions":["@currenttech","Julia"],"tags":["#currenttrend"]})"); +} + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithMultipleUtfCharsStreaming) { + std::vector>> chunkToDeltaVec{ + {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"call:", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"post", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"_tweet", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{content", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"post_tweet"}}]}})"}, + {":<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Check out the sorted report!", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 🚀", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" We've made improvements", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" to the content.", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" Tagging @currenttech", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" and mentioning Julia", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" for our insightful team.", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" #currenttech #trend", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"mentions", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"@currenttech", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Julia", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"tags", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {":[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"#currenttrend", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"]}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"content\":\"Check out the sorted report! 🚀 We've made improvements to the content. Tagging @currenttech and mentioning Julia for our insightful team. #currenttech #trend\",\"mentions\":[\"@currenttech\",\"Julia\"],\"tags\":[\"#currenttrend\"]}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk (ignoring id value): " << chunk; + } else { + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; + } + } else { + std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; + std::string docStr = doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "std::nullopt"; + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk + << "\nexpectedDelta: " << expectedStr + << "\ndoc: " << docStr; + } + } } TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { @@ -795,3 +892,20 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"text":"zażółć gęślą jaźń","lang":"pl"})"); } + +TEST_F(Gemma4OutputParserTest, ParseToolCallWithPythonCodeAsArgument) { + std::string input = R"x(<|tool_call>call:string_tool{param:<|"|> + if __name__ == "__main__": + addresses = {} + addresses["Hodor"] = """The door""" + addresses["Arya"] = "Winterfell" + for name, address in addresses.items(): + print(f'\n\t{name} lives at {address}\n\r')<|"|>})x"; + auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "string_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"x({"param":"\n if __name__ == \"__main__\":\n addresses = {}\n addresses[\"Hodor\"] = \"\"\"The door\"\"\"\n addresses[\"Arya\"] = \"Winterfell\"\n for name, address in addresses.items():\n print(f'\\n\\t{name} lives at {address}\\n\\r')"})x"); +} From a907b83b1da45450a0b6c203177fafd3662e7be6 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 15:34:31 +0200 Subject: [PATCH 31/33] cpplint --- src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp index 9a83ea969c..f4a6f48a41 100644 --- a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp @@ -14,6 +14,8 @@ // limitations under the License. //***************************************************************************** #pragma once +#include +#include #include "../base_output_parser.hpp" From 0faff458d2798fb7cefde3f249663414c5ba993e Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 15:38:10 +0200 Subject: [PATCH 32/33] fix test --- src/test/llm/output_parsers/gemma4_output_parser_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 17ab3d39ca..8a383c9ead 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -334,7 +334,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyArguments) { ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "no_args_tool"); - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, ""); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{}"); } TEST_F(Gemma4OutputParserTest, ParseToolCallWithMultipleUtfChars) { From 894b794b99400ef95abcb7d235e7e09823843fe3 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Fri, 8 May 2026 15:52:55 +0200 Subject: [PATCH 33/33] whitespace tests --- src/llm/io_processing/gemma4/tool_parser.cpp | 2 ++ .../gemma4_output_parser_test.cpp | 17 ++++------------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/llm/io_processing/gemma4/tool_parser.cpp b/src/llm/io_processing/gemma4/tool_parser.cpp index 29dc4be648..33332cee90 100644 --- a/src/llm/io_processing/gemma4/tool_parser.cpp +++ b/src/llm/io_processing/gemma4/tool_parser.cpp @@ -264,6 +264,7 @@ bool Gemma4ToolParser::parseInToolCallState() { } std::string toolName = this->streamingContent.substr(toolNameStart, argsPos - toolNameStart); + trim(toolName); this->toolCall = ToolCall{generateRandomId(), toolName, ""}; SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); this->streamingPosition = argsPos + TOOL_ARGS_START_INDICATOR.length(); @@ -415,6 +416,7 @@ bool Gemma4ToolParser::parseSingleToolCall(const std::string& toolStr, ToolCall& return false; } std::string toolName = toolNameWithPrefix.substr(TOOL_CALL_NAME_PREFIX.length()); + trim(toolName); SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); int argsStrLen = toolStr.length() - argsPos - TOOL_ARGS_START_INDICATOR.length() - TOOL_ARGS_END_INDICATOR.length(); diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 8a383c9ead..2cd39e3cc2 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -613,10 +613,11 @@ TEST_F(Gemma4OutputParserTest, StreamingWithBiggerChunks) { } } -TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { +TEST_F(Gemma4OutputParserTest, StreamingWithWhitespacesBetweenToolCalls) { std::vector>> chunkToDeltaVec{ {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"call:sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"{array", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, {":[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, @@ -635,14 +636,7 @@ TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"<|\"|>}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, - {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"Some ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Some "}})"}, - {"content ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"content "}})"}, - {"between ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"between "}})"}, - {"tool ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tool "}})"}, - {"calls.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"calls."}})"}, - {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"call:d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" call:d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"{config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"dummy"}}]}})"}, {":{", ov::genai::GenerationFinishReason::NONE, std::nullopt}, @@ -656,10 +650,7 @@ TEST_F(Gemma4OutputParserTest, StreamingWithContentBetweenToolCalls) { {":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"}}", ov ::genai ::GenerationFinishReason ::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, - {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, - {"<|tool_call>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"call:solve", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"call: solve", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"{e", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"solve"}}]}})"}, {"quation", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {":<|\"|>", ov::genai::GenerationFinishReason::NONE, std::nullopt},