diff --git a/src/genai/instrumentations/langchain/utils.ts b/src/genai/instrumentations/langchain/utils.ts index 11f0caa..9a92d96 100644 --- a/src/genai/instrumentations/langchain/utils.ts +++ b/src/genai/instrumentations/langchain/utils.ts @@ -461,12 +461,24 @@ export function getRequestModel(run: Run): string | undefined { // served the request). export function getResponseModel(run: Run): string | undefined { const llmOutput = run.outputs?.llmOutput as Record | undefined; + const v1Metadata = run.outputs?.generations?.[0]?.[0]?.message?.response_metadata as + | Record + | undefined; + const v0Metadata = run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.response_metadata as + | Record + | undefined; + return [ - // v1: response_metadata directly on message - run.outputs?.generations?.[0]?.[0]?.message?.response_metadata?.model_name, - // v0: response_metadata nested under kwargs - run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.response_metadata?.model_name, - // LLMResult.llmOutput.model_name (common for Chat models) + // v1: response_metadata directly on message. Prefer the canonical OpenAI + // Responses-API field (`model`) and fall back to the `model_name` alias + // LangChain keeps "for backwards compat with chat completion calls" (see + // langchain-ai/langchainjs libs/providers/langchain-openai/src/converters/responses.ts). + v1Metadata?.model, + v1Metadata?.model_name, + // v0: response_metadata nested under kwargs. + v0Metadata?.model, + v0Metadata?.model_name, + // LLMResult.llmOutput.* (common for Chat Completions API). llmOutput?.model_name, llmOutput?.model, ] diff --git a/test/internal/unit/genai/langchain/tracer.test.ts b/test/internal/unit/genai/langchain/tracer.test.ts index 25799a3..dec2f17 100644 --- a/test/internal/unit/genai/langchain/tracer.test.ts +++ b/test/internal/unit/genai/langchain/tracer.test.ts @@ -16,6 +16,11 @@ import { ATTR_ERROR_MESSAGE, ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_PROVIDER_NAME, + ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_RESPONSE_ID, + ATTR_GEN_AI_RESPONSE_MODEL, + ATTR_GEN_AI_USAGE_INPUT_TOKENS, + ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, } from "../../../../../src/genai/index.js"; function makeRun(overrides: Partial = {}): Run { @@ -318,4 +323,89 @@ describe("LangChainTracer", () => { assert.strictEqual(startSpanCalls.length, 0, "should not create span when MAX_RUNS exceeded"); }); }); + + // End-to-end shape test for #128 scenario 3 / #150. Mirrors the Run object + // LangChain's `langchain-openai/converters/responses.ts` produces for a single + // ChatOpenAI invocation with `useResponsesApi: true`, and asserts the full + // bundle of GenAI semconv attributes lands on the span. + describe("Responses API (useResponsesApi: true) end-to-end", () => { + it("populates request/response model, response id, and token attrs on a RAPI run", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + + const runId = "rapi-run-1"; + const startedRun = makeRun({ + id: runId, + run_type: "llm", + name: "ChatOpenAI", + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { + metadata: { ls_model_name: "deployment-o4-mini", ls_provider: "openai" }, + invocation_params: { model: "deployment-o4-mini" }, + }, + inputs: { + messages: [[{ role: "user", content: "hello" }]], + }, + }); + await lct.onRunCreate(startedRun); + + const completedRun = { + ...startedRun, + outputs: { + generations: [ + [ + { + message: { + // Shape produced by libs/providers/langchain-openai/src/converters/responses.ts. + // We pin distinct sentinels on the fields the test is supposed + // to ignore so the assertions actually prove that `model` + // (canonical) and `response_metadata.id` (provider-supplied) + // win over the `model_name` alias and `message.id`. + response_metadata: { + model_provider: "openai", + model: "o4-mini-2025-04-16", + model_name: "model_name-alias-should-be-ignored", + id: "resp_rapi_abc", + created_at: 1_700_000_000, + }, + usage_metadata: { + input_tokens: 4, + output_tokens: 7, + }, + id: "message-id-should-be-ignored", + }, + }, + ], + ], + }, + } as unknown as Run; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(completedRun); + + const span = tracer.lastSpan!; + const calls = (span.setAttribute as ReturnType).mock.calls; + const got = (key: string) => calls.find((c: unknown[]) => c[0] === key)?.[1]; + + assert.strictEqual(got(ATTR_GEN_AI_OPERATION_NAME), "chat", "operation name"); + assert.strictEqual( + got(ATTR_GEN_AI_REQUEST_MODEL), + "deployment-o4-mini", + "request model should be the deployment alias", + ); + assert.strictEqual( + got(ATTR_GEN_AI_RESPONSE_MODEL), + "o4-mini-2025-04-16", + "response model should come from response_metadata.model (not the model_name alias)", + ); + assert.strictEqual( + got(ATTR_GEN_AI_RESPONSE_ID), + "resp_rapi_abc", + "response id should come from response_metadata.id (not message.id)", + ); + assert.strictEqual(got(ATTR_GEN_AI_USAGE_INPUT_TOKENS), 4, "input tokens"); + assert.strictEqual(got(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS), 7, "output tokens"); + assert.strictEqual(span.statusObj?.code, SpanStatusCode.OK); + }); + }); }); diff --git a/test/internal/unit/genai/langchain/utils.test.ts b/test/internal/unit/genai/langchain/utils.test.ts index 822dd32..237be2d 100644 --- a/test/internal/unit/genai/langchain/utils.test.ts +++ b/test/internal/unit/genai/langchain/utils.test.ts @@ -544,6 +544,175 @@ describe("setModelAttribute", () => { ), ); }); + + // Responses API (useResponsesApi: true) — LangChain's openai provider + // populates response_metadata.model (canonical) and, for backwards compat + // with chat completion calls, also response_metadata.model_name. We must + // honor both shapes so non-OpenAI RAPI providers (e.g. @langchain/perplexity) + // and any future major where the model_name alias is dropped keep working. + it("RAPI v1: extracts response model from response_metadata.model when only `model` is set", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "o4-mini-2025-04-16", + model_provider: "openai", + id: "resp_abc", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "deployment-o4-mini", + ), + "request model should come from invocation_params.model for non-Azure RAPI clients", + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + "response model should be sourced from response_metadata.model (RAPI canonical field)", + ); + }); + + it("RAPI v1: prefers response_metadata.model over response_metadata.model_name when both are present", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "o4-mini-2025-04-16", + // LangChain duplicates `model` into `model_name` "for + // backwards compat with chat completion calls". We pin a + // distinct sentinel here so the assertion proves we read + // the canonical `model` field first rather than coupling + // to the `model_name` alias. + model_name: "model_name-alias-should-be-ignored", + model_provider: "openai", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + "response model should come from response_metadata.model (canonical)", + ); + assert.ok( + !calls.some( + (c: unknown[]) => + c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "model_name-alias-should-be-ignored", + ), + "response model must not fall back to the model_name alias when model is set", + ); + }); + + it("RAPI v0: extracts response model from kwargs.response_metadata.model", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + kwargs: { + response_metadata: { + model: "o4-mini-2025-04-16", + model_provider: "openai", + }, + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "deployment-o4-mini", + ), + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + ); + }); + + it("AzureChatOpenAI + RAPI: response_metadata.model still drives the workaround request model", () => { + // Combines the AzureChatOpenAI ls_model_name=gpt-3.5-turbo regression (see + // langchain-ai/langchainjs#10874) with the RAPI response shape. The + // response-side model must populate gen_ai.request.model (via the Azure + // workaround) AND gen_ai.response.model, even when LangChain only sets + // `model` (no `model_name` alias). + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "azure_openai", "AzureChatOpenAI"], + }, + extra: { metadata: { ls_model_name: "gpt-3.5-turbo" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "gpt-4o-mini-2024-07-18", + model_provider: "openai", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "gpt-4o-mini-2024-07-18", + ), + "AzureChatOpenAI request model should use response_metadata.model when model_name is absent", + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "gpt-4o-mini-2024-07-18", + ), + ); + }); }); describe("setResponseIdAttribute", () => {