From 2f20f3612e40917d8ad87fc1947e25012f618376 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Fri, 22 May 2026 13:50:39 -0700 Subject: [PATCH 1/3] fix(langchain): populate gen_ai.response.model for Responses API Read response_metadata.model (canonical OpenAI Responses API field) before falling back to response_metadata.model_name (LangChain's backwards-compat alias) on both v1 and v0 message paths. This makes gen_ai.response.model populate reliably when useResponsesApi: true, and keeps working if upstream LangChain ever drops the model_name alias. Fixes #128 Refs #150 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/genai/instrumentations/langchain/utils.ts | 22 ++- .../unit/genai/langchain/tracer.test.ts | 81 +++++++++ .../unit/genai/langchain/utils.test.ts | 160 ++++++++++++++++++ 3 files changed, 258 insertions(+), 5 deletions(-) diff --git a/src/genai/instrumentations/langchain/utils.ts b/src/genai/instrumentations/langchain/utils.ts index 11f0caa..9a92d96 100644 --- a/src/genai/instrumentations/langchain/utils.ts +++ b/src/genai/instrumentations/langchain/utils.ts @@ -461,12 +461,24 @@ export function getRequestModel(run: Run): string | undefined { // served the request). export function getResponseModel(run: Run): string | undefined { const llmOutput = run.outputs?.llmOutput as Record | undefined; + const v1Metadata = run.outputs?.generations?.[0]?.[0]?.message?.response_metadata as + | Record + | undefined; + const v0Metadata = run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.response_metadata as + | Record + | undefined; + return [ - // v1: response_metadata directly on message - run.outputs?.generations?.[0]?.[0]?.message?.response_metadata?.model_name, - // v0: response_metadata nested under kwargs - run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.response_metadata?.model_name, - // LLMResult.llmOutput.model_name (common for Chat models) + // v1: response_metadata directly on message. Prefer the canonical OpenAI + // Responses-API field (`model`) and fall back to the `model_name` alias + // LangChain keeps "for backwards compat with chat completion calls" (see + // langchain-ai/langchainjs libs/providers/langchain-openai/src/converters/responses.ts). + v1Metadata?.model, + v1Metadata?.model_name, + // v0: response_metadata nested under kwargs. + v0Metadata?.model, + v0Metadata?.model_name, + // LLMResult.llmOutput.* (common for Chat Completions API). llmOutput?.model_name, llmOutput?.model, ] diff --git a/test/internal/unit/genai/langchain/tracer.test.ts b/test/internal/unit/genai/langchain/tracer.test.ts index 25799a3..3848354 100644 --- a/test/internal/unit/genai/langchain/tracer.test.ts +++ b/test/internal/unit/genai/langchain/tracer.test.ts @@ -318,4 +318,85 @@ describe("LangChainTracer", () => { assert.strictEqual(startSpanCalls.length, 0, "should not create span when MAX_RUNS exceeded"); }); }); + + // End-to-end shape test for #128 scenario 3 / #150. Mirrors the Run object + // LangChain's `langchain-openai/converters/responses.ts` produces for a single + // ChatOpenAI invocation with `useResponsesApi: true`, and asserts the full + // bundle of GenAI semconv attributes lands on the span. + describe("Responses API (useResponsesApi: true) end-to-end", () => { + it("populates request/response model, response id, and token attrs on a RAPI run", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + + const runId = "rapi-run-1"; + const startedRun = makeRun({ + id: runId, + run_type: "llm", + name: "ChatOpenAI", + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { + metadata: { ls_model_name: "deployment-o4-mini", ls_provider: "openai" }, + invocation_params: { model: "deployment-o4-mini" }, + }, + inputs: { + messages: [[{ role: "user", content: "hello" }]], + }, + }); + await lct.onRunCreate(startedRun); + + const completedRun = { + ...startedRun, + outputs: { + generations: [ + [ + { + message: { + // Shape produced by libs/providers/langchain-openai/src/converters/responses.ts + response_metadata: { + model_provider: "openai", + model: "o4-mini-2025-04-16", + model_name: "o4-mini-2025-04-16", + id: "resp_rapi_abc", + created_at: 1_700_000_000, + }, + usage_metadata: { + input_tokens: 4, + output_tokens: 7, + }, + id: "resp_rapi_abc", + }, + }, + ], + ], + }, + } as unknown as Run; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(completedRun); + + const span = tracer.lastSpan!; + const calls = (span.setAttribute as ReturnType).mock.calls; + const got = (key: string) => calls.find((c: unknown[]) => c[0] === key)?.[1]; + + assert.strictEqual(got(ATTR_GEN_AI_OPERATION_NAME), "chat", "operation name"); + assert.strictEqual( + got("gen_ai.request.model"), + "deployment-o4-mini", + "request model should be the deployment alias", + ); + assert.strictEqual( + got("gen_ai.response.model"), + "o4-mini-2025-04-16", + "response model should come from response_metadata.model", + ); + assert.strictEqual( + got("gen_ai.response.id"), + "resp_rapi_abc", + "response id should come from response_metadata.id", + ); + assert.strictEqual(got("gen_ai.usage.input_tokens"), 4, "input tokens"); + assert.strictEqual(got("gen_ai.usage.output_tokens"), 7, "output tokens"); + assert.strictEqual(span.statusObj?.code, SpanStatusCode.OK); + }); + }); }); diff --git a/test/internal/unit/genai/langchain/utils.test.ts b/test/internal/unit/genai/langchain/utils.test.ts index 822dd32..28faa79 100644 --- a/test/internal/unit/genai/langchain/utils.test.ts +++ b/test/internal/unit/genai/langchain/utils.test.ts @@ -544,6 +544,166 @@ describe("setModelAttribute", () => { ), ); }); + + // Responses API (useResponsesApi: true) — LangChain's openai provider + // populates response_metadata.model (canonical) and, for backwards compat + // with chat completion calls, also response_metadata.model_name. We must + // honor both shapes so non-OpenAI RAPI providers (e.g. @langchain/perplexity) + // and any future major where the model_name alias is dropped keep working. + it("RAPI v1: extracts response model from response_metadata.model when only `model` is set", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "o4-mini-2025-04-16", + model_provider: "openai", + id: "resp_abc", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "deployment-o4-mini", + ), + "request model should come from invocation_params.model for non-Azure RAPI clients", + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + "response model should be sourced from response_metadata.model (RAPI canonical field)", + ); + }); + + it("RAPI v1: prefers response_metadata.model over response_metadata.model_name when both are present", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "o4-mini-2025-04-16", + // LangChain duplicates `model` into `model_name` "for + // backwards compat with chat completion calls". Both + // currently carry the same value, but we read the canonical + // field first to avoid coupling to that alias. + model_name: "o4-mini-2025-04-16", + model_provider: "openai", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + ); + }); + + it("RAPI v0: extracts response model from kwargs.response_metadata.model", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + kwargs: { + response_metadata: { + model: "o4-mini-2025-04-16", + model_provider: "openai", + }, + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "deployment-o4-mini", + ), + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + ); + }); + + it("AzureChatOpenAI + RAPI: response_metadata.model still drives the workaround request model", () => { + // Combines the AzureChatOpenAI ls_model_name=gpt-3.5-turbo regression (see + // langchain-ai/langchainjs#10874) with the RAPI response shape. The + // response-side model must populate gen_ai.request.model (via the Azure + // workaround) AND gen_ai.response.model, even when LangChain only sets + // `model` (no `model_name` alias). + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "azure_openai", "AzureChatOpenAI"], + }, + extra: { metadata: { ls_model_name: "gpt-3.5-turbo" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "gpt-4o-mini-2024-07-18", + model_provider: "openai", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "gpt-4o-mini-2024-07-18", + ), + "AzureChatOpenAI request model should use response_metadata.model when model_name is absent", + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "gpt-4o-mini-2024-07-18", + ), + ); + }); }); describe("setResponseIdAttribute", () => { From 2b0acc32cdbaefe5fe25f190ad32ba75852d1708 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Sat, 23 May 2026 21:46:42 -0700 Subject: [PATCH 2/3] test(langchain): address Copilot review feedback on RAPI tests - Pin distinct sentinel values for response_metadata.model_name (alias) and message.id so the assertions actually prove the canonical fields (response_metadata.model, response_metadata.id) win over the fallbacks, rather than passing tautologically when all sources carry the same value. - Switch the RAPI end-to-end test in tracer.test.ts to use ATTR_GEN_AI_* semconv constants instead of raw attribute string literals. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../unit/genai/langchain/tracer.test.ts | 47 +++++++++++-------- .../unit/genai/langchain/utils.test.ts | 21 ++++++--- 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/test/internal/unit/genai/langchain/tracer.test.ts b/test/internal/unit/genai/langchain/tracer.test.ts index 3848354..9d017a6 100644 --- a/test/internal/unit/genai/langchain/tracer.test.ts +++ b/test/internal/unit/genai/langchain/tracer.test.ts @@ -16,6 +16,11 @@ import { ATTR_ERROR_MESSAGE, ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_PROVIDER_NAME, + ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_RESPONSE_ID, + ATTR_GEN_AI_RESPONSE_MODEL, + ATTR_GEN_AI_USAGE_INPUT_TOKENS, + ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, } from "../../../../../src/genai/index.js"; function makeRun(overrides: Partial = {}): Run { @@ -353,20 +358,24 @@ describe("LangChainTracer", () => { [ { message: { - // Shape produced by libs/providers/langchain-openai/src/converters/responses.ts - response_metadata: { - model_provider: "openai", - model: "o4-mini-2025-04-16", - model_name: "o4-mini-2025-04-16", - id: "resp_rapi_abc", - created_at: 1_700_000_000, + // Shape produced by libs/providers/langchain-openai/src/converters/responses.ts. + // We pin distinct sentinels on the fields the test is supposed + // to ignore so the assertions actually prove that `model` + // (canonical) and `response_metadata.id` (provider-supplied) + // win over the `model_name` alias and `message.id`. + response_metadata: { + model_provider: "openai", + model: "o4-mini-2025-04-16", + model_name: "model_name-alias-should-be-ignored", + id: "resp_rapi_abc", + created_at: 1_700_000_000, }, - usage_metadata: { - input_tokens: 4, - output_tokens: 7, + usage_metadata: { + input_tokens: 4, + output_tokens: 7, + }, + id: "message-id-should-be-ignored", }, - id: "resp_rapi_abc", - }, }, ], ], @@ -380,22 +389,22 @@ describe("LangChainTracer", () => { assert.strictEqual(got(ATTR_GEN_AI_OPERATION_NAME), "chat", "operation name"); assert.strictEqual( - got("gen_ai.request.model"), + got(ATTR_GEN_AI_REQUEST_MODEL), "deployment-o4-mini", "request model should be the deployment alias", ); assert.strictEqual( - got("gen_ai.response.model"), + got(ATTR_GEN_AI_RESPONSE_MODEL), "o4-mini-2025-04-16", - "response model should come from response_metadata.model", + "response model should come from response_metadata.model (not the model_name alias)", ); assert.strictEqual( - got("gen_ai.response.id"), + got(ATTR_GEN_AI_RESPONSE_ID), "resp_rapi_abc", - "response id should come from response_metadata.id", + "response id should come from response_metadata.id (not message.id)", ); - assert.strictEqual(got("gen_ai.usage.input_tokens"), 4, "input tokens"); - assert.strictEqual(got("gen_ai.usage.output_tokens"), 7, "output tokens"); + assert.strictEqual(got(ATTR_GEN_AI_USAGE_INPUT_TOKENS), 4, "input tokens"); + assert.strictEqual(got(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS), 7, "output tokens"); assert.strictEqual(span.statusObj?.code, SpanStatusCode.OK); }); }); diff --git a/test/internal/unit/genai/langchain/utils.test.ts b/test/internal/unit/genai/langchain/utils.test.ts index 28faa79..46c794c 100644 --- a/test/internal/unit/genai/langchain/utils.test.ts +++ b/test/internal/unit/genai/langchain/utils.test.ts @@ -604,12 +604,13 @@ describe("setModelAttribute", () => { response_metadata: { model: "o4-mini-2025-04-16", // LangChain duplicates `model` into `model_name` "for - // backwards compat with chat completion calls". Both - // currently carry the same value, but we read the canonical - // field first to avoid coupling to that alias. - model_name: "o4-mini-2025-04-16", - model_provider: "openai", - }, + // backwards compat with chat completion calls". We pin a + // distinct sentinel here so the assertion proves we read + // the canonical `model` field first rather than coupling + // to the `model_name` alias. + model_name: "model_name-alias-should-be-ignored", + model_provider: "openai", + }, }, }, ], @@ -622,6 +623,14 @@ describe("setModelAttribute", () => { calls.some( (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", ), + "response model should come from response_metadata.model (canonical)", + ); + assert.ok( + !calls.some( + (c: unknown[]) => + c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "model_name-alias-should-be-ignored", + ), + "response model must not fall back to the model_name alias when model is set", ); }); From 4dd5703c9e518979288c5ae8aa97c2a7a156904d Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Sat, 23 May 2026 21:51:23 -0700 Subject: [PATCH 3/3] style: apply prettier formatting to RAPI tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../unit/genai/langchain/tracer.test.ts | 32 +++++++++---------- .../unit/genai/langchain/utils.test.ts | 14 ++++---- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/test/internal/unit/genai/langchain/tracer.test.ts b/test/internal/unit/genai/langchain/tracer.test.ts index 9d017a6..dec2f17 100644 --- a/test/internal/unit/genai/langchain/tracer.test.ts +++ b/test/internal/unit/genai/langchain/tracer.test.ts @@ -358,24 +358,24 @@ describe("LangChainTracer", () => { [ { message: { - // Shape produced by libs/providers/langchain-openai/src/converters/responses.ts. - // We pin distinct sentinels on the fields the test is supposed - // to ignore so the assertions actually prove that `model` - // (canonical) and `response_metadata.id` (provider-supplied) - // win over the `model_name` alias and `message.id`. - response_metadata: { - model_provider: "openai", - model: "o4-mini-2025-04-16", - model_name: "model_name-alias-should-be-ignored", - id: "resp_rapi_abc", - created_at: 1_700_000_000, + // Shape produced by libs/providers/langchain-openai/src/converters/responses.ts. + // We pin distinct sentinels on the fields the test is supposed + // to ignore so the assertions actually prove that `model` + // (canonical) and `response_metadata.id` (provider-supplied) + // win over the `model_name` alias and `message.id`. + response_metadata: { + model_provider: "openai", + model: "o4-mini-2025-04-16", + model_name: "model_name-alias-should-be-ignored", + id: "resp_rapi_abc", + created_at: 1_700_000_000, }, - usage_metadata: { - input_tokens: 4, - output_tokens: 7, - }, - id: "message-id-should-be-ignored", + usage_metadata: { + input_tokens: 4, + output_tokens: 7, }, + id: "message-id-should-be-ignored", + }, }, ], ], diff --git a/test/internal/unit/genai/langchain/utils.test.ts b/test/internal/unit/genai/langchain/utils.test.ts index 46c794c..237be2d 100644 --- a/test/internal/unit/genai/langchain/utils.test.ts +++ b/test/internal/unit/genai/langchain/utils.test.ts @@ -604,13 +604,13 @@ describe("setModelAttribute", () => { response_metadata: { model: "o4-mini-2025-04-16", // LangChain duplicates `model` into `model_name` "for - // backwards compat with chat completion calls". We pin a - // distinct sentinel here so the assertion proves we read - // the canonical `model` field first rather than coupling - // to the `model_name` alias. - model_name: "model_name-alias-should-be-ignored", - model_provider: "openai", - }, + // backwards compat with chat completion calls". We pin a + // distinct sentinel here so the assertion proves we read + // the canonical `model` field first rather than coupling + // to the `model_name` alias. + model_name: "model_name-alias-should-be-ignored", + model_provider: "openai", + }, }, }, ],