Feat/ci/automate integration tests

.github/workflows/integration-test.yml

@@ -0,0 +1,129 @@
+name: "Integration Tests"
+
+on:
+  workflow_dispatch:
+    inputs:
+      language:
+        description: 'Language to test (or all)'
+        required: true
+        type: choice
+        options:
+          - all
+          - nodejs
+          - python
+        default: all
+
+permissions:
+  contents: read
+
+jobs:
+  build-collector:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version-file: collector/go.mod
+      - name: Build Collector
+        run: make -C collector package GOARCH=amd64
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: opentelemetry-collector-layer-amd64.zip
+          path: collector/build/opentelemetry-collector-layer-amd64.zip
+  build-nodejs-layer:
+    if: inputs.language == 'all' || inputs.language == 'nodejs'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 18
+      - name: Build Node.js Layer
+        run: |
+          npm ci
+          npm run build
+        working-directory: nodejs
+      - name: Rename layer zip
+        run: mv layer.zip opentelemetry-nodejs-layer.zip
+        working-directory: nodejs/packages/layer/build
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: opentelemetry-nodejs-layer.zip
+          path: nodejs/packages/layer/build/opentelemetry-nodejs-layer.zip
+  build-python-layer:
+    if: inputs.language == 'all' || inputs.language == 'python'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Build Python Layer
+        run: ./build.sh
+        working-directory: python/src
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: opentelemetry-python-layer.zip
+          path: python/src/build/opentelemetry-python-layer.zip
+  prepare-languages:
+    runs-on: ubuntu-latest
+    outputs:
+      languages: ${{ steps.prepare-languages.outputs.languages }}
+    steps:
+      - id: prepare-languages
+        name: Prepare Languages
+        run: |
+          if [ ${{ inputs.language }} == 'all' ]; then
+            languages='["nodejs", "python"]'
+          else
+            languages='["${{ inputs.language }}"]'
+          fi
+          echo "languages=${languages}" >> $GITHUB_OUTPUT
+  test:
+    needs: [build-collector, build-nodejs-layer, build-python-layer, prepare-languages]
+    if: |
+      !cancelled() &&
+      needs.build-collector.result == 'success' &&
+      needs.build-nodejs-layer.result != 'failure' &&
+      needs.build-python-layer.result != 'failure'
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ${{ fromJson(needs.prepare-languages.outputs.languages) }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 22
+
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: opentelemetry-collector-layer-amd64.zip
+          path: artifacts/
+
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: opentelemetry-${{ matrix.language }}-layer.zip
+          path: artifacts/
+
+      - uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
+        with:
+          role-to-assume: ${{ secrets.OTEL_LAMBDA_INTEG_TEST_ROLE_ARN }}
+          role-duration-seconds: 1200
+          aws-region: us-east-1
+
+      - name: Install integration test dependencies
+        run: npm ci
+        working-directory: integration-tests
+
+      - name: Run integration tests
+        run: npx vitest run --config vitest.config.ts
+        working-directory: integration-tests
+        env:
+          TEST_LANGUAGE: ${{ matrix.language }}
+          COLLECTOR_LAYER_ZIP: ${{ github.workspace }}/artifacts/opentelemetry-collector-layer-amd64.zip
+          INSTRUMENTATION_LAYER_ZIP: ${{ github.workspace }}/artifacts/opentelemetry-${{ matrix.language }}-layer.zip
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}

.gitignore

@@ -20,3 +20,10 @@ build.toml
 *.zip
 
 collector/VERSION
+
+# Riptide artifacts (cloud-synced)
+.humanlayer/tasks/
+
+# Integration tests
+integration-tests/cdk.out/
+integration-tests/node_modules/

integration-tests/README.md

@@ -0,0 +1,4 @@
+# Integration Tests
+
+This test suite contains a simple setup to deploy lambda functions using the otel layers. These functions then use the aws-sdk library provided in the lamba runtime to make an sts call. We evaluate whether the expected telemetry was generated for this aws-sdk call.
+The setup is very basic, it serves more as a smoke check than an "all-covering" test suite.

integration-tests/cdk/stack.ts

@@ -0,0 +1,52 @@
+import * as lambda from "aws-cdk-lib/aws-lambda";
+import type { Construct } from "constructs";
+import { CfnOutput, CliCredentialsStackSynthesizer, Duration, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib";
+import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs";
+
+export interface IntegrationTestStackProps extends StackProps {
+  runtime: lambda.Runtime;
+  handler: string;
+  handlerCodePath: string;
+  collectorLayerZipPath: string;
+  instrumentationLayerZipPath: string;
+}
+
+export class IntegrationTestStack extends Stack {
+  constructor(scope: Construct, id: string, props: IntegrationTestStackProps) {
+    super(scope, id, {
+      ...props,
+      synthesizer: new CliCredentialsStackSynthesizer(),
+    });
+
+    const collectorLayer = new lambda.LayerVersion(this, "CollectorLayer", {
+      layerVersionName: `${this.stackName}-CollectorLayer`,
+      code: lambda.Code.fromAsset(props.collectorLayerZipPath),
+      compatibleArchitectures: [lambda.Architecture.X86_64],
+    });
+
+    const instrumentationLayer = new lambda.LayerVersion(this, "InstrumentationLayer", {
+      layerVersionName: `${this.stackName}-InstrumentationLayer`,
+      code: lambda.Code.fromAsset(props.instrumentationLayerZipPath),
+      compatibleArchitectures: [lambda.Architecture.X86_64],
+    });
+
+    const lambdaFunction = new lambda.Function(this, "TestFunction", {
+      runtime: props.runtime,
+      handler: props.handler,
+      code: lambda.Code.fromAsset(props.handlerCodePath),
+      layers: [collectorLayer, instrumentationLayer],
+      environment: {
+        AWS_LAMBDA_EXEC_WRAPPER: "/opt/otel-handler",
+      },
+      logGroup: new LogGroup(this, "FunctionLogGroup", {
+        retention: RetentionDays.ONE_DAY,
+        removalPolicy: RemovalPolicy.DESTROY,
+      }),
+      timeout: Duration.seconds(30),
+      memorySize: 512,
+    });
+
+    new CfnOutput(this, "FunctionName", { value: lambdaFunction.functionName });
+    new CfnOutput(this, "LogGroupName", { value: lambdaFunction.logGroup.logGroupName });
+  }
+}

integration-tests/globalSetup.ts

@@ -0,0 +1,119 @@
+import { resolve } from "node:path";
+import {
+  Toolkit,
+  NonInteractiveIoHost,
+  StackSelectionStrategy,
+} from "@aws-cdk/toolkit-lib";
+import { Runtime } from "aws-cdk-lib/aws-lambda";
+import type { TestProject } from "vitest/node";
+import { IntegrationTestStack } from "./cdk/stack.js";
+import { App, Tags } from "aws-cdk-lib";
+
+declare module "vitest" {
+  export interface ProvidedContext {
+    functionName: string;
+    logGroupName: string;
+  }
+}
+
+type LanguageConfig = {
+  runtime: Runtime;
+  handler: string;
+  handlerDir: string;
+};
+
+const LANGUAGE_CONFIG = {
+  nodejs: {
+    runtime: Runtime.NODEJS_24_X,
+    handler: "index.handler",
+    handlerDir: "handlers/nodejs",
+  },
+  python: {
+    runtime: Runtime.PYTHON_3_14,
+    handler: "lambda_function.lambda_handler",
+    handlerDir: "handlers/python",
+  },
+} satisfies Record<string, LanguageConfig>;
+
+type SupportedLanguage = keyof typeof LANGUAGE_CONFIG;
+
+function isSupportedLanguage(language: string): language is SupportedLanguage {
+  return language in LANGUAGE_CONFIG;
+}
+
+export async function setup({ provide }: TestProject) {
+  const language = process.env.TEST_LANGUAGE;
+  const collectorZip = process.env.COLLECTOR_LAYER_ZIP;
+  const instrumentationZip = process.env.INSTRUMENTATION_LAYER_ZIP;
+
+  if (!language || !collectorZip || !instrumentationZip) {
+    throw new Error(
+      "Required env vars: TEST_LANGUAGE, COLLECTOR_LAYER_ZIP, INSTRUMENTATION_LAYER_ZIP",
+    );
+  }
+
+  if (!isSupportedLanguage(language)) {
+    throw new Error(
+      `Unsupported language: ${language}`
+    )
+  }
+  const config = LANGUAGE_CONFIG[language];
+
+  const runId = process.env.GITHUB_RUN_ID;
+  const runAttempt = process.env.GITHUB_RUN_ATTEMPT;
+  const stackName = runId
+    ? `IntegrationTest-${language}-${runId}-${runAttempt}`
+    : `IntegrationTest-${language}`;
+
+  const toolkit = new Toolkit({
+    ioHost: new NonInteractiveIoHost(),
+  });
+
+  const source = await toolkit.fromAssemblyBuilder(async (props) => {
+    const app = new App({ outdir: props.outdir, context: props.context });
+
+    Tags.of(app).add("Purpose", "integration-test");
+    Tags.of(app).add("Language", language);
+    if (runId) {
+      Tags.of(app).add("GitHubRunId", runId);
+      Tags.of(app).add("GitHubRunAttempt", runAttempt ?? "1");
+    }
+
+    new IntegrationTestStack(app, stackName, {
+      runtime: config.runtime,
+      handler: config.handler,
+      handlerCodePath: resolve(config.handlerDir),
+      collectorLayerZipPath: resolve(collectorZip),
+      instrumentationLayerZipPath: resolve(instrumentationZip),
+    });
+    return app.synth();
+  });
+
+  const result = await toolkit.deploy(source, {
+    stacks: {
+      strategy: StackSelectionStrategy.ALL_STACKS,
+    },
+  });
+
+  const stack = result.stacks[0];
+  if (!stack) {
+    throw new Error(`Deploy of ${stackName} returned no stacks`);
+  }
+  const { FunctionName, LogGroupName } = stack.outputs;
+  if (!FunctionName || !LogGroupName) {
+    throw new Error(
+      `Stack ${stackName} missing required outputs (got: ${Object.keys(stack.outputs).join(", ")})`,
+    );
+  }
+
+  provide("functionName", FunctionName);
+  provide("logGroupName", LogGroupName);
+
+  return async () => {
+    await toolkit.destroy(source, {
+      stacks: {
+        strategy: StackSelectionStrategy.ALL_STACKS,
+      },
+    });
+  };
+}

integration-tests/handlers/nodejs/index.mjs

@@ -0,0 +1,11 @@
+import { STSClient, GetCallerIdentityCommand } from '@aws-sdk/client-sts';
+
+const sts = new STSClient({});
+
+export const handler = async () => {
+  const identity = await sts.send(new GetCallerIdentityCommand({}));
+  return {
+    statusCode: 200,
+    body: JSON.stringify({ status: 'ok', account: identity.Account }),
+  };
+};

integration-tests/handlers/python/lambda_function.py

@@ -0,0 +1,12 @@
+import json
+import boto3
+
+sts = boto3.client("sts")
+
+
+def lambda_handler(event, context):
+    identity = sts.get_caller_identity()
+    return {
+        "statusCode": 200,
+        "body": json.dumps({"status": "ok", "account": identity["Account"]}),
+    }

integration-tests/helpers/cloudwatch.ts

@@ -0,0 +1,48 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import {
+  CloudWatchLogsClient,
+  FilterLogEventsCommand,
+  type FilteredLogEvent,
+} from '@aws-sdk/client-cloudwatch-logs';
+
+const cwl = new CloudWatchLogsClient({});
+
+interface CwlOptions {
+  logGroupName: string;
+  filterPattern: string;
+  startTime: number;
+  timeoutMs?: number;
+  pollIntervalMs?: number;
+}
+
+export async function waitForSpans(options: CwlOptions): Promise<FilteredLogEvent[]> {
+  const {
+    logGroupName,
+    filterPattern,
+    startTime,
+    timeoutMs = 60_000,
+    pollIntervalMs = 2_000,
+  } = options;
+
+  const deadline = Date.now() + timeoutMs;
+
+  while (Date.now() < deadline) {
+    const response = await cwl.send(
+      new FilterLogEventsCommand({
+        logGroupName,
+        filterPattern,
+        startTime,
+      }),
+    );
+
+    if (response.events && response.events.length > 0) {
+      return response.events;
+    }
+
+    await sleep(pollIntervalMs);
+  }
+
+  throw new Error(
+    `Timed out waiting for spans matching "${filterPattern}" in ${logGroupName} after ${timeoutMs}ms`,
+  );
+}