Extend GetObjectAttributes and ListObjectsV2 with user metadata support

.github/docker.env

@@ -3,7 +3,7 @@ MONGODB_IMAGE=mongo:4.4
 MONGODB_PORT=27018
 
 # Cloudserver
-CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.0-preview.1
+CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.4
 CLOUDSERVER_PORT=8000
 
 # Metadata

examples/s3extended.ts → examples/s3Extended.ts

@@ -1,6 +1,6 @@
 
 import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
-import { ListObjectsV2ExtendedCommand } from '@scality/cloudserverclient/clients/s3Extended';
+import { ListObjectsV2ExtendedCommand } from '@scality/cloudserverclient/commands/s3Extended';
 
 const config: S3ClientConfig = {
     endpoint: 'http://localhost:8000',

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scality/cloudserverclient",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "engines": {
     "node": ">=20"
   },

src/commands/s3Extended/getObjectAttributes.ts

@@ -0,0 +1,45 @@
+import {
+    GetObjectAttributesCommand,
+    GetObjectAttributesCommandInput,
+    GetObjectAttributesCommandOutput,
+    ObjectAttributes,
+} from '@aws-sdk/client-s3';
+import {
+    overrideObjectAttributesHeaderMiddleware,
+    captureResponseBodyMiddleware,
+    parseUserMetadataMiddleware,
+} from './utils';
+
+export interface GetObjectAttributesExtendedInput extends Omit<GetObjectAttributesCommandInput, 'ObjectAttributes'> {
+    ObjectAttributes: (ObjectAttributes | `x-amz-meta-${string}`)[];
+}
+
+export interface GetObjectAttributesExtendedOutput extends GetObjectAttributesCommandOutput {
+    [key: `x-amz-meta-${string}`]: string;
+}
+
+export class GetObjectAttributesExtendedCommand extends GetObjectAttributesCommand {
+    constructor(input: GetObjectAttributesExtendedInput) {
+        super(input as GetObjectAttributesCommandInput);
+
+        const captured = { xml: '' };
+
+        this.middlewareStack.add(
+            overrideObjectAttributesHeaderMiddleware('x-amz-object-attributes', input.ObjectAttributes), {
+                step: 'build',
+                name: 'overrideObjectAttributesHeader',
+            });
+
+        this.middlewareStack.add(captureResponseBodyMiddleware(captured), {
+            step: 'deserialize',
+            name: 'captureResponseBody',
+            priority: 'low', // runs before SDK deserializer
+        });
+
+        this.middlewareStack.add(parseUserMetadataMiddleware(captured), {
+            step: 'deserialize',
+            name: 'parseUserMetadata',
+            priority: 'high', // runs after SDK deserializer
+        });
+    }
+}

src/commands/s3Extended/index.ts

@@ -0,0 +1,4 @@
+export * from './listObjects';
+export * from './listObjectsV2';
+export * from './listObjectVersions';
+export * from './getObjectAttributes';

src/commands/s3Extended/listObjectVersions.ts

@@ -0,0 +1,17 @@
+import { ListObjectVersionsCommand, ListObjectVersionsCommandInput } from '@aws-sdk/client-s3';
+import { extendCommandWithExtraParametersMiddleware } from './utils';
+
+export interface ListObjectVersionsExtendedInput extends ListObjectVersionsCommandInput {
+    Query: string;
+}
+
+export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand {
+    constructor(input: ListObjectVersionsExtendedInput) {
+        super(input);
+
+        this.middlewareStack.add(
+            extendCommandWithExtraParametersMiddleware(input.Query),
+            { step: 'build', name: 'extendCommandWithExtraParameters' }
+        );
+    }
+}

src/commands/s3Extended/listObjects.ts

@@ -0,0 +1,17 @@
+import { ListObjectsCommand, ListObjectsCommandInput } from '@aws-sdk/client-s3';
+import { extendCommandWithExtraParametersMiddleware } from './utils';
+
+export interface ListObjectsExtendedInput extends ListObjectsCommandInput {
+    Query: string;
+}
+
+export class ListObjectsExtendedCommand extends ListObjectsCommand {
+    constructor(input: ListObjectsExtendedInput) {
+        super(input);
+
+        this.middlewareStack.add(
+            extendCommandWithExtraParametersMiddleware(input.Query),
+            { step: 'build', name: 'extendCommandWithExtraParameters' }
+        );
+    }
+}

src/commands/s3Extended/listObjectsV2.ts

@@ -0,0 +1,60 @@
+import {
+    ListObjectsV2Command,
+    ListObjectsV2CommandInput,
+    ListObjectsV2CommandOutput,
+    OptionalObjectAttributes,
+    _Object,
+} from '@aws-sdk/client-s3';
+import {
+    extendCommandWithExtraParametersMiddleware,
+    overrideObjectAttributesHeaderMiddleware,
+    captureResponseBodyMiddleware,
+    parseListObjectsUserMetadataMiddleware,
+} from './utils';
+
+export interface ListObjectsV2ExtendedInput extends ListObjectsV2CommandInput {
+    Query?: string;
+    ObjectAttributes?: (OptionalObjectAttributes | `x-amz-meta-${string}`)[];
+}
+
+export interface ListObjectsV2ExtendedContentEntry extends _Object {
+    [key: `x-amz-meta-${string}`]: string;
+}
+
+export interface ListObjectsV2ExtendedOutput extends ListObjectsV2CommandOutput {
+    Contents?: ListObjectsV2ExtendedContentEntry[];
+}
+
+export class ListObjectsV2ExtendedCommand extends ListObjectsV2Command {
+    constructor(input: ListObjectsV2ExtendedInput) {
+        super(input);
+
+        this.middlewareStack.add(
+            extendCommandWithExtraParametersMiddleware(input.Query),
+            { step: 'build', name: 'extendCommandWithExtraParameters' }
+        );
+
+        if (input.ObjectAttributes?.length) {
+            const captured = { xml: '' };
+
+            this.middlewareStack.add(
+                overrideObjectAttributesHeaderMiddleware(
+                    'x-amz-optional-object-attributes', input.ObjectAttributes), {
+                    step: 'build',
+                    name: 'overrideObjectAttributesHeader',
+                });
+
+            this.middlewareStack.add(captureResponseBodyMiddleware(captured), {
+                step: 'deserialize',
+                name: 'captureResponseBody',
+                priority: 'low',
+            });
+
+            this.middlewareStack.add(parseListObjectsUserMetadataMiddleware(captured), {
+                step: 'deserialize',
+                name: 'parseUserMetadata',
+                priority: 'high',
+            });
+        }
+    }
+}

src/commands/s3Extended/utils.ts

@@ -0,0 +1,87 @@
+import { Readable } from 'stream';
+import { streamCollector } from '@smithy/node-http-handler';
+import { XMLParser } from 'fast-xml-parser';
+
+export const USER_METADATA_PREFIX = 'x-amz-meta-';
+
+
+export function extendCommandWithExtraParametersMiddleware(query: string) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return (next: any) => async (args: any) => {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const request = args.request as any;
+        if (request.query) {
+            request.query.search = query;
+        } else {
+            request.query = { search: query };
+        }
+        return next(args);
+    };
+}
+
+
+export function overrideObjectAttributesHeaderMiddleware(headerName: string, attributes: string[]) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return (next: any) => async (args: any) => {
+        const request = args.request;
+        request.headers[headerName] = attributes.join(',');
+        return next(args);
+    };
+}
+
+export function captureResponseBodyMiddleware(captured: { xml: string }) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return (next: any) => async (args: any) => {
+        const { response } = await next(args);
+
+        if (response?.body) {
+            const collected = await streamCollector(response.body);
+            const buffer = Buffer.from(collected);
+            // eslint-disable-next-line no-param-reassign
+            captured.xml = buffer.toString('utf-8');
+            // Re-create the body stream so the SDK deserializer can still consume it
+            response.body = Readable.from([buffer]);
+        }
+
+        return { response };
+    };
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function extractUserMetadata(source: Record<string, any>, target: Record<string, string>): void {
+    for (const [key, value] of Object.entries(source)) {
+        if (key.startsWith(USER_METADATA_PREFIX)) {
+            // eslint-disable-next-line no-param-reassign
+            target[key] = String(value);
+        }
+    }
+}
+
+export function parseUserMetadataMiddleware(captured: { xml: string }) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return (next: any) => async (args: any) => {
+        const result = await next(args);
+        const parsed = new XMLParser().parse(captured.xml);
+        const response = parsed?.GetObjectAttributesResponse;
+        if (response) {
+            extractUserMetadata(response, result.output);
+        }
+        return result;
+    };
+}
+
+
+export function parseListObjectsUserMetadataMiddleware(captured: { xml: string }) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return (next: any) => async (args: any) => {
+        const result = await next(args);
+        const parsed = new XMLParser().parse(captured.xml);
+        const xmlContents = parsed?.ListBucketResult?.Contents;
+        if (result.output.Contents && xmlContents) {
+            for (let i = 0; i < result.output.Contents.length; i++) {
+                extractUserMetadata(xmlContents[i], result.output.Contents[i]);
+            }
+        }
+        return result;
+    };
+}

src/index.ts

@@ -1,6 +1,6 @@
 export * from './clients/backbeatRoutes';
 export * from './clients/bucketQuota';
 export * from './clients/proxyBackbeatApis';
-export * from './clients/s3Extended';
+export * from './commands/s3Extended';
 export { CloudserverClient, CloudserverClientConfig } from './clients/cloudserver';
 export * from './utils';