From 1bba0e7121faf2e7c5eec44bccd3663d9ac25e3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Wed, 11 Mar 2026 16:02:08 +0100 Subject: [PATCH 1/5] Extend basic command for GetObjectAttributes Issue: CLDSRVCLT-10 --- src/clients/s3Extended.ts | 98 +++++++- tests/testS3ExtendedApis.test.ts | 415 +++++++++++++++++++++++-------- 2 files changed, 398 insertions(+), 115 deletions(-) diff --git a/src/clients/s3Extended.ts b/src/clients/s3Extended.ts index c3555a6e..1e0e2660 100644 --- a/src/clients/s3Extended.ts +++ b/src/clients/s3Extended.ts @@ -1,13 +1,20 @@ -import { +import { Readable } from 'stream'; +import { + GetObjectAttributesCommand, + GetObjectAttributesCommandInput, + GetObjectAttributesCommandOutput, ListObjectsCommand, ListObjectsCommandInput, - ListObjectsV2Command, + ListObjectsV2Command, ListObjectsV2CommandInput, ListObjectVersionsCommand, - ListObjectVersionsCommandInput + ListObjectVersionsCommandInput, + ObjectAttributes } from '@aws-sdk/client-s3'; +import { streamCollector } from '@smithy/node-http-handler'; +import { XMLParser } from 'fast-xml-parser'; -const extendCommandWithExtraParametersMiddleware = (query: string) => +const extendCommandWithExtraParametersMiddleware = (query: string) => (next: any) => async (args: any) => { const request = args.request as any; if (request.query) { @@ -55,10 +62,91 @@ export interface ListObjectVersionsExtendedInput extends ListObjectVersionsComma export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand { constructor(input: ListObjectVersionsExtendedInput) { super(input); - + this.middlewareStack.add( extendCommandWithExtraParametersMiddleware(input.Query), { step: 'build', name: 'extendCommandWithExtraParameters' } ); } } + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const overrideObjectAttributesHeaderMiddleware = (attributes: string[]) => (next: any) => async (args: any) => { + const request = args.request; + request.headers['x-amz-object-attributes'] = attributes.join(','); + return next(args); +}; + +const USER_METADATA_PREFIX = 'x-amz-meta-'; + +function parseUserMetadataFromXml(xml: string): Record { + const parsed = new XMLParser().parse(xml); + const response = parsed?.GetObjectAttributesResponse; + if (!response) { + return {}; + } + + const metadata: Record = {}; + for (const [key, value] of Object.entries(response)) { + if (key.startsWith(USER_METADATA_PREFIX)) { + metadata[key] = String(value); + } + } + + return metadata; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const captureResponseBodyMiddleware = (captured: { xml: string }) => (next: any) => async (args: any) => { + const { response } = await next(args); + + if (response?.body) { + const collected = await streamCollector(response.body); + const buffer = Buffer.from(collected); + // eslint-disable-next-line no-param-reassign + captured.xml = buffer.toString('utf-8'); + response.body = Readable.from([buffer]); + } + + return { response }; +}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const parseUserMetadataMiddleware = (captured: { xml: string }) => (next: any) => async (args: any) => { + const result = await next(args); + Object.assign(result.output, parseUserMetadataFromXml(captured.xml)); + return result; +}; + +export interface GetObjectAttributesExtendedInput extends Omit { + ObjectAttributes: (ObjectAttributes | `x-amz-meta-${string}`)[]; +} + +export interface GetObjectAttributesExtendedOutput extends GetObjectAttributesCommandOutput { + [key: `x-amz-meta-${string}`]: string; +} + +export class GetObjectAttributesExtendedCommand extends GetObjectAttributesCommand { + constructor(input: GetObjectAttributesExtendedInput) { + super(input as GetObjectAttributesCommandInput); + + const captured = { xml: '' }; + + this.middlewareStack.add(overrideObjectAttributesHeaderMiddleware(input.ObjectAttributes), { + step: 'build', + name: 'overrideObjectAttributesHeader', + }); + + this.middlewareStack.add(captureResponseBodyMiddleware(captured), { + step: 'deserialize', + name: 'captureResponseBody', + priority: 'low', // runs before SDK deserializer + }); + + this.middlewareStack.add(parseUserMetadataMiddleware(captured), { + step: 'deserialize', + name: 'parseUserMetadata', + priority: 'high', // runs after SDK deserializer + }); + } +} diff --git a/tests/testS3ExtendedApis.test.ts b/tests/testS3ExtendedApis.test.ts index 0e43b52e..14ebe0d1 100644 --- a/tests/testS3ExtendedApis.test.ts +++ b/tests/testS3ExtendedApis.test.ts @@ -1,8 +1,10 @@ -import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; +import { S3Client, PutObjectCommand, DeleteObjectCommand, GetObjectAttributesCommandOutput } from '@aws-sdk/client-s3'; import { createTestClient, testConfig } from './testSetup'; import { describeForMongoBackend } from './testHelpers'; import assert from 'assert'; import { + GetObjectAttributesExtendedCommand, + GetObjectAttributesExtendedOutput, ListObjectsExtendedCommand, ListObjectsV2ExtendedCommand, ListObjectVersionsExtendedCommand, @@ -16,120 +18,313 @@ describeForMongoBackend('S3 Extended API Tests', () => { beforeAll(async () => { const testClients = createTestClient(); s3client = testClients.s3client; + }); - const putObjectCommand = new PutObjectCommand({ - Bucket: testConfig.bucketName, - Key: key2ndObject, - Body: body2ndObject, + describe('ListObjects', () => { + beforeAll(async () => { + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); }); - await s3client.send(putObjectCommand); - }); - it('should test ListObjectsExtended', async () => { - const getCommand1= new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: 5 - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Contents?.length, 2); - - const maxKey = 1; - const getCommand2= new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: maxKey - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.Contents?.length, maxKey); - assert.strictEqual(getData2.IsTruncated, true); - - const getCommand3 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length > ${testConfig.objectData.length}`, - MaxKeys: 5 - }); - const getData3 = await s3client.send(getCommand3); - assert.strictEqual(getData3.Contents?.length, 1); - assert.strictEqual(getData3.Contents[0].Key, key2ndObject); - - const getCommand4 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `key = ${key2ndObject}`, - MaxKeys: 5 - }); - const getData4 = await s3client.send(getCommand4); - assert.strictEqual(getData4.Contents?.length, 1); - assert.strictEqual(getData4.Contents[0].Key, key2ndObject); - - const getCommand5 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `key = iDontExists`, - MaxKeys: 5 - }); - const getData5 = await s3client.send(getCommand5); - assert.strictEqual(getData5.Contents, undefined); - }); + it('should test ListObjectsExtended', async () => { + const getCommand1= new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + + const maxKey = 1; + const getCommand2= new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: maxKey + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, maxKey); + assert.strictEqual(getData2.IsTruncated, true); + + const getCommand3 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length > ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData3 = await s3client.send(getCommand3); + assert.strictEqual(getData3.Contents?.length, 1); + assert.strictEqual(getData3.Contents[0].Key, key2ndObject); + + const getCommand4 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `key = ${key2ndObject}`, + MaxKeys: 5 + }); + const getData4 = await s3client.send(getCommand4); + assert.strictEqual(getData4.Contents?.length, 1); + assert.strictEqual(getData4.Contents[0].Key, key2ndObject); + + const getCommand5 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'key = iDontExists', + MaxKeys: 5 + }); + const getData5 = await s3client.send(getCommand5); + assert.strictEqual(getData5.Contents, undefined); + }); + + it('should test ListObjectsV2Extended', async () => { + const getCommand1= new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5, + FetchOwner: true, + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + assert.strictEqual(getData1.KeyCount, 2); + assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart'); + + const getCommand2 = new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 5, + StartAfter: testConfig.objectKey, // Skip first object + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, 1); + assert.strictEqual(getData2.Contents[0].Key, key2ndObject); + }); + + it('should test ListObjectVersionsExtended', async () => { + const getCommand1 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 100, + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Versions?.length, 2); + assert.strictEqual(getData1.Versions[0].IsLatest, true); + + // Delete one object to create a DeleteMarker + const deleteCommand = new DeleteObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + }); + await s3client.send(deleteCommand); + + const getCommand2 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 100, + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.DeleteMarkers?.[0].Key, key2ndObject); + assert.ok(getData2.DeleteMarkers?.[0].VersionId); - it('should test ListObjectsV2Extended', async () => { - const getCommand1= new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: 5, - FetchOwner: true, - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Contents?.length, 2); - assert.strictEqual(getData1.KeyCount, 2); - assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart'); - - const getCommand2 = new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= 0`, - MaxKeys: 5, - StartAfter: testConfig.objectKey, // Skip first object - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.Contents?.length, 1); - assert.strictEqual(getData2.Contents[0].Key, key2ndObject); + assert.ok(getData2.Versions); + const firstVersion = getData2.Versions[0]; + const getCommand3 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + KeyMarker: firstVersion.Key, + VersionIdMarker: firstVersion.VersionId, + MaxKeys: 100, + }); + const getData3 = await s3client.send(getCommand3); + assert.notStrictEqual(getData3.Versions?.[0]?.Key, firstVersion.Key); + }); }); - it('should test ListObjectVersionsExtended', async () => { - const getCommand1 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= 0`, - MaxKeys: 100, - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Versions?.length, 2); - assert.strictEqual(getData1.Versions[0].IsLatest, true); - - // Delete one object to create a DeleteMarker - const deleteCommand = new DeleteObjectCommand({ - Bucket: testConfig.bucketName, - Key: key2ndObject, - }); - await s3client.send(deleteCommand); - - const getCommand2 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= 0`, - MaxKeys: 100, - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.DeleteMarkers?.[0].Key, key2ndObject); - assert.ok(getData2.DeleteMarkers?.[0].VersionId); - - assert.ok(getData2.Versions); - const firstVersion = getData2.Versions[0]; - const getCommand3 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= 0`, - KeyMarker: firstVersion.Key, - VersionIdMarker: firstVersion.VersionId, - MaxKeys: 100, - }); - const getData3 = await s3client.send(getCommand3); - assert.notStrictEqual(getData3.Versions?.[0]?.Key, firstVersion.Key); + describe('GetObjectAttributes', () => { + const metadataKey = `${testConfig.objectKey}-with-meta`; + const metadataBody = 'data-with-metadata'; + const metadataETag = '0235b027419caf7c0e4b0840f7ec21a6'; + + beforeAll(async () => { + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + Body: metadataBody, + Metadata: { + foo: 'bar', + bar: 'baz', + baz: 'qux', + }, + })); + }); + + it('should get ETag only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + }); + + it('should get ObjectSize only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ObjectSize'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ObjectSize, 11); + }); + + it('should get StorageClass only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['StorageClass'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.StorageClass, 'STANDARD'); + }); + + it('should get all standard attributes at once', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag', 'ObjectParts', 'StorageClass', 'ObjectSize'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + assert.strictEqual(result.ObjectSize, 11); + assert.strictEqual(result.StorageClass, 'STANDARD'); + }); + + it('should get a single user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + }); + + it('should get multiple user metadata keys', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-bar', 'x-amz-meta-baz'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get all user metadata with wildcard', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-*'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get wildcard combined with a specific user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-*', 'x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should handle non-existing user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['x-amz-meta-nonexistent'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-nonexistent'], undefined); + }); + + it('should get standard attribute combined with non-existing user metadata', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag', 'x-amz-meta-nonexistent'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + assert.strictEqual(result['x-amz-meta-nonexistent'], undefined); + }); + + it('should get ETag combined with a single user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['ETag', 'x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + }); + + it('should get all standard attributes combined with all user metadata', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: [ + 'ETag', + 'ObjectParts', + 'StorageClass', + 'ObjectSize', + 'x-amz-meta-foo', + 'x-amz-meta-bar', + 'x-amz-meta-baz', + ], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result.ObjectSize, 18); + assert.strictEqual(result.StorageClass, 'STANDARD'); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get standard attributes combined with user metadata wildcard', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['ETag', 'ObjectSize', 'x-amz-meta-*'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result.ObjectSize, 18); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); }); }); From d46894cf9a5d42a0e936b0554e276663db41e1fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Wed, 11 Mar 2026 17:35:27 +0100 Subject: [PATCH 2/5] Extend basic command for ListObjectV2 Issue: CLDSRVCLT-10 --- src/clients/s3Extended.ts | 95 +++++++++++++++++++++++---- tests/testS3ExtendedApis.test.ts | 109 ++++++++++++++++++++++++++++++- 2 files changed, 189 insertions(+), 15 deletions(-) diff --git a/src/clients/s3Extended.ts b/src/clients/s3Extended.ts index 1e0e2660..d9c8e988 100644 --- a/src/clients/s3Extended.ts +++ b/src/clients/s3Extended.ts @@ -7,9 +7,12 @@ import { ListObjectsCommandInput, ListObjectsV2Command, ListObjectsV2CommandInput, + ListObjectsV2CommandOutput, + _Object, ListObjectVersionsCommand, ListObjectVersionsCommandInput, - ObjectAttributes + ObjectAttributes, + OptionalObjectAttributes } from '@aws-sdk/client-s3'; import { streamCollector } from '@smithy/node-http-handler'; import { XMLParser } from 'fast-xml-parser'; @@ -41,17 +44,47 @@ export class ListObjectsExtendedCommand extends ListObjectsCommand { } export interface ListObjectsV2ExtendedInput extends ListObjectsV2CommandInput { - Query: string; + Query?: string; + ObjectAttributes?: (OptionalObjectAttributes | `x-amz-meta-${string}`)[]; +} + +export interface ListObjectsV2ExtendedContentEntry extends _Object { + [key: `x-amz-meta-${string}`]: string; +} + +export interface ListObjectsV2ExtendedOutput extends ListObjectsV2CommandOutput { + Contents?: ListObjectsV2ExtendedContentEntry[]; } export class ListObjectsV2ExtendedCommand extends ListObjectsV2Command { constructor(input: ListObjectsV2ExtendedInput) { super(input); - + this.middlewareStack.add( extendCommandWithExtraParametersMiddleware(input.Query), { step: 'build', name: 'extendCommandWithExtraParameters' } ); + + if (input.ObjectAttributes?.length) { + const captured = { xml: '' }; + + this.middlewareStack.add(overrideObjectAttributesHeaderMiddleware('x-amz-optional-object-attributes', input.ObjectAttributes), { + step: 'build', + name: 'overrideObjectAttributesHeader', + }); + + this.middlewareStack.add(captureResponseBodyMiddleware(captured), { + step: 'deserialize', + name: 'captureResponseBody', + priority: 'low', + }); + + this.middlewareStack.add(parseListObjectsUserMetadataMiddleware(captured), { + step: 'deserialize', + name: 'parseUserMetadata', + priority: 'high', + }); + } } } @@ -71,29 +104,48 @@ export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand } // eslint-disable-next-line @typescript-eslint/no-explicit-any -const overrideObjectAttributesHeaderMiddleware = (attributes: string[]) => (next: any) => async (args: any) => { +const overrideObjectAttributesHeaderMiddleware = (headerName: string, attributes: string[]) => (next: any) => async (args: any) => { const request = args.request; - request.headers['x-amz-object-attributes'] = attributes.join(','); + request.headers[headerName] = attributes.join(','); return next(args); }; const USER_METADATA_PREFIX = 'x-amz-meta-'; -function parseUserMetadataFromXml(xml: string): Record { +function extractUserMetadata(obj: Record): Record { + const metadata: Record = {}; + for (const [key, value] of Object.entries(obj)) { + if (key.startsWith(USER_METADATA_PREFIX)) { + metadata[key] = String(value); + } + } + return metadata; +} + +function parseGetObjectAttributesUserMetadata(xml: string): Record { const parsed = new XMLParser().parse(xml); const response = parsed?.GetObjectAttributesResponse; if (!response) { return {}; } + return extractUserMetadata(response); +} - const metadata: Record = {}; - for (const [key, value] of Object.entries(response)) { - if (key.startsWith(USER_METADATA_PREFIX)) { - metadata[key] = String(value); +function parseListObjectsUserMetadata(xml: string): Map> { + const parsed = new XMLParser().parse(xml); + const result = parsed?.ListBucketResult; + if (!result?.Contents) { + return new Map(); + } + const contents = Array.isArray(result.Contents) ? result.Contents : [result.Contents]; + const metadataByKey = new Map>(); + for (const entry of contents) { + const metadata = extractUserMetadata(entry); + if (Object.keys(metadata).length > 0 && entry.Key) { + metadataByKey.set(String(entry.Key), metadata); } } - - return metadata; + return metadataByKey; } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -114,7 +166,22 @@ const captureResponseBodyMiddleware = (captured: { xml: string }) => (next: any) // eslint-disable-next-line @typescript-eslint/no-explicit-any const parseUserMetadataMiddleware = (captured: { xml: string }) => (next: any) => async (args: any) => { const result = await next(args); - Object.assign(result.output, parseUserMetadataFromXml(captured.xml)); + Object.assign(result.output, parseGetObjectAttributesUserMetadata(captured.xml)); + return result; +}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const parseListObjectsUserMetadataMiddleware = (captured: { xml: string }) => (next: any) => async (args: any) => { + const result = await next(args); + const metadataByKey = parseListObjectsUserMetadata(captured.xml); + if (result.output.Contents && metadataByKey.size > 0) { + for (const content of result.output.Contents) { + const metadata = metadataByKey.get(content.Key); + if (metadata) { + Object.assign(content, metadata); + } + } + } return result; }; @@ -132,7 +199,7 @@ export class GetObjectAttributesExtendedCommand extends GetObjectAttributesComma const captured = { xml: '' }; - this.middlewareStack.add(overrideObjectAttributesHeaderMiddleware(input.ObjectAttributes), { + this.middlewareStack.add(overrideObjectAttributesHeaderMiddleware('x-amz-object-attributes', input.ObjectAttributes), { step: 'build', name: 'overrideObjectAttributesHeader', }); diff --git a/tests/testS3ExtendedApis.test.ts b/tests/testS3ExtendedApis.test.ts index 14ebe0d1..85e170db 100644 --- a/tests/testS3ExtendedApis.test.ts +++ b/tests/testS3ExtendedApis.test.ts @@ -1,4 +1,4 @@ -import { S3Client, PutObjectCommand, DeleteObjectCommand, GetObjectAttributesCommandOutput } from '@aws-sdk/client-s3'; +import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; import { createTestClient, testConfig } from './testSetup'; import { describeForMongoBackend } from './testHelpers'; import assert from 'assert'; @@ -7,6 +7,7 @@ import { GetObjectAttributesExtendedOutput, ListObjectsExtendedCommand, ListObjectsV2ExtendedCommand, + ListObjectsV2ExtendedOutput, ListObjectVersionsExtendedCommand, } from '../src/clients/s3Extended'; @@ -139,6 +140,112 @@ describeForMongoBackend('S3 Extended API Tests', () => { }); }); + describe.skip('ListObjectsV2 with ObjectAttributes', () => { + const metaKey1 = `${testConfig.objectKey}-listv2-meta1`; + const metaKey2 = `${testConfig.objectKey}-listv2-meta2`; + + beforeAll(async () => { + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metaKey1, + Body: 'data1', + Metadata: { foo: 'bar', baz: 'qux' }, + })); + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metaKey2, + Body: 'data2', + Metadata: { foo: 'hello' }, + })); + }); + + it('should list objects with a single user metadata key', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + }); + + it('should list objects with multiple user metadata keys', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-baz'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj1['x-amz-meta-baz'], 'qux'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + assert.strictEqual(obj2['x-amz-meta-baz'], undefined); + }); + + it('should list objects with wildcard user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-*'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj1['x-amz-meta-baz'], 'qux'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + }); + + it('should list objects with non-existing user metadata key', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-nonexistent'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-nonexistent'], undefined); + }); + + it('should list objects with RestoreStatus combined with user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['RestoreStatus', 'x-amz-meta-foo'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.deepStrictEqual(obj1.RestoreStatus, { IsRestoreInProgress: false }); + }); + + it('should list objects with RestoreStatus combined with non-existing user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['RestoreStatus', 'x-amz-meta-nonexistent'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-nonexistent'], undefined); + assert.deepStrictEqual(obj1.RestoreStatus, { IsRestoreInProgress: false }); + }); + }); + describe('GetObjectAttributes', () => { const metadataKey = `${testConfig.objectKey}-with-meta`; const metadataBody = 'data-with-metadata'; From 5b92339ce022bfa0ed754cf2a4b1ec6dd1965a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Wed, 11 Mar 2026 18:46:00 +0100 Subject: [PATCH 3/5] Split s3Extended into per-command files under src/commands Issue: CLDSRVCLT-10 --- examples/{s3extended.ts => s3Extended.ts} | 2 +- src/clients/s3Extended.ts | 219 --------- .../s3Extended/getObjectAttributes.ts | 45 ++ src/commands/s3Extended/index.ts | 4 + src/commands/s3Extended/listObjectVersions.ts | 17 + src/commands/s3Extended/listObjects.ts | 17 + src/commands/s3Extended/listObjectsV2.ts | 60 +++ src/commands/s3Extended/utils.ts | 87 ++++ src/index.ts | 2 +- .../s3Extended/getObjectAttributes.test.ts | 203 ++++++++ .../s3Extended/listObjectVersions.test.ts | 64 +++ tests/commands/s3Extended/listObjects.test.ts | 71 +++ .../commands/s3Extended/listObjectsV2.test.ts | 157 +++++++ tests/testS3ExtendedApis.test.ts | 437 ------------------ 14 files changed, 727 insertions(+), 658 deletions(-) rename examples/{s3extended.ts => s3Extended.ts} (94%) delete mode 100644 src/clients/s3Extended.ts create mode 100644 src/commands/s3Extended/getObjectAttributes.ts create mode 100644 src/commands/s3Extended/index.ts create mode 100644 src/commands/s3Extended/listObjectVersions.ts create mode 100644 src/commands/s3Extended/listObjects.ts create mode 100644 src/commands/s3Extended/listObjectsV2.ts create mode 100644 src/commands/s3Extended/utils.ts create mode 100644 tests/commands/s3Extended/getObjectAttributes.test.ts create mode 100644 tests/commands/s3Extended/listObjectVersions.test.ts create mode 100644 tests/commands/s3Extended/listObjects.test.ts create mode 100644 tests/commands/s3Extended/listObjectsV2.test.ts delete mode 100644 tests/testS3ExtendedApis.test.ts diff --git a/examples/s3extended.ts b/examples/s3Extended.ts similarity index 94% rename from examples/s3extended.ts rename to examples/s3Extended.ts index 4303e2cd..dae48e41 100644 --- a/examples/s3extended.ts +++ b/examples/s3Extended.ts @@ -1,6 +1,6 @@ import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3'; -import { ListObjectsV2ExtendedCommand } from '@scality/cloudserverclient/clients/s3Extended'; +import { ListObjectsV2ExtendedCommand } from '@scality/cloudserverclient/commands/s3Extended'; const config: S3ClientConfig = { endpoint: 'http://localhost:8000', diff --git a/src/clients/s3Extended.ts b/src/clients/s3Extended.ts deleted file mode 100644 index d9c8e988..00000000 --- a/src/clients/s3Extended.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { Readable } from 'stream'; -import { - GetObjectAttributesCommand, - GetObjectAttributesCommandInput, - GetObjectAttributesCommandOutput, - ListObjectsCommand, - ListObjectsCommandInput, - ListObjectsV2Command, - ListObjectsV2CommandInput, - ListObjectsV2CommandOutput, - _Object, - ListObjectVersionsCommand, - ListObjectVersionsCommandInput, - ObjectAttributes, - OptionalObjectAttributes -} from '@aws-sdk/client-s3'; -import { streamCollector } from '@smithy/node-http-handler'; -import { XMLParser } from 'fast-xml-parser'; - -const extendCommandWithExtraParametersMiddleware = (query: string) => - (next: any) => async (args: any) => { - const request = args.request as any; - if (request.query) { - request.query.search = query; - } else { - request.query = { search: query }; - } - return next(args); - }; - -export interface ListObjectsExtendedInput extends ListObjectsCommandInput { - Query: string; -} - -export class ListObjectsExtendedCommand extends ListObjectsCommand { - constructor(input: ListObjectsExtendedInput) { - super(input); - - this.middlewareStack.add( - extendCommandWithExtraParametersMiddleware(input.Query), - { step: 'build', name: 'extendCommandWithExtraParameters' } - ); - } -} - -export interface ListObjectsV2ExtendedInput extends ListObjectsV2CommandInput { - Query?: string; - ObjectAttributes?: (OptionalObjectAttributes | `x-amz-meta-${string}`)[]; -} - -export interface ListObjectsV2ExtendedContentEntry extends _Object { - [key: `x-amz-meta-${string}`]: string; -} - -export interface ListObjectsV2ExtendedOutput extends ListObjectsV2CommandOutput { - Contents?: ListObjectsV2ExtendedContentEntry[]; -} - -export class ListObjectsV2ExtendedCommand extends ListObjectsV2Command { - constructor(input: ListObjectsV2ExtendedInput) { - super(input); - - this.middlewareStack.add( - extendCommandWithExtraParametersMiddleware(input.Query), - { step: 'build', name: 'extendCommandWithExtraParameters' } - ); - - if (input.ObjectAttributes?.length) { - const captured = { xml: '' }; - - this.middlewareStack.add(overrideObjectAttributesHeaderMiddleware('x-amz-optional-object-attributes', input.ObjectAttributes), { - step: 'build', - name: 'overrideObjectAttributesHeader', - }); - - this.middlewareStack.add(captureResponseBodyMiddleware(captured), { - step: 'deserialize', - name: 'captureResponseBody', - priority: 'low', - }); - - this.middlewareStack.add(parseListObjectsUserMetadataMiddleware(captured), { - step: 'deserialize', - name: 'parseUserMetadata', - priority: 'high', - }); - } - } -} - -export interface ListObjectVersionsExtendedInput extends ListObjectVersionsCommandInput { - Query: string; -} - -export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand { - constructor(input: ListObjectVersionsExtendedInput) { - super(input); - - this.middlewareStack.add( - extendCommandWithExtraParametersMiddleware(input.Query), - { step: 'build', name: 'extendCommandWithExtraParameters' } - ); - } -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const overrideObjectAttributesHeaderMiddleware = (headerName: string, attributes: string[]) => (next: any) => async (args: any) => { - const request = args.request; - request.headers[headerName] = attributes.join(','); - return next(args); -}; - -const USER_METADATA_PREFIX = 'x-amz-meta-'; - -function extractUserMetadata(obj: Record): Record { - const metadata: Record = {}; - for (const [key, value] of Object.entries(obj)) { - if (key.startsWith(USER_METADATA_PREFIX)) { - metadata[key] = String(value); - } - } - return metadata; -} - -function parseGetObjectAttributesUserMetadata(xml: string): Record { - const parsed = new XMLParser().parse(xml); - const response = parsed?.GetObjectAttributesResponse; - if (!response) { - return {}; - } - return extractUserMetadata(response); -} - -function parseListObjectsUserMetadata(xml: string): Map> { - const parsed = new XMLParser().parse(xml); - const result = parsed?.ListBucketResult; - if (!result?.Contents) { - return new Map(); - } - const contents = Array.isArray(result.Contents) ? result.Contents : [result.Contents]; - const metadataByKey = new Map>(); - for (const entry of contents) { - const metadata = extractUserMetadata(entry); - if (Object.keys(metadata).length > 0 && entry.Key) { - metadataByKey.set(String(entry.Key), metadata); - } - } - return metadataByKey; -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const captureResponseBodyMiddleware = (captured: { xml: string }) => (next: any) => async (args: any) => { - const { response } = await next(args); - - if (response?.body) { - const collected = await streamCollector(response.body); - const buffer = Buffer.from(collected); - // eslint-disable-next-line no-param-reassign - captured.xml = buffer.toString('utf-8'); - response.body = Readable.from([buffer]); - } - - return { response }; -}; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const parseUserMetadataMiddleware = (captured: { xml: string }) => (next: any) => async (args: any) => { - const result = await next(args); - Object.assign(result.output, parseGetObjectAttributesUserMetadata(captured.xml)); - return result; -}; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const parseListObjectsUserMetadataMiddleware = (captured: { xml: string }) => (next: any) => async (args: any) => { - const result = await next(args); - const metadataByKey = parseListObjectsUserMetadata(captured.xml); - if (result.output.Contents && metadataByKey.size > 0) { - for (const content of result.output.Contents) { - const metadata = metadataByKey.get(content.Key); - if (metadata) { - Object.assign(content, metadata); - } - } - } - return result; -}; - -export interface GetObjectAttributesExtendedInput extends Omit { - ObjectAttributes: (ObjectAttributes | `x-amz-meta-${string}`)[]; -} - -export interface GetObjectAttributesExtendedOutput extends GetObjectAttributesCommandOutput { - [key: `x-amz-meta-${string}`]: string; -} - -export class GetObjectAttributesExtendedCommand extends GetObjectAttributesCommand { - constructor(input: GetObjectAttributesExtendedInput) { - super(input as GetObjectAttributesCommandInput); - - const captured = { xml: '' }; - - this.middlewareStack.add(overrideObjectAttributesHeaderMiddleware('x-amz-object-attributes', input.ObjectAttributes), { - step: 'build', - name: 'overrideObjectAttributesHeader', - }); - - this.middlewareStack.add(captureResponseBodyMiddleware(captured), { - step: 'deserialize', - name: 'captureResponseBody', - priority: 'low', // runs before SDK deserializer - }); - - this.middlewareStack.add(parseUserMetadataMiddleware(captured), { - step: 'deserialize', - name: 'parseUserMetadata', - priority: 'high', // runs after SDK deserializer - }); - } -} diff --git a/src/commands/s3Extended/getObjectAttributes.ts b/src/commands/s3Extended/getObjectAttributes.ts new file mode 100644 index 00000000..55818e17 --- /dev/null +++ b/src/commands/s3Extended/getObjectAttributes.ts @@ -0,0 +1,45 @@ +import { + GetObjectAttributesCommand, + GetObjectAttributesCommandInput, + GetObjectAttributesCommandOutput, + ObjectAttributes, +} from '@aws-sdk/client-s3'; +import { + overrideObjectAttributesHeaderMiddleware, + captureResponseBodyMiddleware, + parseUserMetadataMiddleware, +} from './utils'; + +export interface GetObjectAttributesExtendedInput extends Omit { + ObjectAttributes: (ObjectAttributes | `x-amz-meta-${string}`)[]; +} + +export interface GetObjectAttributesExtendedOutput extends GetObjectAttributesCommandOutput { + [key: `x-amz-meta-${string}`]: string; +} + +export class GetObjectAttributesExtendedCommand extends GetObjectAttributesCommand { + constructor(input: GetObjectAttributesExtendedInput) { + super(input as GetObjectAttributesCommandInput); + + const captured = { xml: '' }; + + this.middlewareStack.add( + overrideObjectAttributesHeaderMiddleware('x-amz-object-attributes', input.ObjectAttributes), { + step: 'build', + name: 'overrideObjectAttributesHeader', + }); + + this.middlewareStack.add(captureResponseBodyMiddleware(captured), { + step: 'deserialize', + name: 'captureResponseBody', + priority: 'low', // runs before SDK deserializer + }); + + this.middlewareStack.add(parseUserMetadataMiddleware(captured), { + step: 'deserialize', + name: 'parseUserMetadata', + priority: 'high', // runs after SDK deserializer + }); + } +} diff --git a/src/commands/s3Extended/index.ts b/src/commands/s3Extended/index.ts new file mode 100644 index 00000000..cbe48fcc --- /dev/null +++ b/src/commands/s3Extended/index.ts @@ -0,0 +1,4 @@ +export * from './listObjects'; +export * from './listObjectsV2'; +export * from './listObjectVersions'; +export * from './getObjectAttributes'; diff --git a/src/commands/s3Extended/listObjectVersions.ts b/src/commands/s3Extended/listObjectVersions.ts new file mode 100644 index 00000000..81bfab26 --- /dev/null +++ b/src/commands/s3Extended/listObjectVersions.ts @@ -0,0 +1,17 @@ +import { ListObjectVersionsCommand, ListObjectVersionsCommandInput } from '@aws-sdk/client-s3'; +import { extendCommandWithExtraParametersMiddleware } from './utils'; + +export interface ListObjectVersionsExtendedInput extends ListObjectVersionsCommandInput { + Query: string; +} + +export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand { + constructor(input: ListObjectVersionsExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + } +} diff --git a/src/commands/s3Extended/listObjects.ts b/src/commands/s3Extended/listObjects.ts new file mode 100644 index 00000000..0ca9069a --- /dev/null +++ b/src/commands/s3Extended/listObjects.ts @@ -0,0 +1,17 @@ +import { ListObjectsCommand, ListObjectsCommandInput } from '@aws-sdk/client-s3'; +import { extendCommandWithExtraParametersMiddleware } from './utils'; + +export interface ListObjectsExtendedInput extends ListObjectsCommandInput { + Query: string; +} + +export class ListObjectsExtendedCommand extends ListObjectsCommand { + constructor(input: ListObjectsExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + } +} diff --git a/src/commands/s3Extended/listObjectsV2.ts b/src/commands/s3Extended/listObjectsV2.ts new file mode 100644 index 00000000..9f441d82 --- /dev/null +++ b/src/commands/s3Extended/listObjectsV2.ts @@ -0,0 +1,60 @@ +import { + ListObjectsV2Command, + ListObjectsV2CommandInput, + ListObjectsV2CommandOutput, + OptionalObjectAttributes, + _Object, +} from '@aws-sdk/client-s3'; +import { + extendCommandWithExtraParametersMiddleware, + overrideObjectAttributesHeaderMiddleware, + captureResponseBodyMiddleware, + parseListObjectsUserMetadataMiddleware, +} from './utils'; + +export interface ListObjectsV2ExtendedInput extends ListObjectsV2CommandInput { + Query?: string; + ObjectAttributes?: (OptionalObjectAttributes | `x-amz-meta-${string}`)[]; +} + +export interface ListObjectsV2ExtendedContentEntry extends _Object { + [key: `x-amz-meta-${string}`]: string; +} + +export interface ListObjectsV2ExtendedOutput extends ListObjectsV2CommandOutput { + Contents?: ListObjectsV2ExtendedContentEntry[]; +} + +export class ListObjectsV2ExtendedCommand extends ListObjectsV2Command { + constructor(input: ListObjectsV2ExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + + if (input.ObjectAttributes?.length) { + const captured = { xml: '' }; + + this.middlewareStack.add( + overrideObjectAttributesHeaderMiddleware( + 'x-amz-optional-object-attributes', input.ObjectAttributes), { + step: 'build', + name: 'overrideObjectAttributesHeader', + }); + + this.middlewareStack.add(captureResponseBodyMiddleware(captured), { + step: 'deserialize', + name: 'captureResponseBody', + priority: 'low', + }); + + this.middlewareStack.add(parseListObjectsUserMetadataMiddleware(captured), { + step: 'deserialize', + name: 'parseUserMetadata', + priority: 'high', + }); + } + } +} diff --git a/src/commands/s3Extended/utils.ts b/src/commands/s3Extended/utils.ts new file mode 100644 index 00000000..820139a9 --- /dev/null +++ b/src/commands/s3Extended/utils.ts @@ -0,0 +1,87 @@ +import { Readable } from 'stream'; +import { streamCollector } from '@smithy/node-http-handler'; +import { XMLParser } from 'fast-xml-parser'; + +export const USER_METADATA_PREFIX = 'x-amz-meta-'; + + +export function extendCommandWithExtraParametersMiddleware(query: string) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const request = args.request as any; + if (request.query) { + request.query.search = query; + } else { + request.query = { search: query }; + } + return next(args); + }; +} + + +export function overrideObjectAttributesHeaderMiddleware(headerName: string, attributes: string[]) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + const request = args.request; + request.headers[headerName] = attributes.join(','); + return next(args); + }; +} + +export function captureResponseBodyMiddleware(captured: { xml: string }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + const { response } = await next(args); + + if (response?.body) { + const collected = await streamCollector(response.body); + const buffer = Buffer.from(collected); + // eslint-disable-next-line no-param-reassign + captured.xml = buffer.toString('utf-8'); + // Re-create the body stream so the SDK deserializer can still consume it + response.body = Readable.from([buffer]); + } + + return { response }; + }; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function extractUserMetadata(source: Record, target: Record): void { + for (const [key, value] of Object.entries(source)) { + if (key.startsWith(USER_METADATA_PREFIX)) { + // eslint-disable-next-line no-param-reassign + target[key] = String(value); + } + } +} + +export function parseUserMetadataMiddleware(captured: { xml: string }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + const result = await next(args); + const parsed = new XMLParser().parse(captured.xml); + const response = parsed?.GetObjectAttributesResponse; + if (response) { + extractUserMetadata(response, result.output); + } + return result; + }; +} + + +export function parseListObjectsUserMetadataMiddleware(captured: { xml: string }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + const result = await next(args); + const parsed = new XMLParser().parse(captured.xml); + const xmlContents = parsed?.ListBucketResult?.Contents; + if (result.output.Contents && xmlContents) { + for (let i = 0; i < result.output.Contents.length; i++) { + extractUserMetadata(xmlContents[i], result.output.Contents[i]); + } + } + return result; + }; +} diff --git a/src/index.ts b/src/index.ts index 85656164..d35a8b09 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ export * from './clients/backbeatRoutes'; export * from './clients/bucketQuota'; export * from './clients/proxyBackbeatApis'; -export * from './clients/s3Extended'; +export * from './commands/s3Extended'; export { CloudserverClient, CloudserverClientConfig } from './clients/cloudserver'; export * from './utils'; diff --git a/tests/commands/s3Extended/getObjectAttributes.test.ts b/tests/commands/s3Extended/getObjectAttributes.test.ts new file mode 100644 index 00000000..3d6a077a --- /dev/null +++ b/tests/commands/s3Extended/getObjectAttributes.test.ts @@ -0,0 +1,203 @@ +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from '../../testSetup'; +import { describeForMongoBackend } from '../../testHelpers'; +import assert from 'assert'; +import { + GetObjectAttributesExtendedCommand, + GetObjectAttributesExtendedOutput, +} from '../../../src/commands/s3Extended'; + +describeForMongoBackend('GetObjectAttributesExtended', () => { + let s3client: S3Client; + const metadataKey = `${testConfig.objectKey}-with-meta`; + const metadataBody = 'data-with-metadata'; + const metadataETag = '0235b027419caf7c0e4b0840f7ec21a6'; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + }); + + beforeAll(async () => { + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + Body: metadataBody, + Metadata: { + foo: 'bar', + bar: 'baz', + baz: 'qux', + }, + })); + }); + + it('should get ETag only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + }); + + it('should get ObjectSize only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ObjectSize'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ObjectSize, 11); + }); + + it('should get StorageClass only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['StorageClass'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.StorageClass, 'STANDARD'); + }); + + it('should get all standard attributes at once', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag', 'ObjectParts', 'StorageClass', 'ObjectSize'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + assert.strictEqual(result.ObjectSize, 11); + assert.strictEqual(result.StorageClass, 'STANDARD'); + }); + + it('should get a single user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + }); + + it('should get multiple user metadata keys', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-bar', 'x-amz-meta-baz'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get all user metadata with wildcard', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-*'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get wildcard combined with a specific user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-*', 'x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should handle non-existing user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['x-amz-meta-nonexistent'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-nonexistent'], undefined); + }); + + it('should get standard attribute combined with non-existing user metadata', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag', 'x-amz-meta-nonexistent'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + assert.strictEqual(result['x-amz-meta-nonexistent'], undefined); + }); + + it('should get ETag combined with a single user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['ETag', 'x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + }); + + it('should get all standard attributes combined with all user metadata', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: [ + 'ETag', + 'ObjectParts', + 'StorageClass', + 'ObjectSize', + 'x-amz-meta-foo', + 'x-amz-meta-bar', + 'x-amz-meta-baz', + ], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result.ObjectSize, 18); + assert.strictEqual(result.StorageClass, 'STANDARD'); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get standard attributes combined with user metadata wildcard', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['ETag', 'ObjectSize', 'x-amz-meta-*'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result.ObjectSize, 18); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); +}); diff --git a/tests/commands/s3Extended/listObjectVersions.test.ts b/tests/commands/s3Extended/listObjectVersions.test.ts new file mode 100644 index 00000000..1d037251 --- /dev/null +++ b/tests/commands/s3Extended/listObjectVersions.test.ts @@ -0,0 +1,64 @@ +import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from '../../testSetup'; +import { describeForMongoBackend } from '../../testHelpers'; +import assert from 'assert'; +import { ListObjectVersionsExtendedCommand } from '../../../src/commands/s3Extended'; + +describeForMongoBackend('ListObjectVersionsExtended', () => { + let s3client: S3Client; + const key2ndObject = `${testConfig.objectKey}2nd`; + const body2ndObject = `${testConfig.objectData}2nd`; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + }); + + beforeAll(async () => { + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); + }); + + it('should test ListObjectVersionsExtended', async () => { + const getCommand1 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 100, + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Versions?.length, 2); + assert.strictEqual(getData1.Versions[0].IsLatest, true); + + // Delete one object to create a DeleteMarker + const deleteCommand = new DeleteObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + }); + await s3client.send(deleteCommand); + + const getCommand2 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 100, + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.DeleteMarkers?.[0].Key, key2ndObject); + assert.ok(getData2.DeleteMarkers?.[0].VersionId); + + assert.ok(getData2.Versions); + const firstVersion = getData2.Versions[0]; + const getCommand3 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + KeyMarker: firstVersion.Key, + VersionIdMarker: firstVersion.VersionId, + MaxKeys: 100, + }); + const getData3 = await s3client.send(getCommand3); + assert.notStrictEqual(getData3.Versions?.[0]?.Key, firstVersion.Key); + }); +}); diff --git a/tests/commands/s3Extended/listObjects.test.ts b/tests/commands/s3Extended/listObjects.test.ts new file mode 100644 index 00000000..50aa5489 --- /dev/null +++ b/tests/commands/s3Extended/listObjects.test.ts @@ -0,0 +1,71 @@ +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from '../../testSetup'; +import { describeForMongoBackend } from '../../testHelpers'; +import assert from 'assert'; +import { ListObjectsExtendedCommand } from '../../../src/commands/s3Extended'; + +describeForMongoBackend('ListObjectsExtended', () => { + let s3client: S3Client; + const key2ndObject = `${testConfig.objectKey}2nd`; + const body2ndObject = `${testConfig.objectData}2nd`; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + }); + + beforeAll(async () => { + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); + }); + + it('should test ListObjectsExtended', async () => { + const getCommand1= new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + + const maxKey = 1; + const getCommand2= new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: maxKey + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, maxKey); + assert.strictEqual(getData2.IsTruncated, true); + + const getCommand3 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length > ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData3 = await s3client.send(getCommand3); + assert.strictEqual(getData3.Contents?.length, 1); + assert.strictEqual(getData3.Contents[0].Key, key2ndObject); + + const getCommand4 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `key = ${key2ndObject}`, + MaxKeys: 5 + }); + const getData4 = await s3client.send(getCommand4); + assert.strictEqual(getData4.Contents?.length, 1); + assert.strictEqual(getData4.Contents[0].Key, key2ndObject); + + const getCommand5 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'key = iDontExists', + MaxKeys: 5 + }); + const getData5 = await s3client.send(getCommand5); + assert.strictEqual(getData5.Contents, undefined); + }); +}); diff --git a/tests/commands/s3Extended/listObjectsV2.test.ts b/tests/commands/s3Extended/listObjectsV2.test.ts new file mode 100644 index 00000000..d952c326 --- /dev/null +++ b/tests/commands/s3Extended/listObjectsV2.test.ts @@ -0,0 +1,157 @@ +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from '../../testSetup'; +import { describeForMongoBackend } from '../../testHelpers'; +import assert from 'assert'; +import { + ListObjectsV2ExtendedCommand, + ListObjectsV2ExtendedOutput, +} from '../../../src/commands/s3Extended'; + +describeForMongoBackend('ListObjectsV2Extended', () => { + let s3client: S3Client; + const key2ndObject = `${testConfig.objectKey}2nd`; + const body2ndObject = `${testConfig.objectData}2nd`; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + }); + + beforeAll(async () => { + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); + }); + + it('should test ListObjectsV2Extended', async () => { + const getCommand1= new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5, + FetchOwner: true, + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + assert.strictEqual(getData1.KeyCount, 2); + assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart'); + + const getCommand2 = new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 5, + StartAfter: testConfig.objectKey, // Skip first object + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, 1); + assert.strictEqual(getData2.Contents[0].Key, key2ndObject); + }); + + describe.skip('ListObjectsV2 with ObjectAttributes', () => { + const metaKey1 = `${testConfig.objectKey}-listv2-meta1`; + const metaKey2 = `${testConfig.objectKey}-listv2-meta2`; + + beforeAll(async () => { + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metaKey1, + Body: 'data1', + Metadata: { foo: 'bar', baz: 'qux' }, + })); + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metaKey2, + Body: 'data2', + Metadata: { foo: 'hello' }, + })); + }); + + it('should list objects with a single user metadata key', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + }); + + it('should list objects with multiple user metadata keys', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-baz'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj1['x-amz-meta-baz'], 'qux'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + assert.strictEqual(obj2['x-amz-meta-baz'], undefined); + }); + + it('should list objects with wildcard user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-*'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj1['x-amz-meta-baz'], 'qux'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + }); + + it('should list objects with non-existing user metadata key', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-nonexistent'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-nonexistent'], undefined); + }); + + it('should list objects with RestoreStatus combined with user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['RestoreStatus', 'x-amz-meta-foo'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.deepStrictEqual(obj1.RestoreStatus, { IsRestoreInProgress: false }); + }); + + it('should list objects with RestoreStatus combined with non-existing user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['RestoreStatus', 'x-amz-meta-nonexistent'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-nonexistent'], undefined); + assert.deepStrictEqual(obj1.RestoreStatus, { IsRestoreInProgress: false }); + }); + }); +}); diff --git a/tests/testS3ExtendedApis.test.ts b/tests/testS3ExtendedApis.test.ts deleted file mode 100644 index 85e170db..00000000 --- a/tests/testS3ExtendedApis.test.ts +++ /dev/null @@ -1,437 +0,0 @@ -import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; -import { createTestClient, testConfig } from './testSetup'; -import { describeForMongoBackend } from './testHelpers'; -import assert from 'assert'; -import { - GetObjectAttributesExtendedCommand, - GetObjectAttributesExtendedOutput, - ListObjectsExtendedCommand, - ListObjectsV2ExtendedCommand, - ListObjectsV2ExtendedOutput, - ListObjectVersionsExtendedCommand, -} from '../src/clients/s3Extended'; - -describeForMongoBackend('S3 Extended API Tests', () => { - let s3client: S3Client; - const key2ndObject = `${testConfig.objectKey}2nd`; - const body2ndObject = `${testConfig.objectData}2nd`; - - beforeAll(async () => { - const testClients = createTestClient(); - s3client = testClients.s3client; - }); - - describe('ListObjects', () => { - beforeAll(async () => { - const putObjectCommand = new PutObjectCommand({ - Bucket: testConfig.bucketName, - Key: key2ndObject, - Body: body2ndObject, - }); - await s3client.send(putObjectCommand); - }); - - it('should test ListObjectsExtended', async () => { - const getCommand1= new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: 5 - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Contents?.length, 2); - - const maxKey = 1; - const getCommand2= new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: maxKey - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.Contents?.length, maxKey); - assert.strictEqual(getData2.IsTruncated, true); - - const getCommand3 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length > ${testConfig.objectData.length}`, - MaxKeys: 5 - }); - const getData3 = await s3client.send(getCommand3); - assert.strictEqual(getData3.Contents?.length, 1); - assert.strictEqual(getData3.Contents[0].Key, key2ndObject); - - const getCommand4 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `key = ${key2ndObject}`, - MaxKeys: 5 - }); - const getData4 = await s3client.send(getCommand4); - assert.strictEqual(getData4.Contents?.length, 1); - assert.strictEqual(getData4.Contents[0].Key, key2ndObject); - - const getCommand5 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: 'key = iDontExists', - MaxKeys: 5 - }); - const getData5 = await s3client.send(getCommand5); - assert.strictEqual(getData5.Contents, undefined); - }); - - it('should test ListObjectsV2Extended', async () => { - const getCommand1= new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: 5, - FetchOwner: true, - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Contents?.length, 2); - assert.strictEqual(getData1.KeyCount, 2); - assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart'); - - const getCommand2 = new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - Query: 'content-length >= 0', - MaxKeys: 5, - StartAfter: testConfig.objectKey, // Skip first object - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.Contents?.length, 1); - assert.strictEqual(getData2.Contents[0].Key, key2ndObject); - }); - - it('should test ListObjectVersionsExtended', async () => { - const getCommand1 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: 'content-length >= 0', - MaxKeys: 100, - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Versions?.length, 2); - assert.strictEqual(getData1.Versions[0].IsLatest, true); - - // Delete one object to create a DeleteMarker - const deleteCommand = new DeleteObjectCommand({ - Bucket: testConfig.bucketName, - Key: key2ndObject, - }); - await s3client.send(deleteCommand); - - const getCommand2 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: 'content-length >= 0', - MaxKeys: 100, - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.DeleteMarkers?.[0].Key, key2ndObject); - assert.ok(getData2.DeleteMarkers?.[0].VersionId); - - assert.ok(getData2.Versions); - const firstVersion = getData2.Versions[0]; - const getCommand3 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: 'content-length >= 0', - KeyMarker: firstVersion.Key, - VersionIdMarker: firstVersion.VersionId, - MaxKeys: 100, - }); - const getData3 = await s3client.send(getCommand3); - assert.notStrictEqual(getData3.Versions?.[0]?.Key, firstVersion.Key); - }); - }); - - describe.skip('ListObjectsV2 with ObjectAttributes', () => { - const metaKey1 = `${testConfig.objectKey}-listv2-meta1`; - const metaKey2 = `${testConfig.objectKey}-listv2-meta2`; - - beforeAll(async () => { - await s3client.send(new PutObjectCommand({ - Bucket: testConfig.bucketName, - Key: metaKey1, - Body: 'data1', - Metadata: { foo: 'bar', baz: 'qux' }, - })); - await s3client.send(new PutObjectCommand({ - Bucket: testConfig.bucketName, - Key: metaKey2, - Body: 'data2', - Metadata: { foo: 'hello' }, - })); - }); - - it('should list objects with a single user metadata key', async () => { - const result = await s3client.send(new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })) as ListObjectsV2ExtendedOutput; - - const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; - const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.Contents?.length, 2); - assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); - assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); - }); - - it('should list objects with multiple user metadata keys', async () => { - const result = await s3client.send(new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-baz'], - })) as ListObjectsV2ExtendedOutput; - - const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; - const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); - assert.strictEqual(obj1['x-amz-meta-baz'], 'qux'); - assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); - assert.strictEqual(obj2['x-amz-meta-baz'], undefined); - }); - - it('should list objects with wildcard user metadata', async () => { - const result = await s3client.send(new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - ObjectAttributes: ['x-amz-meta-*'], - })) as ListObjectsV2ExtendedOutput; - - const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; - const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); - assert.strictEqual(obj1['x-amz-meta-baz'], 'qux'); - assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); - }); - - it('should list objects with non-existing user metadata key', async () => { - const result = await s3client.send(new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - ObjectAttributes: ['x-amz-meta-nonexistent'], - })) as ListObjectsV2ExtendedOutput; - - const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.Contents?.length, 2); - assert.strictEqual(obj1['x-amz-meta-nonexistent'], undefined); - }); - - it('should list objects with RestoreStatus combined with user metadata', async () => { - const result = await s3client.send(new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - ObjectAttributes: ['RestoreStatus', 'x-amz-meta-foo'], - })) as ListObjectsV2ExtendedOutput; - - const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); - assert.deepStrictEqual(obj1.RestoreStatus, { IsRestoreInProgress: false }); - }); - - it('should list objects with RestoreStatus combined with non-existing user metadata', async () => { - const result = await s3client.send(new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - ObjectAttributes: ['RestoreStatus', 'x-amz-meta-nonexistent'], - })) as ListObjectsV2ExtendedOutput; - - const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.Contents?.length, 2); - assert.strictEqual(obj1['x-amz-meta-nonexistent'], undefined); - assert.deepStrictEqual(obj1.RestoreStatus, { IsRestoreInProgress: false }); - }); - }); - - describe('GetObjectAttributes', () => { - const metadataKey = `${testConfig.objectKey}-with-meta`; - const metadataBody = 'data-with-metadata'; - const metadataETag = '0235b027419caf7c0e4b0840f7ec21a6'; - - beforeAll(async () => { - await s3client.send(new PutObjectCommand({ - Bucket: testConfig.bucketName, - Key: metadataKey, - Body: metadataBody, - Metadata: { - foo: 'bar', - bar: 'baz', - baz: 'qux', - }, - })); - }); - - it('should get ETag only', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: testConfig.objectKey, - ObjectAttributes: ['ETag'], - })); - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); - }); - - it('should get ObjectSize only', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: testConfig.objectKey, - ObjectAttributes: ['ObjectSize'], - })); - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.ObjectSize, 11); - }); - - it('should get StorageClass only', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: testConfig.objectKey, - ObjectAttributes: ['StorageClass'], - })); - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.StorageClass, 'STANDARD'); - }); - - it('should get all standard attributes at once', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: testConfig.objectKey, - ObjectAttributes: ['ETag', 'ObjectParts', 'StorageClass', 'ObjectSize'], - })); - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); - assert.strictEqual(result.ObjectSize, 11); - assert.strictEqual(result.StorageClass, 'STANDARD'); - }); - - it('should get a single user metadata key', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: metadataKey, - ObjectAttributes: ['x-amz-meta-foo'], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result['x-amz-meta-foo'], 'bar'); - }); - - it('should get multiple user metadata keys', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: metadataKey, - ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-bar', 'x-amz-meta-baz'], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result['x-amz-meta-foo'], 'bar'); - assert.strictEqual(result['x-amz-meta-bar'], 'baz'); - assert.strictEqual(result['x-amz-meta-baz'], 'qux'); - }); - - it('should get all user metadata with wildcard', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: metadataKey, - ObjectAttributes: ['x-amz-meta-*'], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result['x-amz-meta-foo'], 'bar'); - assert.strictEqual(result['x-amz-meta-bar'], 'baz'); - assert.strictEqual(result['x-amz-meta-baz'], 'qux'); - }); - - it('should get wildcard combined with a specific user metadata key', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: metadataKey, - ObjectAttributes: ['x-amz-meta-*', 'x-amz-meta-foo'], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result['x-amz-meta-foo'], 'bar'); - assert.strictEqual(result['x-amz-meta-bar'], 'baz'); - assert.strictEqual(result['x-amz-meta-baz'], 'qux'); - }); - - it('should handle non-existing user metadata key', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: testConfig.objectKey, - ObjectAttributes: ['x-amz-meta-nonexistent'], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result['x-amz-meta-nonexistent'], undefined); - }); - - it('should get standard attribute combined with non-existing user metadata', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: testConfig.objectKey, - ObjectAttributes: ['ETag', 'x-amz-meta-nonexistent'], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); - assert.strictEqual(result['x-amz-meta-nonexistent'], undefined); - }); - - it('should get ETag combined with a single user metadata key', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: metadataKey, - ObjectAttributes: ['ETag', 'x-amz-meta-foo'], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.ETag, metadataETag); - assert.strictEqual(result['x-amz-meta-foo'], 'bar'); - }); - - it('should get all standard attributes combined with all user metadata', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: metadataKey, - ObjectAttributes: [ - 'ETag', - 'ObjectParts', - 'StorageClass', - 'ObjectSize', - 'x-amz-meta-foo', - 'x-amz-meta-bar', - 'x-amz-meta-baz', - ], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.ETag, metadataETag); - assert.strictEqual(result.ObjectSize, 18); - assert.strictEqual(result.StorageClass, 'STANDARD'); - assert.strictEqual(result['x-amz-meta-foo'], 'bar'); - assert.strictEqual(result['x-amz-meta-bar'], 'baz'); - assert.strictEqual(result['x-amz-meta-baz'], 'qux'); - }); - - it('should get standard attributes combined with user metadata wildcard', async () => { - const result = await s3client.send(new GetObjectAttributesExtendedCommand({ - Bucket: testConfig.bucketName, - Key: metadataKey, - ObjectAttributes: ['ETag', 'ObjectSize', 'x-amz-meta-*'], - })) as GetObjectAttributesExtendedOutput; - - assert.strictEqual(result.$metadata.httpStatusCode, 200); - assert.strictEqual(result.ETag, metadataETag); - assert.strictEqual(result.ObjectSize, 18); - assert.strictEqual(result['x-amz-meta-foo'], 'bar'); - assert.strictEqual(result['x-amz-meta-bar'], 'baz'); - assert.strictEqual(result['x-amz-meta-baz'], 'qux'); - }); - }); -}); From 4c958c774d799233d19870339f58f5efccabfad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Thu, 12 Mar 2026 19:22:28 +0100 Subject: [PATCH 4/5] Update cloudserver docker image version Issue: CLDSRVCLT-10 --- .github/docker.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/docker.env b/.github/docker.env index 18a1cc6f..784fe7f4 100644 --- a/.github/docker.env +++ b/.github/docker.env @@ -3,7 +3,7 @@ MONGODB_IMAGE=mongo:4.4 MONGODB_PORT=27018 # Cloudserver -CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.0-preview.1 +CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.4 CLOUDSERVER_PORT=8000 # Metadata From 5eaa299183b5ef9e040f1a7c3919102fc9f51635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Thu, 12 Mar 2026 19:22:45 +0100 Subject: [PATCH 5/5] Bump package.json to 1.0.5 Issue: CLDSRVCLT-10 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5d53a845..005630f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@scality/cloudserverclient", - "version": "1.0.4", + "version": "1.0.5", "engines": { "node": ">=20" },