diff --git a/.github/docker.env b/.github/docker.env index 18a1cc6f..784fe7f4 100644 --- a/.github/docker.env +++ b/.github/docker.env @@ -3,7 +3,7 @@ MONGODB_IMAGE=mongo:4.4 MONGODB_PORT=27018 # Cloudserver -CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.0-preview.1 +CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.4 CLOUDSERVER_PORT=8000 # Metadata diff --git a/examples/s3extended.ts b/examples/s3Extended.ts similarity index 94% rename from examples/s3extended.ts rename to examples/s3Extended.ts index 4303e2cd..db3670da 100644 --- a/examples/s3extended.ts +++ b/examples/s3Extended.ts @@ -1,6 +1,6 @@ import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3'; -import { ListObjectsV2ExtendedCommand } from '@scality/cloudserverclient/clients/s3Extended'; +import { ListObjectsV2ExtendedCommand } from '@scality/cloudserverclient'; const config: S3ClientConfig = { endpoint: 'http://localhost:8000', diff --git a/package.json b/package.json index 5d53a845..005630f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@scality/cloudserverclient", - "version": "1.0.4", + "version": "1.0.5", "engines": { "node": ">=20" }, diff --git a/src/clients/s3Extended.ts b/src/clients/s3Extended.ts deleted file mode 100644 index c3555a6e..00000000 --- a/src/clients/s3Extended.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { - ListObjectsCommand, - ListObjectsCommandInput, - ListObjectsV2Command, - ListObjectsV2CommandInput, - ListObjectVersionsCommand, - ListObjectVersionsCommandInput -} from '@aws-sdk/client-s3'; - -const extendCommandWithExtraParametersMiddleware = (query: string) => - (next: any) => async (args: any) => { - const request = args.request as any; - if (request.query) { - request.query.search = query; - } else { - request.query = { search: query }; - } - return next(args); - }; - -export interface ListObjectsExtendedInput extends ListObjectsCommandInput { - Query: string; -} - -export class ListObjectsExtendedCommand extends ListObjectsCommand { - constructor(input: ListObjectsExtendedInput) { - super(input); - - this.middlewareStack.add( - extendCommandWithExtraParametersMiddleware(input.Query), - { step: 'build', name: 'extendCommandWithExtraParameters' } - ); - } -} - -export interface ListObjectsV2ExtendedInput extends ListObjectsV2CommandInput { - Query: string; -} - -export class ListObjectsV2ExtendedCommand extends ListObjectsV2Command { - constructor(input: ListObjectsV2ExtendedInput) { - super(input); - - this.middlewareStack.add( - extendCommandWithExtraParametersMiddleware(input.Query), - { step: 'build', name: 'extendCommandWithExtraParameters' } - ); - } -} - -export interface ListObjectVersionsExtendedInput extends ListObjectVersionsCommandInput { - Query: string; -} - -export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand { - constructor(input: ListObjectVersionsExtendedInput) { - super(input); - - this.middlewareStack.add( - extendCommandWithExtraParametersMiddleware(input.Query), - { step: 'build', name: 'extendCommandWithExtraParameters' } - ); - } -} diff --git a/src/commands/s3Extended/getObjectAttributes.ts b/src/commands/s3Extended/getObjectAttributes.ts new file mode 100644 index 00000000..55818e17 --- /dev/null +++ b/src/commands/s3Extended/getObjectAttributes.ts @@ -0,0 +1,45 @@ +import { + GetObjectAttributesCommand, + GetObjectAttributesCommandInput, + GetObjectAttributesCommandOutput, + ObjectAttributes, +} from '@aws-sdk/client-s3'; +import { + overrideObjectAttributesHeaderMiddleware, + captureResponseBodyMiddleware, + parseUserMetadataMiddleware, +} from './utils'; + +export interface GetObjectAttributesExtendedInput extends Omit { + ObjectAttributes: (ObjectAttributes | `x-amz-meta-${string}`)[]; +} + +export interface GetObjectAttributesExtendedOutput extends GetObjectAttributesCommandOutput { + [key: `x-amz-meta-${string}`]: string; +} + +export class GetObjectAttributesExtendedCommand extends GetObjectAttributesCommand { + constructor(input: GetObjectAttributesExtendedInput) { + super(input as GetObjectAttributesCommandInput); + + const captured = { xml: '' }; + + this.middlewareStack.add( + overrideObjectAttributesHeaderMiddleware('x-amz-object-attributes', input.ObjectAttributes), { + step: 'build', + name: 'overrideObjectAttributesHeader', + }); + + this.middlewareStack.add(captureResponseBodyMiddleware(captured), { + step: 'deserialize', + name: 'captureResponseBody', + priority: 'low', // runs before SDK deserializer + }); + + this.middlewareStack.add(parseUserMetadataMiddleware(captured), { + step: 'deserialize', + name: 'parseUserMetadata', + priority: 'high', // runs after SDK deserializer + }); + } +} diff --git a/src/commands/s3Extended/index.ts b/src/commands/s3Extended/index.ts new file mode 100644 index 00000000..cbe48fcc --- /dev/null +++ b/src/commands/s3Extended/index.ts @@ -0,0 +1,4 @@ +export * from './listObjects'; +export * from './listObjectsV2'; +export * from './listObjectVersions'; +export * from './getObjectAttributes'; diff --git a/src/commands/s3Extended/listObjectVersions.ts b/src/commands/s3Extended/listObjectVersions.ts new file mode 100644 index 00000000..81bfab26 --- /dev/null +++ b/src/commands/s3Extended/listObjectVersions.ts @@ -0,0 +1,17 @@ +import { ListObjectVersionsCommand, ListObjectVersionsCommandInput } from '@aws-sdk/client-s3'; +import { extendCommandWithExtraParametersMiddleware } from './utils'; + +export interface ListObjectVersionsExtendedInput extends ListObjectVersionsCommandInput { + Query: string; +} + +export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand { + constructor(input: ListObjectVersionsExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + } +} diff --git a/src/commands/s3Extended/listObjects.ts b/src/commands/s3Extended/listObjects.ts new file mode 100644 index 00000000..0ca9069a --- /dev/null +++ b/src/commands/s3Extended/listObjects.ts @@ -0,0 +1,17 @@ +import { ListObjectsCommand, ListObjectsCommandInput } from '@aws-sdk/client-s3'; +import { extendCommandWithExtraParametersMiddleware } from './utils'; + +export interface ListObjectsExtendedInput extends ListObjectsCommandInput { + Query: string; +} + +export class ListObjectsExtendedCommand extends ListObjectsCommand { + constructor(input: ListObjectsExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + } +} diff --git a/src/commands/s3Extended/listObjectsV2.ts b/src/commands/s3Extended/listObjectsV2.ts new file mode 100644 index 00000000..9f441d82 --- /dev/null +++ b/src/commands/s3Extended/listObjectsV2.ts @@ -0,0 +1,60 @@ +import { + ListObjectsV2Command, + ListObjectsV2CommandInput, + ListObjectsV2CommandOutput, + OptionalObjectAttributes, + _Object, +} from '@aws-sdk/client-s3'; +import { + extendCommandWithExtraParametersMiddleware, + overrideObjectAttributesHeaderMiddleware, + captureResponseBodyMiddleware, + parseListObjectsUserMetadataMiddleware, +} from './utils'; + +export interface ListObjectsV2ExtendedInput extends ListObjectsV2CommandInput { + Query?: string; + ObjectAttributes?: (OptionalObjectAttributes | `x-amz-meta-${string}`)[]; +} + +export interface ListObjectsV2ExtendedContentEntry extends _Object { + [key: `x-amz-meta-${string}`]: string; +} + +export interface ListObjectsV2ExtendedOutput extends ListObjectsV2CommandOutput { + Contents?: ListObjectsV2ExtendedContentEntry[]; +} + +export class ListObjectsV2ExtendedCommand extends ListObjectsV2Command { + constructor(input: ListObjectsV2ExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + + if (input.ObjectAttributes?.length) { + const captured = { xml: '' }; + + this.middlewareStack.add( + overrideObjectAttributesHeaderMiddleware( + 'x-amz-optional-object-attributes', input.ObjectAttributes), { + step: 'build', + name: 'overrideObjectAttributesHeader', + }); + + this.middlewareStack.add(captureResponseBodyMiddleware(captured), { + step: 'deserialize', + name: 'captureResponseBody', + priority: 'low', + }); + + this.middlewareStack.add(parseListObjectsUserMetadataMiddleware(captured), { + step: 'deserialize', + name: 'parseUserMetadata', + priority: 'high', + }); + } + } +} diff --git a/src/commands/s3Extended/utils.ts b/src/commands/s3Extended/utils.ts new file mode 100644 index 00000000..820139a9 --- /dev/null +++ b/src/commands/s3Extended/utils.ts @@ -0,0 +1,87 @@ +import { Readable } from 'stream'; +import { streamCollector } from '@smithy/node-http-handler'; +import { XMLParser } from 'fast-xml-parser'; + +export const USER_METADATA_PREFIX = 'x-amz-meta-'; + + +export function extendCommandWithExtraParametersMiddleware(query: string) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const request = args.request as any; + if (request.query) { + request.query.search = query; + } else { + request.query = { search: query }; + } + return next(args); + }; +} + + +export function overrideObjectAttributesHeaderMiddleware(headerName: string, attributes: string[]) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + const request = args.request; + request.headers[headerName] = attributes.join(','); + return next(args); + }; +} + +export function captureResponseBodyMiddleware(captured: { xml: string }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + const { response } = await next(args); + + if (response?.body) { + const collected = await streamCollector(response.body); + const buffer = Buffer.from(collected); + // eslint-disable-next-line no-param-reassign + captured.xml = buffer.toString('utf-8'); + // Re-create the body stream so the SDK deserializer can still consume it + response.body = Readable.from([buffer]); + } + + return { response }; + }; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function extractUserMetadata(source: Record, target: Record): void { + for (const [key, value] of Object.entries(source)) { + if (key.startsWith(USER_METADATA_PREFIX)) { + // eslint-disable-next-line no-param-reassign + target[key] = String(value); + } + } +} + +export function parseUserMetadataMiddleware(captured: { xml: string }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + const result = await next(args); + const parsed = new XMLParser().parse(captured.xml); + const response = parsed?.GetObjectAttributesResponse; + if (response) { + extractUserMetadata(response, result.output); + } + return result; + }; +} + + +export function parseListObjectsUserMetadataMiddleware(captured: { xml: string }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (next: any) => async (args: any) => { + const result = await next(args); + const parsed = new XMLParser().parse(captured.xml); + const xmlContents = parsed?.ListBucketResult?.Contents; + if (result.output.Contents && xmlContents) { + for (let i = 0; i < result.output.Contents.length; i++) { + extractUserMetadata(xmlContents[i], result.output.Contents[i]); + } + } + return result; + }; +} diff --git a/src/index.ts b/src/index.ts index 85656164..d35a8b09 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ export * from './clients/backbeatRoutes'; export * from './clients/bucketQuota'; export * from './clients/proxyBackbeatApis'; -export * from './clients/s3Extended'; +export * from './commands/s3Extended'; export { CloudserverClient, CloudserverClientConfig } from './clients/cloudserver'; export * from './utils'; diff --git a/tests/commands/s3Extended/getObjectAttributes.test.ts b/tests/commands/s3Extended/getObjectAttributes.test.ts new file mode 100644 index 00000000..3d6a077a --- /dev/null +++ b/tests/commands/s3Extended/getObjectAttributes.test.ts @@ -0,0 +1,203 @@ +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from '../../testSetup'; +import { describeForMongoBackend } from '../../testHelpers'; +import assert from 'assert'; +import { + GetObjectAttributesExtendedCommand, + GetObjectAttributesExtendedOutput, +} from '../../../src/commands/s3Extended'; + +describeForMongoBackend('GetObjectAttributesExtended', () => { + let s3client: S3Client; + const metadataKey = `${testConfig.objectKey}-with-meta`; + const metadataBody = 'data-with-metadata'; + const metadataETag = '0235b027419caf7c0e4b0840f7ec21a6'; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + }); + + beforeAll(async () => { + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + Body: metadataBody, + Metadata: { + foo: 'bar', + bar: 'baz', + baz: 'qux', + }, + })); + }); + + it('should get ETag only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + }); + + it('should get ObjectSize only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ObjectSize'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ObjectSize, 11); + }); + + it('should get StorageClass only', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['StorageClass'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.StorageClass, 'STANDARD'); + }); + + it('should get all standard attributes at once', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag', 'ObjectParts', 'StorageClass', 'ObjectSize'], + })); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + assert.strictEqual(result.ObjectSize, 11); + assert.strictEqual(result.StorageClass, 'STANDARD'); + }); + + it('should get a single user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + }); + + it('should get multiple user metadata keys', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-bar', 'x-amz-meta-baz'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get all user metadata with wildcard', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-*'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get wildcard combined with a specific user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['x-amz-meta-*', 'x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should handle non-existing user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['x-amz-meta-nonexistent'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result['x-amz-meta-nonexistent'], undefined); + }); + + it('should get standard attribute combined with non-existing user metadata', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + ObjectAttributes: ['ETag', 'x-amz-meta-nonexistent'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, '8c68b1ec59642e3994c995eccfee553b'); + assert.strictEqual(result['x-amz-meta-nonexistent'], undefined); + }); + + it('should get ETag combined with a single user metadata key', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['ETag', 'x-amz-meta-foo'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + }); + + it('should get all standard attributes combined with all user metadata', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: [ + 'ETag', + 'ObjectParts', + 'StorageClass', + 'ObjectSize', + 'x-amz-meta-foo', + 'x-amz-meta-bar', + 'x-amz-meta-baz', + ], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result.ObjectSize, 18); + assert.strictEqual(result.StorageClass, 'STANDARD'); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); + + it('should get standard attributes combined with user metadata wildcard', async () => { + const result = await s3client.send(new GetObjectAttributesExtendedCommand({ + Bucket: testConfig.bucketName, + Key: metadataKey, + ObjectAttributes: ['ETag', 'ObjectSize', 'x-amz-meta-*'], + })) as GetObjectAttributesExtendedOutput; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.ETag, metadataETag); + assert.strictEqual(result.ObjectSize, 18); + assert.strictEqual(result['x-amz-meta-foo'], 'bar'); + assert.strictEqual(result['x-amz-meta-bar'], 'baz'); + assert.strictEqual(result['x-amz-meta-baz'], 'qux'); + }); +}); diff --git a/tests/commands/s3Extended/listObjectVersions.test.ts b/tests/commands/s3Extended/listObjectVersions.test.ts new file mode 100644 index 00000000..1d037251 --- /dev/null +++ b/tests/commands/s3Extended/listObjectVersions.test.ts @@ -0,0 +1,64 @@ +import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from '../../testSetup'; +import { describeForMongoBackend } from '../../testHelpers'; +import assert from 'assert'; +import { ListObjectVersionsExtendedCommand } from '../../../src/commands/s3Extended'; + +describeForMongoBackend('ListObjectVersionsExtended', () => { + let s3client: S3Client; + const key2ndObject = `${testConfig.objectKey}2nd`; + const body2ndObject = `${testConfig.objectData}2nd`; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + }); + + beforeAll(async () => { + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); + }); + + it('should test ListObjectVersionsExtended', async () => { + const getCommand1 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 100, + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Versions?.length, 2); + assert.strictEqual(getData1.Versions[0].IsLatest, true); + + // Delete one object to create a DeleteMarker + const deleteCommand = new DeleteObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + }); + await s3client.send(deleteCommand); + + const getCommand2 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 100, + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.DeleteMarkers?.[0].Key, key2ndObject); + assert.ok(getData2.DeleteMarkers?.[0].VersionId); + + assert.ok(getData2.Versions); + const firstVersion = getData2.Versions[0]; + const getCommand3 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + KeyMarker: firstVersion.Key, + VersionIdMarker: firstVersion.VersionId, + MaxKeys: 100, + }); + const getData3 = await s3client.send(getCommand3); + assert.notStrictEqual(getData3.Versions?.[0]?.Key, firstVersion.Key); + }); +}); diff --git a/tests/commands/s3Extended/listObjects.test.ts b/tests/commands/s3Extended/listObjects.test.ts new file mode 100644 index 00000000..50aa5489 --- /dev/null +++ b/tests/commands/s3Extended/listObjects.test.ts @@ -0,0 +1,71 @@ +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from '../../testSetup'; +import { describeForMongoBackend } from '../../testHelpers'; +import assert from 'assert'; +import { ListObjectsExtendedCommand } from '../../../src/commands/s3Extended'; + +describeForMongoBackend('ListObjectsExtended', () => { + let s3client: S3Client; + const key2ndObject = `${testConfig.objectKey}2nd`; + const body2ndObject = `${testConfig.objectData}2nd`; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + }); + + beforeAll(async () => { + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); + }); + + it('should test ListObjectsExtended', async () => { + const getCommand1= new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + + const maxKey = 1; + const getCommand2= new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: maxKey + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, maxKey); + assert.strictEqual(getData2.IsTruncated, true); + + const getCommand3 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length > ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData3 = await s3client.send(getCommand3); + assert.strictEqual(getData3.Contents?.length, 1); + assert.strictEqual(getData3.Contents[0].Key, key2ndObject); + + const getCommand4 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `key = ${key2ndObject}`, + MaxKeys: 5 + }); + const getData4 = await s3client.send(getCommand4); + assert.strictEqual(getData4.Contents?.length, 1); + assert.strictEqual(getData4.Contents[0].Key, key2ndObject); + + const getCommand5 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'key = iDontExists', + MaxKeys: 5 + }); + const getData5 = await s3client.send(getCommand5); + assert.strictEqual(getData5.Contents, undefined); + }); +}); diff --git a/tests/commands/s3Extended/listObjectsV2.test.ts b/tests/commands/s3Extended/listObjectsV2.test.ts new file mode 100644 index 00000000..d952c326 --- /dev/null +++ b/tests/commands/s3Extended/listObjectsV2.test.ts @@ -0,0 +1,157 @@ +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from '../../testSetup'; +import { describeForMongoBackend } from '../../testHelpers'; +import assert from 'assert'; +import { + ListObjectsV2ExtendedCommand, + ListObjectsV2ExtendedOutput, +} from '../../../src/commands/s3Extended'; + +describeForMongoBackend('ListObjectsV2Extended', () => { + let s3client: S3Client; + const key2ndObject = `${testConfig.objectKey}2nd`; + const body2ndObject = `${testConfig.objectData}2nd`; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + }); + + beforeAll(async () => { + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); + }); + + it('should test ListObjectsV2Extended', async () => { + const getCommand1= new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5, + FetchOwner: true, + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + assert.strictEqual(getData1.KeyCount, 2); + assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart'); + + const getCommand2 = new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + Query: 'content-length >= 0', + MaxKeys: 5, + StartAfter: testConfig.objectKey, // Skip first object + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, 1); + assert.strictEqual(getData2.Contents[0].Key, key2ndObject); + }); + + describe.skip('ListObjectsV2 with ObjectAttributes', () => { + const metaKey1 = `${testConfig.objectKey}-listv2-meta1`; + const metaKey2 = `${testConfig.objectKey}-listv2-meta2`; + + beforeAll(async () => { + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metaKey1, + Body: 'data1', + Metadata: { foo: 'bar', baz: 'qux' }, + })); + await s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: metaKey2, + Body: 'data2', + Metadata: { foo: 'hello' }, + })); + }); + + it('should list objects with a single user metadata key', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + }); + + it('should list objects with multiple user metadata keys', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-baz'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj1['x-amz-meta-baz'], 'qux'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + assert.strictEqual(obj2['x-amz-meta-baz'], undefined); + }); + + it('should list objects with wildcard user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-*'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + const obj2 = result.Contents!.find(ccontent => ccontent.Key === metaKey2)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.strictEqual(obj1['x-amz-meta-baz'], 'qux'); + assert.strictEqual(obj2['x-amz-meta-foo'], 'hello'); + }); + + it('should list objects with non-existing user metadata key', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['x-amz-meta-nonexistent'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-nonexistent'], undefined); + }); + + it('should list objects with RestoreStatus combined with user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['RestoreStatus', 'x-amz-meta-foo'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(obj1['x-amz-meta-foo'], 'bar'); + assert.deepStrictEqual(obj1.RestoreStatus, { IsRestoreInProgress: false }); + }); + + it('should list objects with RestoreStatus combined with non-existing user metadata', async () => { + const result = await s3client.send(new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + ObjectAttributes: ['RestoreStatus', 'x-amz-meta-nonexistent'], + })) as ListObjectsV2ExtendedOutput; + + const obj1 = result.Contents!.find(content => content.Key === metaKey1)!; + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual(result.Contents?.length, 2); + assert.strictEqual(obj1['x-amz-meta-nonexistent'], undefined); + assert.deepStrictEqual(obj1.RestoreStatus, { IsRestoreInProgress: false }); + }); + }); +}); diff --git a/tests/testS3ExtendedApis.test.ts b/tests/testS3ExtendedApis.test.ts deleted file mode 100644 index 0e43b52e..00000000 --- a/tests/testS3ExtendedApis.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; -import { createTestClient, testConfig } from './testSetup'; -import { describeForMongoBackend } from './testHelpers'; -import assert from 'assert'; -import { - ListObjectsExtendedCommand, - ListObjectsV2ExtendedCommand, - ListObjectVersionsExtendedCommand, -} from '../src/clients/s3Extended'; - -describeForMongoBackend('S3 Extended API Tests', () => { - let s3client: S3Client; - const key2ndObject = `${testConfig.objectKey}2nd`; - const body2ndObject = `${testConfig.objectData}2nd`; - - beforeAll(async () => { - const testClients = createTestClient(); - s3client = testClients.s3client; - - const putObjectCommand = new PutObjectCommand({ - Bucket: testConfig.bucketName, - Key: key2ndObject, - Body: body2ndObject, - }); - await s3client.send(putObjectCommand); - }); - - it('should test ListObjectsExtended', async () => { - const getCommand1= new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: 5 - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Contents?.length, 2); - - const maxKey = 1; - const getCommand2= new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: maxKey - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.Contents?.length, maxKey); - assert.strictEqual(getData2.IsTruncated, true); - - const getCommand3 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length > ${testConfig.objectData.length}`, - MaxKeys: 5 - }); - const getData3 = await s3client.send(getCommand3); - assert.strictEqual(getData3.Contents?.length, 1); - assert.strictEqual(getData3.Contents[0].Key, key2ndObject); - - const getCommand4 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `key = ${key2ndObject}`, - MaxKeys: 5 - }); - const getData4 = await s3client.send(getCommand4); - assert.strictEqual(getData4.Contents?.length, 1); - assert.strictEqual(getData4.Contents[0].Key, key2ndObject); - - const getCommand5 = new ListObjectsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `key = iDontExists`, - MaxKeys: 5 - }); - const getData5 = await s3client.send(getCommand5); - assert.strictEqual(getData5.Contents, undefined); - }); - - it('should test ListObjectsV2Extended', async () => { - const getCommand1= new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= ${testConfig.objectData.length}`, - MaxKeys: 5, - FetchOwner: true, - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Contents?.length, 2); - assert.strictEqual(getData1.KeyCount, 2); - assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart'); - - const getCommand2 = new ListObjectsV2ExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= 0`, - MaxKeys: 5, - StartAfter: testConfig.objectKey, // Skip first object - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.Contents?.length, 1); - assert.strictEqual(getData2.Contents[0].Key, key2ndObject); - }); - - it('should test ListObjectVersionsExtended', async () => { - const getCommand1 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= 0`, - MaxKeys: 100, - }); - const getData1 = await s3client.send(getCommand1); - assert.strictEqual(getData1.Versions?.length, 2); - assert.strictEqual(getData1.Versions[0].IsLatest, true); - - // Delete one object to create a DeleteMarker - const deleteCommand = new DeleteObjectCommand({ - Bucket: testConfig.bucketName, - Key: key2ndObject, - }); - await s3client.send(deleteCommand); - - const getCommand2 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= 0`, - MaxKeys: 100, - }); - const getData2 = await s3client.send(getCommand2); - assert.strictEqual(getData2.DeleteMarkers?.[0].Key, key2ndObject); - assert.ok(getData2.DeleteMarkers?.[0].VersionId); - - assert.ok(getData2.Versions); - const firstVersion = getData2.Versions[0]; - const getCommand3 = new ListObjectVersionsExtendedCommand({ - Bucket: testConfig.bucketName, - Query: `content-length >= 0`, - KeyMarker: firstVersion.Key, - VersionIdMarker: firstVersion.VersionId, - MaxKeys: 100, - }); - const getData3 = await s3client.send(getCommand3); - assert.notStrictEqual(getData3.Versions?.[0]?.Key, firstVersion.Key); - }); -});