diff --git a/.gitignore b/.gitignore index 84cb567..8f64c61 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,6 @@ dist # Jest jest_html_reporters.html reports + +# tmp files +tmp diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 2b50ae2..fdb4e2b 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -42,6 +42,8 @@ export class FsStorageProvider implements IStorageProvider { const relativePath = paths[idx]!; this.logger.warn({ msg: 'Failed to delete file', path: join(storageTarget, relativePath), error: result.reason }); failedPaths.push(relativePath); + } else if (result.value !== undefined) { + failedPaths.push(result.value); } } diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 5311639..6c0da2e 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -5,7 +5,6 @@ import type { ConfigType } from '@common/config'; import type { IStorageProvider } from './iStorageProvider'; const S3_MAX_DELETE_BATCH = 1000; -const S3_ERROR_NO_SUCH_KEY = 'NoSuchKey'; interface S3Config { endpoint: string; diff --git a/tests/errorHandler.spec.ts b/tests/errorHandler.spec.ts index aac7a8b..76e49e2 100644 --- a/tests/errorHandler.spec.ts +++ b/tests/errorHandler.spec.ts @@ -1,14 +1,14 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { faker } from '@faker-js/faker'; -import jsLogger from '@map-colonies/js-logger'; +import { jsLogger } from '@map-colonies/js-logger'; import { toError, ErrorHandler, RecoverableError, UnrecoverableError, ValidationError } from '../src/cleaner/errors'; import type { ErrorContext } from '../src/cleaner/types'; describe('ErrorHandler', () => { let errorHandler: ErrorHandler; - beforeEach(() => { - errorHandler = new ErrorHandler(jsLogger({ enabled: false })); + beforeEach(async () => { + errorHandler = new ErrorHandler(await jsLogger({ enabled: false })); }); describe('handleError', () => { diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index 37ce5ab..2eeb1bc 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -3,6 +3,7 @@ import type { Logger } from '@map-colonies/js-logger'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import type { ConfigType } from '../../src/common/config'; import type { ITaskStrategy, StrategyFactory } from '../../src/cleaner/strategies'; +import type { IStorageProvider } from '../../src/cleaner/storageProviders'; import type { ErrorHandler } from '../../src/cleaner/errors'; import type { ErrorDecision, PollingPairConfig } from '../../src/cleaner/types'; import { TaskPoller } from '../../src/worker/taskPoller'; @@ -34,6 +35,7 @@ export function createMockQueueClient(): QueueClient { dequeue: vi.fn().mockResolvedValue(null), ack: vi.fn().mockResolvedValue(undefined), reject: vi.fn().mockResolvedValue(undefined), + updateProgress: vi.fn().mockResolvedValue(undefined), } as unknown as QueueClient; } @@ -61,6 +63,54 @@ export function createMockErrorHandler(defaultDecision: ErrorDecision = { should } as unknown as ErrorHandler; } +// ─── StorageProvider ───────────────────────────────────────────────────────── + +export function createMockStorageProvider(): IStorageProvider { + return { + delete: vi.fn().mockResolvedValue([]), + targetExists: vi.fn().mockResolvedValue(true), + }; +} + +// ─── Strategy Config (TilesDeletionStrategy) ───────────────────────────────── + +export const TILES_DELETION_CONFIG_DEFAULTS = { + batchSize: 100, + concurrency: 2, + failureSampleSize: 3, + s3Bucket: 'test-bucket', + fsBasePath: '/test/tiles', +} as const; + +export function createMockStrategyConfig(overrides: Record = {}): ConfigType { + const values: Record = { + 'strategies.tilesDeletion.batchSize': TILES_DELETION_CONFIG_DEFAULTS.batchSize, + 'strategies.tilesDeletion.concurrency': TILES_DELETION_CONFIG_DEFAULTS.concurrency, + 'strategies.tilesDeletion.failureSampleSize': TILES_DELETION_CONFIG_DEFAULTS.failureSampleSize, + 'strategies.tilesDeletion.s3Bucket': TILES_DELETION_CONFIG_DEFAULTS.s3Bucket, + 'strategies.tilesDeletion.fsBasePath': TILES_DELETION_CONFIG_DEFAULTS.fsBasePath, + ...overrides, + }; + return { get: vi.fn().mockImplementation((key: string) => values[key]) } as unknown as ConfigType; +} + +// ─── S3 Storage Config (S3StorageProvider) ─────────────────────────────────── + +export const S3_STORAGE_CONFIG_DEFAULTS = { + endpoint: 'http://localhost:9000', + accessKeyId: 'test-key', + secretAccessKey: 'test-secret', + sslEnabled: false, + forcePathStyle: true, + region: 'us-east-1', +} as const; + +export function createMockS3Config(): ConfigType { + return { + get: vi.fn().mockReturnValue({ ...S3_STORAGE_CONFIG_DEFAULTS }), + } as unknown as ConfigType; +} + // ─── TaskPoller factory ─────────────────────────────────────────────────────── /** diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts new file mode 100644 index 0000000..f824601 --- /dev/null +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -0,0 +1,166 @@ +import { stat, unlink, rmdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { Stats } from 'node:fs'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { FsStorageProvider } from '@src/cleaner/storageProviders/fsStorageProvider'; +import { createMockLogger } from '../helpers/mocks'; + +vi.mock('node:fs/promises', () => ({ + stat: vi.fn(), + unlink: vi.fn(), + rmdir: vi.fn(), +})); + +const BASE_PATH = '/tiles/test'; + +describe('FsStorageProvider', () => { + let provider: FsStorageProvider; + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(stat).mockResolvedValue({} as Stats); + vi.mocked(unlink).mockResolvedValue(undefined); + vi.mocked(rmdir).mockResolvedValue(undefined); + provider = new FsStorageProvider(createMockLogger()); + }); + + describe('targetExists', () => { + const RELATIVE_PATH = 'layer/v1'; + + it('should call stat with full target path', async () => { + const targetPath = join(BASE_PATH, RELATIVE_PATH); + const result = await provider.targetExists(BASE_PATH, RELATIVE_PATH); + + expect(result).toBe(true); + expect(stat).toHaveBeenCalledWith(targetPath); + }); + + it('should return false when path does not exist (ENOENT)', async () => { + vi.mocked(stat).mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })); + + const result = await provider.targetExists(BASE_PATH, RELATIVE_PATH); + + expect(result).toBe(false); + }); + + it('should throw an errors that are not ENOENT', async () => { + vi.mocked(stat).mockRejectedValue(Object.assign(new Error('EACCES'), { code: 'EACCES' })); + + await expect(provider.targetExists(BASE_PATH, RELATIVE_PATH)).rejects.toThrow('EACCES'); + }); + }); + + describe('delete', () => { + it('should return empty array for empty input', async () => { + const result = await provider.delete([], BASE_PATH); + expect(result).toEqual([]); + expect(unlink).not.toHaveBeenCalled(); + }); + + it('should call unlink with joined base path and relative path', async () => { + await provider.delete(['layer/v1/10/0/0.png'], BASE_PATH); + + expect(unlink).toHaveBeenCalledWith(join(BASE_PATH, 'layer/v1/10/0/0.png')); + }); + + it('should call unlink for every path', async () => { + const paths = ['tile/10/0/0.png', 'tile/10/0/1.png', 'tile/10/1/0.png']; + + await provider.delete(paths, BASE_PATH); + + expect(unlink).toHaveBeenCalledTimes(3); + for (const p of paths) { + expect(unlink).toHaveBeenCalledWith(join(BASE_PATH, p)); + } + }); + + it('should return empty array when all unlinks succeed', async () => { + const result = await provider.delete(['tile/10/0/0.png', 'tile/10/0/1.png'], BASE_PATH); + expect(result).toEqual([]); + }); + + it('should treat ENOENT as a failed deletion (included in failure report)', async () => { + const enoent = Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + vi.mocked(unlink).mockRejectedValue(enoent); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual(['tile/10/0/0.png']); + }); + + it('should return failed path for non-ENOENT errors', async () => { + const permError = Object.assign(new Error('EACCES'), { code: 'EACCES' }); + vi.mocked(unlink).mockRejectedValue(permError); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual(['tile/10/0/0.png']); + }); + + it('should handle mixed success, ENOENT and real errors', async () => { + const enoent = Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + const permError = Object.assign(new Error('EACCES'), { code: 'EACCES' }); + + vi.mocked(unlink) + .mockResolvedValueOnce(undefined) // success + .mockRejectedValueOnce(enoent) // ENOENT → failure + .mockRejectedValueOnce(permError); // real error → failure + + const paths = ['tile/10/0/0.png', 'tile/10/0/1.png', 'tile/10/0/2.png']; + const result = await provider.delete(paths, BASE_PATH); + + expect(result).toEqual(['tile/10/0/1.png', 'tile/10/0/2.png']); + }); + + it('should use relative path as key in failed paths (not the full absolute path)', async () => { + const permError = Object.assign(new Error('EACCES'), { code: 'EACCES' }); + vi.mocked(unlink).mockRejectedValue(permError); + + const relativePath = 'layer/v1/10/5/3.png'; + const result = await provider.delete([relativePath], BASE_PATH); + + expect(result).toEqual([relativePath]); + expect(result[0]).not.toContain(BASE_PATH); + }); + + describe('cleanupEmptyDirs', () => { + it('should attempt to rmdir the parent directory after deletion', async () => { + await provider.delete(['layer/v1/10/0/0.png'], BASE_PATH); + + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'layer/v1/10/0')); + }); + + it('should attempt to rmdir all ancestor directories bottom-up', async () => { + await provider.delete(['layer/v1/10/0/0.png'], BASE_PATH); + + // x dir → zoom dir → version dir → layer dir (deepest first) + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'layer/v1/10/0')); + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'layer/v1/10')); + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'layer/v1')); + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'layer')); + }); + + it('should deduplicate rmdir calls for shared parent directories', async () => { + // Both tiles share the same x-dir and zoom dir + await provider.delete(['tile/10/0/0.png', 'tile/10/0/1.png'], BASE_PATH); + + const rmdirCalls = vi.mocked(rmdir).mock.calls.map(([p]) => p); + const xDirCalls = rmdirCalls.filter((p) => p === join(BASE_PATH, 'tile/10/0')); + expect(xDirCalls).toHaveLength(1); + }); + + it('should silently ignore rmdir failures (ENOTEMPTY)', async () => { + const enotempty = Object.assign(new Error('ENOTEMPTY'), { code: 'ENOTEMPTY' }); + vi.mocked(rmdir).mockRejectedValue(enotempty); + + // Should not throw and should return correct failed paths + await expect(provider.delete(['tile/10/0/0.png'], BASE_PATH)).resolves.toEqual([]); + }); + + it('should not call rmdir when input is empty', async () => { + await provider.delete([], BASE_PATH); + expect(rmdir).not.toHaveBeenCalled(); + }); + }); + }); +}); diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts new file mode 100644 index 0000000..b9001cc --- /dev/null +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -0,0 +1,199 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { S3Client, DeleteObjectsCommand, ListObjectsV2Command, NoSuchBucket } from '@aws-sdk/client-s3'; +import { S3StorageProvider } from '@src/cleaner/storageProviders/s3StorageProvider'; +import { createMockLogger, createMockS3Config, S3_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; + +const mockSend = vi.fn(); + +vi.mock('@aws-sdk/client-s3', () => { + class NoSuchBucket extends Error { + public constructor() { + super('NoSuchBucket'); + this.name = 'NoSuchBucket'; + } + } + return { + S3Client: vi.fn(() => ({ send: mockSend })), + DeleteObjectsCommand: vi.fn((input: unknown) => input), + ListObjectsV2Command: vi.fn((input: unknown) => input), + NoSuchBucket, + }; +}); + +const BUCKET = 'test-bucket'; + +describe('S3StorageProvider', () => { + let provider: S3StorageProvider; + + beforeEach(() => { + vi.clearAllMocks(); + mockSend.mockResolvedValue({ Errors: [] }); + provider = new S3StorageProvider(createMockS3Config(), createMockLogger()); + }); + + describe('delete', () => { + it('should return empty array for empty input', async () => { + const result = await provider.delete([], BUCKET); + expect(result).toEqual([]); + expect(mockSend).not.toHaveBeenCalled(); + }); + + it('should send DeleteObjectsCommand with correct keys', async () => { + const paths = ['folder/a.txt', 'folder/b.txt']; + + await provider.delete(paths, BUCKET); + + expect(DeleteObjectsCommand).toHaveBeenCalledWith({ + Bucket: BUCKET, + Delete: { Objects: [{ Key: 'folder/a.txt' }, { Key: 'folder/b.txt' }] }, + }); + expect(mockSend).toHaveBeenCalledTimes(1); + }); + + it('should return empty array when all deletes succeed', async () => { + const paths = ['a.txt', 'b.txt']; + + const result = await provider.delete(paths, BUCKET); + + expect(result).toEqual([]); + }); + + it('should return failed paths from response.Errors', async () => { + mockSend.mockResolvedValue({ + Errors: [{ Key: 'a.txt', Code: 'AccessDenied', Message: 'Forbidden' }], + }); + const paths = ['a.txt', 'b.txt']; + + const result = await provider.delete(paths, BUCKET); + + expect(result).toEqual(['a.txt']); + }); + + it('should treat NoSuchKey as a failed deletion (included in failure report)', async () => { + mockSend.mockResolvedValue({ + Errors: [{ Key: 'missing.txt', Code: 'NoSuchKey', Message: 'Not Found' }], + }); + const paths = ['missing.txt']; + + const result = await provider.delete(paths, BUCKET); + + expect(result).toEqual(['missing.txt']); + }); + + it('should return all errors including NoSuchKey', async () => { + mockSend.mockResolvedValue({ + Errors: [ + { Key: 'a.txt', Code: 'NoSuchKey' }, + { Key: 'b.txt', Code: 'AccessDenied' }, + { Key: 'c.txt', Code: 'NoSuchKey' }, + { Key: 'd.txt', Code: 'InternalError' }, + ], + }); + + const result = await provider.delete(['a.txt', 'b.txt', 'c.txt', 'd.txt'], BUCKET); + + expect(result).toEqual(['a.txt', 'b.txt', 'c.txt', 'd.txt']); + }); + + it('should batch paths into chunks of 1000 (S3 limit)', async () => { + const paths = Array.from({ length: 1500 }, (_, i) => `object-${i}.txt`); + + await provider.delete(paths, BUCKET); + + expect(mockSend).toHaveBeenCalledTimes(2); + const firstCallInput = vi.mocked(DeleteObjectsCommand).mock.calls[0]![0] as { + Delete: { Objects: { Key: string }[] }; + }; + const secondCallInput = vi.mocked(DeleteObjectsCommand).mock.calls[1]![0] as { + Delete: { Objects: { Key: string }[] }; + }; + expect(firstCallInput.Delete.Objects).toHaveLength(1000); + expect(secondCallInput.Delete.Objects).toHaveLength(500); + }); + + it('should accumulate failures across multiple chunks', async () => { + const paths = Array.from({ length: 1500 }, (_, i) => `object-${i}.txt`); + mockSend + .mockResolvedValueOnce({ Errors: [{ Key: 'object-0.txt', Code: 'AccessDenied' }] }) + .mockResolvedValueOnce({ Errors: [{ Key: 'object-1000.txt', Code: 'AccessDenied' }] }); + + const result = await provider.delete(paths, BUCKET); + + expect(result).toEqual(['object-0.txt', 'object-1000.txt']); + }); + + it('should add entire chunk to failed paths when send throws', async () => { + mockSend.mockRejectedValue(new Error('Network error')); + const paths = ['a.txt', 'b.txt']; + + const result = await provider.delete(paths, BUCKET); + + expect(result).toEqual(paths); + }); + }); + + describe('targetExists', () => { + const PREFIX = 'some/prefix'; + + it('should list objects with bucket and relativePath prefix', async () => { + mockSend.mockResolvedValue({ KeyCount: 1 }); + + const result = await provider.targetExists(BUCKET, PREFIX); + + expect(result).toBe(true); + expect(ListObjectsV2Command).toHaveBeenCalledWith({ Bucket: BUCKET, Prefix: `${PREFIX}/`, MaxKeys: 1 }); + }); + + it('should return false when no objects exist under the prefix', async () => { + mockSend.mockResolvedValue({ KeyCount: 0 }); + + const result = await provider.targetExists(BUCKET, PREFIX); + + expect(result).toBe(false); + }); + + it('should return false when KeyCount is undefined', async () => { + mockSend.mockResolvedValue({}); + + const result = await provider.targetExists(BUCKET, PREFIX); + + expect(result).toBe(false); + }); + + it('should not double-add trailing slash when relativePath already ends with one', async () => { + mockSend.mockResolvedValue({ KeyCount: 1 }); + + await provider.targetExists(BUCKET, `${PREFIX}/`); + + expect(ListObjectsV2Command).toHaveBeenCalledWith({ Bucket: BUCKET, Prefix: `${PREFIX}/`, MaxKeys: 1 }); + }); + + it('should return false when bucket does not exist (NoSuchBucket)', async () => { + mockSend.mockRejectedValue(new NoSuchBucket({ message: 'Bucket not found', $metadata: { httpStatusCode: 404 } })); + + const result = await provider.targetExists(BUCKET, PREFIX); + + expect(result).toBe(false); + }); + + it('should re-throw errors that are not NoSuchBucket', async () => { + mockSend.mockRejectedValue(new Error('NetworkError')); + + await expect(provider.targetExists(BUCKET, PREFIX)).rejects.toThrow('NetworkError'); + }); + }); + + describe('constructor', () => { + it('should construct S3Client with config values', () => { + expect(S3Client).toHaveBeenCalledWith( + expect.objectContaining({ + endpoint: S3_STORAGE_CONFIG_DEFAULTS.endpoint, + forcePathStyle: S3_STORAGE_CONFIG_DEFAULTS.forcePathStyle, + region: S3_STORAGE_CONFIG_DEFAULTS.region, + tls: S3_STORAGE_CONFIG_DEFAULTS.sslEnabled, + }) + ); + }); + }); +}); diff --git a/tests/strategyFactory.spec.ts b/tests/strategyFactory.spec.ts index 0f613e8..2daa995 100644 --- a/tests/strategyFactory.spec.ts +++ b/tests/strategyFactory.spec.ts @@ -5,7 +5,7 @@ import type { Logger } from '@map-colonies/js-logger'; import { SERVICES } from '../src/common/constants'; import { StrategyFactory, TilesDeletionStrategy, type ITaskStrategy, type TaskContext } from '../src/cleaner/strategies'; import { StrategyNotFoundError } from '../src/cleaner/errors'; -import { createMockLogger } from './helpers/mocks'; +import { createMockLogger, createMockConfig, createMockQueueClient } from './helpers/mocks'; class MockStrategy implements ITaskStrategy { public validate(params: unknown): Record { @@ -24,8 +24,10 @@ describe('StrategyFactory', () => { beforeEach(() => { mockLogger = createMockLogger(); - // Register in global container (which StrategyFactory uses) container.register(SERVICES.LOGGER, { useValue: mockLogger }); + container.register(SERVICES.CONFIG, { useValue: createMockConfig() }); + container.register(SERVICES.STORAGE_PROVIDERS, { useValue: new Map() }); + container.register(SERVICES.QUEUE_CLIENT, { useValue: createMockQueueClient() }); strategyFactory = new StrategyFactory(mockLogger); }); diff --git a/tests/tilesDeletionStrategy.spec.ts b/tests/tilesDeletionStrategy.spec.ts index 6a18fae..b2d0c26 100644 --- a/tests/tilesDeletionStrategy.spec.ts +++ b/tests/tilesDeletionStrategy.spec.ts @@ -1,55 +1,258 @@ -import { describe, it, expect, beforeEach } from 'vitest'; -import { container } from 'tsyringe'; -import { SERVICES } from '@common/constants'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { type TilesDeletionParams, SourceType } from '@map-colonies/raster-shared'; +import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; +import { faker } from '@faker-js/faker'; import { TilesDeletionStrategy } from '@src/cleaner/strategies/tilesDeletionStrategy'; -import { ValidationError } from '@src/cleaner/errors'; -import { createMockLogger } from './helpers/mocks'; +import type { TaskContext } from '@src/cleaner/strategies/strategyFactory'; +import { ValidationError, RecoverableError, UnrecoverableError } from '@src/cleaner/errors'; +import type { IStorageProvider } from '@src/cleaner/storageProviders'; +import { createMockLogger, createMockStorageProvider, createMockStrategyConfig, TILES_DELETION_CONFIG_DEFAULTS } from './helpers/mocks'; + +const { s3Bucket: S3_BUCKET, fsBasePath: FS_BASE_PATH } = TILES_DELETION_CONFIG_DEFAULTS; + +const JOB_ID = faker.string.uuid(); +const TASK_ID = faker.string.uuid(); +const TASK_CONTEXT: TaskContext = { jobId: JOB_ID, taskId: TASK_ID, jobType: 'Ingestion_Update', taskType: 'tiles-deletion' }; + +const s3Params: TilesDeletionParams = { + sourceProvider: 'S3', + tilesPath: 'layer/v1', + fileExtension: 'png', + ranges: [{ zoom: 10, minX: 0, maxX: 1, minY: 0, maxY: 1 }], +}; + +const fsParams: TilesDeletionParams = { ...s3Params, sourceProvider: 'FS' }; + +// Builds an expected tile path under the standard s3Params tilesPath/fileExtension. +const tilePath = (z: number, x: number, y: number): string => `${s3Params.tilesPath}/${z}/${x}/${y}.${s3Params.fileExtension}`; describe('TilesDeletionStrategy', () => { let strategy: TilesDeletionStrategy; - const mockLogger = createMockLogger(); + let MockS3Provider: IStorageProvider; + let MockFsProvider: IStorageProvider; + let mockUpdateProgress: ReturnType; beforeEach(() => { - container.clearInstances(); - container.register(SERVICES.LOGGER, { useValue: mockLogger }); - strategy = container.resolve(TilesDeletionStrategy); + MockS3Provider = createMockStorageProvider(); + MockFsProvider = createMockStorageProvider(); + mockUpdateProgress = vi.fn().mockResolvedValue(undefined); + + const storageProviders = new Map([ + [SourceType.S3, MockS3Provider], + [SourceType.FS, MockFsProvider], + ]); + const queueClient = { updateProgress: mockUpdateProgress } as unknown as QueueClient; + + strategy = new TilesDeletionStrategy(createMockLogger(), createMockStrategyConfig(), storageProviders, queueClient, TASK_CONTEXT); }); describe('validate', () => { - it('should validate and return typed parameters when schema passes', () => { - const params = {}; // Empty object is valid for current TODO schema + it('should validate and return S3 params', () => { + expect(strategy.validate(s3Params)).toEqual(s3Params); + }); - const result = strategy.validate(params); + it('should validate and return FS params', () => { + expect(strategy.validate(fsParams)).toEqual(fsParams); + }); + + it('should accept JPEG file extension', () => { + const result = strategy.validate({ ...s3Params, fileExtension: 'jpeg' }); + expect(result).toMatchObject({ fileExtension: 'jpeg' }); + }); + + it('should accept multiple ranges', () => { + const ranges = [ + { zoom: 10, minX: 0, maxX: 5, minY: 0, maxY: 5 }, + { zoom: 11, minX: 0, maxX: 3, minY: 0, maxY: 3 }, + ]; + expect(strategy.validate({ ...s3Params, ranges })).toEqual({ ...s3Params, ranges }); + }); - expect(result).toEqual({}); + it('should throw ValidationError when sourceProvider is missing', () => { + expect(() => strategy.validate({ ...s3Params, sourceProvider: undefined })).toThrow(ValidationError); }); - it('should throw ValidationError when schema validation fails', () => { - const invalidParams = null; // null is not a valid object + it('should throw ValidationError for unsupported sourceProvider value', () => { + expect(() => strategy.validate({ ...s3Params, sourceProvider: 'GCS' })).toThrow(ValidationError); + }); - expect(() => strategy.validate(invalidParams)).toThrow(ValidationError); + it('should throw ValidationError for empty ranges array', () => { + expect(() => strategy.validate({ ...s3Params, ranges: [] })).toThrow(ValidationError); }); - it('should throw ValidationError with Zod error details', () => { - const invalidParams = 'not an object'; + it('should throw ValidationError for empty tilesPath', () => { + expect(() => strategy.validate({ ...s3Params, tilesPath: '' })).toThrow(ValidationError); + }); - try { - strategy.validate(invalidParams); - expect.fail('Should have thrown ValidationError'); - } catch (error) { - expect(error).toBeInstanceOf(ValidationError); - const validationError = error as ValidationError; - expect(validationError.validationDetails).toBeDefined(); - expect(Array.isArray(validationError.validationDetails)).toBe(true); - } + it('should throw ValidationError when params is null', () => { + expect(() => strategy.validate(null)).toThrow(ValidationError); }); }); describe('execute', () => { - it('should execute with valid typed parameters', async () => { - const validParams = {}; // Matches TilesDeletionParams type + describe('target validation', () => { + it('should throw UnrecoverableError when S3 storage target does not exist', async () => { + vi.mocked(MockS3Provider.targetExists).mockResolvedValue(false); + + await expect(strategy.execute(s3Params)).rejects.toThrow(UnrecoverableError); + expect(MockS3Provider.delete).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when FS storage target does not exist', async () => { + vi.mocked(MockFsProvider.targetExists).mockResolvedValue(false); + + await expect(strategy.execute(fsParams)).rejects.toThrow(UnrecoverableError); + expect(MockFsProvider.delete).not.toHaveBeenCalled(); + }); + + it('should check targetExists with S3 bucket and tilesPath as relativePath', async () => { + await strategy.execute(s3Params); + + expect(MockS3Provider.targetExists).toHaveBeenCalledWith(S3_BUCKET, s3Params.tilesPath); + }); + + it('should check targetExists with FS base path and tilesPath as relativePath', async () => { + await strategy.execute(fsParams); + + expect(MockFsProvider.targetExists).toHaveBeenCalledWith(FS_BASE_PATH, fsParams.tilesPath); + }); + }); + + describe('provider routing', () => { + it('should call S3 provider with s3Bucket as storage target', async () => { + await strategy.execute(s3Params); + + expect(MockS3Provider.delete).toHaveBeenCalledWith(expect.any(Array), S3_BUCKET); + expect(MockFsProvider.delete).not.toHaveBeenCalled(); + }); + + it('should call FS provider with fsBasePath as storage target', async () => { + await strategy.execute(fsParams); + + expect(MockFsProvider.delete).toHaveBeenCalledWith(expect.any(Array), FS_BASE_PATH); + expect(MockS3Provider.delete).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError for unknown provider', async () => { + const unknownParams = { ...s3Params, sourceProvider: 'UNKNOWN' } as unknown as TilesDeletionParams; + + await expect(strategy.execute(unknownParams)).rejects.toThrow(UnrecoverableError); + }); + }); + + describe('tile path generation', () => { + it('should generate paths in z/x/y order with correct format', async () => { + await strategy.execute(s3Params); + + // range: minX=0,maxX=1 minY=0,maxY=1 → 4 tiles, x iterates outer + expect(MockS3Provider.delete).toHaveBeenCalledWith( + [tilePath(10, 0, 0), tilePath(10, 0, 1), tilePath(10, 1, 0), tilePath(10, 1, 1)], + S3_BUCKET + ); + }); + + it('should use the specified file extension', async () => { + const params: TilesDeletionParams = { ...s3Params, fileExtension: 'jpeg' }; + + await strategy.execute(params); + + const [paths] = vi.mocked(MockS3Provider.delete).mock.calls[0]!; + expect(paths.every((p) => p.endsWith('.jpeg'))).toBe(true); + }); + + it('should concatenate tiles from multiple ranges', async () => { + const params: TilesDeletionParams = { + ...s3Params, + ranges: [ + { zoom: 5, minX: 0, maxX: 0, minY: 0, maxY: 0 }, + { zoom: 6, minX: 0, maxX: 0, minY: 0, maxY: 0 }, + ], + }; + + await strategy.execute(params); + + expect(MockS3Provider.delete).toHaveBeenCalledWith([tilePath(5, 0, 0), tilePath(6, 0, 0)], S3_BUCKET); + }); + + it('should offset x/y correctly when range does not start at 0', async () => { + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 7, minX: 3, maxX: 4, minY: 8, maxY: 9 }], + }; + + await strategy.execute(params); + + expect(MockS3Provider.delete).toHaveBeenCalledWith([tilePath(7, 3, 8), tilePath(7, 3, 9), tilePath(7, 4, 8), tilePath(7, 4, 9)], S3_BUCKET); + }); + }); + + describe('progress reporting', () => { + it('should call updateProgress with 100 when tiles fit in a single flush', async () => { + await strategy.execute(s3Params); + + expect(mockUpdateProgress).toHaveBeenCalledWith(JOB_ID, TASK_ID, 100); + }); + + it('should call updateProgress mid-stream and at 100 for large tile sets', async () => { + // batchSize=100, concurrency=2 → flush after 200 tiles, then final flush for remainder + // 14 * 15 = 210 tiles + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 13, minY: 0, maxY: 14 }], + }; + + await strategy.execute(params); + + expect(mockUpdateProgress).toHaveBeenCalledTimes(2); + expect(mockUpdateProgress).toHaveBeenLastCalledWith(JOB_ID, TASK_ID, 100); + }); + + it('should call updateProgress with the correct jobId and taskId', async () => { + await strategy.execute(s3Params); + + expect(mockUpdateProgress).toHaveBeenCalledWith(JOB_ID, TASK_ID, expect.any(Number)); + }); + + it('should not call updateProgress with 100 when the final flush has failures', async () => { + vi.mocked(MockS3Provider.delete).mockResolvedValue([tilePath(10, 0, 0)]); + + await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); + + expect(mockUpdateProgress).not.toHaveBeenCalledWith(JOB_ID, TASK_ID, 100); + }); + }); + + describe('failure handling', () => { + it('should throw RecoverableError when provider returns failed paths', async () => { + vi.mocked(MockS3Provider.delete).mockResolvedValue([tilePath(10, 0, 0)]); + + await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); + }); + + it('should include failed count in RecoverableError message', async () => { + vi.mocked(MockS3Provider.delete).mockResolvedValue([tilePath(10, 0, 0), tilePath(10, 0, 1)]); + + await expect(strategy.execute(s3Params)).rejects.toThrow(/2/); + }); + + it('should resolve successfully when provider returns no failed paths', async () => { + vi.mocked(MockS3Provider.delete).mockResolvedValue([]); + + await expect(strategy.execute(s3Params)).resolves.toBeUndefined(); + }); + + it('should throw RecoverableError when a batch rejects entirely (hard failure)', async () => { + vi.mocked(MockS3Provider.delete).mockRejectedValue(new Error('S3 connection lost')); + + await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); + }); + + it('should count all paths in a hard-rejected batch as failed', async () => { + // s3Params has 4 tiles (2×2); all 4 must surface in the error when the batch rejects + vi.mocked(MockS3Provider.delete).mockRejectedValue(new Error('S3 connection lost')); - await expect(strategy.execute(validParams)).resolves.toBeUndefined(); + await expect(strategy.execute(s3Params)).rejects.toThrow(/Failed to delete 4/); + }); }); }); }); diff --git a/vitest.config.mts b/vitest.config.mts index bb610c0..919635a 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -21,11 +21,15 @@ if (process.env.GITHUB_ACTIONS) { export default defineConfig({ resolve: { - alias: pathAlias, + alias: { + ...pathAlias, + '@map-colonies/raster-shared': path.resolve(__dirname, 'node_modules/@map-colonies/raster-shared/dist/index.js'), + }, }, test: { setupFiles: ['./tests/setup/vite.setup.ts'], include: ['tests/**/*.spec.ts'], + exclude: ['tests/**/*.integration.spec.ts'], environment: 'node', reporters, @@ -33,15 +37,32 @@ export default defineConfig({ enabled: true, reporter: ['text', 'html', 'json', 'json-summary'], include: ['src/**/*.ts'], - exclude: ['**/vendor/**', 'node_modules/**', 'src/index.ts', 'src/worker.ts'], + exclude: [ + '**/vendor/**', + 'node_modules/**', + // Application entry points + 'src/index.ts', + 'src/worker.ts', + // DI wiring and bootstrap — integration concerns, not unit concerns + 'src/containerConfig.ts', + 'src/common/dependencyRegistration.ts', + 'src/common/config.ts', + 'src/common/tracing.ts', + 'src/worker/workerBuilder.ts', + // Pure TypeScript interfaces — no executable code + 'src/cleaner/types.ts', + 'src/cleaner/strategies/taskStrategy.ts', + 'src/common/interfaces.ts', + 'src/cleaner/storageProviders/iStorageProvider.ts', + // Barrel re-export files + 'src/cleaner/storageProviders/index.ts', + ], reportOnFailure: true, thresholds: { - global: { - statements: 80, - branches: 80, - functions: 80, - lines: 80, - }, + statements: 80, + branches: 80, + functions: 80, + lines: 80, }, }, },