From d7abb53400e09cd9de268cf75bf8e50b5c5e83e5 Mon Sep 17 00:00:00 2001 From: Adrian Flores Cortes Date: Mon, 26 Jan 2026 20:12:34 -0600 Subject: [PATCH] feat(storage): Add S3/R2 storage service with multipart upload (ST4.3.2) Service: StorageService Features: - Simple upload (<5GB) - Multipart upload (>100MB recommended) - initMultipartUpload() - uploadPart() - completeMultipartUpload() - abortMultipartUpload() - Presigned URLs (client-side upload/download) - Object operations (get, delete, copy, list, metadata) - URL generation (S3/R2/CDN) - Helper: generateKey() Supports: - AWS S3 - Cloudflare R2 (S3-compatible API) - CloudFront/Cloudflare CDN URLs Uses AWS SDK v3: - @aws-sdk/client-s3 - @aws-sdk/s3-request-presigner Env vars needed: - STORAGE_PROVIDER (s3 | r2) - STORAGE_BUCKET - STORAGE_REGION - STORAGE_ACCESS_KEY_ID - STORAGE_SECRET_ACCESS_KEY - STORAGE_ENDPOINT (for R2) - STORAGE_CDN_URL (optional) Blocker: BLOCKER-003 (ST4.3 Video Upload Backend) Epic: OQI-002 Co-Authored-By: Claude Opus 4.5 --- src/shared/services/storage.service.ts | 451 +++++++++++++++++++++++++ 1 file changed, 451 insertions(+) create mode 100644 src/shared/services/storage.service.ts diff --git a/src/shared/services/storage.service.ts b/src/shared/services/storage.service.ts new file mode 100644 index 0000000..63378e5 --- /dev/null +++ b/src/shared/services/storage.service.ts @@ -0,0 +1,451 @@ +// ============================================================================ +// Trading Platform - Storage Service (S3/R2) +// ============================================================================ +// Supports: AWS S3, Cloudflare R2 (S3-compatible API) +// Blocker: BLOCKER-003 (ST4.3) +// ============================================================================ + +import { + S3Client, + PutObjectCommand, + GetObjectCommand, + DeleteObjectCommand, + CopyObjectCommand, + ListObjectsV2Command, + HeadObjectCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, + AbortMultipartUploadCommand, + type CompletedPart, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { config } from '../../config'; +import { logger } from '../utils/logger'; +import { Readable } from 'stream'; + +// ============================================================================ +// Types +// ============================================================================ + +export interface StorageConfig { + provider: 's3' | 'r2'; + bucket: string; + region?: string; + endpoint?: string; // For R2 + accessKeyId: string; + secretAccessKey: string; + cdnUrl?: string; // CloudFront/Cloudflare CDN URL +} + +export interface UploadOptions { + key: string; + body: Buffer | Readable | string; + contentType?: string; + metadata?: Record; + acl?: 'private' | 'public-read'; +} + +export interface MultipartUploadInitResult { + uploadId: string; + key: string; +} + +export interface PresignedUrlOptions { + key: string; + expiresIn?: number; // seconds + contentType?: string; +} + +export interface ObjectMetadata { + key: string; + size: number; + lastModified: Date; + contentType?: string; + etag?: string; + metadata?: Record; +} + +// ============================================================================ +// Storage Service Class +// ============================================================================ + +export class StorageService { + private client: S3Client; + private config: StorageConfig; + + constructor(config: StorageConfig) { + this.config = config; + + // Initialize S3 client + const clientConfig: any = { + region: config.region || 'auto', + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }; + + // For Cloudflare R2, use custom endpoint + if (config.provider === 'r2' && config.endpoint) { + clientConfig.endpoint = config.endpoint; + } + + this.client = new S3Client(clientConfig); + + logger.info(`Storage service initialized`, { + provider: config.provider, + bucket: config.bucket, + hasEndpoint: !!config.endpoint, + }); + } + + // ============================================================================ + // Simple Upload (< 5GB) + // ============================================================================ + + async upload(options: UploadOptions): Promise<{ key: string; url: string }> { + const { key, body, contentType, metadata, acl = 'private' } = options; + + try { + const command = new PutObjectCommand({ + Bucket: this.config.bucket, + Key: key, + Body: body, + ContentType: contentType, + Metadata: metadata, + ACL: acl, + }); + + await this.client.send(command); + + const url = this.getPublicUrl(key); + + logger.info(`Uploaded object to storage`, { key, provider: this.config.provider }); + + return { key, url }; + } catch (error) { + logger.error(`Failed to upload object`, { key, error }); + throw new Error(`Storage upload failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // ============================================================================ + // Multipart Upload (>= 5MB, recommended for >100MB) + // ============================================================================ + + async initMultipartUpload( + key: string, + contentType?: string, + metadata?: Record + ): Promise { + try { + const command = new CreateMultipartUploadCommand({ + Bucket: this.config.bucket, + Key: key, + ContentType: contentType, + Metadata: metadata, + }); + + const response = await this.client.send(command); + + if (!response.UploadId) { + throw new Error('Failed to get uploadId from multipart upload init'); + } + + logger.info(`Initialized multipart upload`, { key, uploadId: response.UploadId }); + + return { + uploadId: response.UploadId, + key, + }; + } catch (error) { + logger.error(`Failed to initialize multipart upload`, { key, error }); + throw new Error(`Multipart upload init failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async uploadPart( + key: string, + uploadId: string, + partNumber: number, + body: Buffer | Readable + ): Promise<{ etag: string; partNumber: number }> { + try { + const command = new UploadPartCommand({ + Bucket: this.config.bucket, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + Body: body, + }); + + const response = await this.client.send(command); + + if (!response.ETag) { + throw new Error('Failed to get ETag from part upload'); + } + + logger.debug(`Uploaded part ${partNumber}`, { key, uploadId }); + + return { + etag: response.ETag, + partNumber, + }; + } catch (error) { + logger.error(`Failed to upload part`, { key, uploadId, partNumber, error }); + throw new Error(`Part upload failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async completeMultipartUpload( + key: string, + uploadId: string, + parts: CompletedPart[] + ): Promise<{ key: string; url: string }> { + try { + const command = new CompleteMultipartUploadCommand({ + Bucket: this.config.bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts, + }, + }); + + await this.client.send(command); + + const url = this.getPublicUrl(key); + + logger.info(`Completed multipart upload`, { key, uploadId, partsCount: parts.length }); + + return { key, url }; + } catch (error) { + logger.error(`Failed to complete multipart upload`, { key, uploadId, error }); + throw new Error(`Multipart upload completion failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async abortMultipartUpload(key: string, uploadId: string): Promise { + try { + const command = new AbortMultipartUploadCommand({ + Bucket: this.config.bucket, + Key: key, + UploadId: uploadId, + }); + + await this.client.send(command); + + logger.info(`Aborted multipart upload`, { key, uploadId }); + } catch (error) { + logger.error(`Failed to abort multipart upload`, { key, uploadId, error }); + throw new Error(`Abort multipart upload failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // ============================================================================ + // Presigned URLs (Client uploads directly to S3/R2) + // ============================================================================ + + async getPresignedUploadUrl(options: PresignedUrlOptions): Promise { + const { key, expiresIn = 3600, contentType } = options; + + try { + const command = new PutObjectCommand({ + Bucket: this.config.bucket, + Key: key, + ContentType: contentType, + }); + + const url = await getSignedUrl(this.client, command, { expiresIn }); + + logger.debug(`Generated presigned upload URL`, { key, expiresIn }); + + return url; + } catch (error) { + logger.error(`Failed to generate presigned upload URL`, { key, error }); + throw new Error(`Presigned URL generation failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async getPresignedDownloadUrl(key: string, expiresIn: number = 3600): Promise { + try { + const command = new GetObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + + const url = await getSignedUrl(this.client, command, { expiresIn }); + + logger.debug(`Generated presigned download URL`, { key, expiresIn }); + + return url; + } catch (error) { + logger.error(`Failed to generate presigned download URL`, { key, error }); + throw new Error(`Presigned URL generation failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // ============================================================================ + // Object Operations + // ============================================================================ + + async getObject(key: string): Promise { + try { + const command = new GetObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + + const response = await this.client.send(command); + + if (!response.Body) { + throw new Error('Empty response body'); + } + + // Convert stream to buffer + const chunks: Uint8Array[] = []; + for await (const chunk of response.Body as any) { + chunks.push(chunk); + } + + return Buffer.concat(chunks); + } catch (error) { + logger.error(`Failed to get object`, { key, error }); + throw new Error(`Get object failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async deleteObject(key: string): Promise { + try { + const command = new DeleteObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + + await this.client.send(command); + + logger.info(`Deleted object`, { key }); + } catch (error) { + logger.error(`Failed to delete object`, { key, error }); + throw new Error(`Delete object failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async copyObject(sourceKey: string, destinationKey: string): Promise { + try { + const command = new CopyObjectCommand({ + Bucket: this.config.bucket, + CopySource: `${this.config.bucket}/${sourceKey}`, + Key: destinationKey, + }); + + await this.client.send(command); + + logger.info(`Copied object`, { sourceKey, destinationKey }); + } catch (error) { + logger.error(`Failed to copy object`, { sourceKey, destinationKey, error }); + throw new Error(`Copy object failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async getObjectMetadata(key: string): Promise { + try { + const command = new HeadObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + + const response = await this.client.send(command); + + return { + key, + size: response.ContentLength || 0, + lastModified: response.LastModified || new Date(), + contentType: response.ContentType, + etag: response.ETag, + metadata: response.Metadata, + }; + } catch (error) { + logger.error(`Failed to get object metadata`, { key, error }); + throw new Error(`Get metadata failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async listObjects(prefix?: string, maxKeys: number = 1000): Promise { + try { + const command = new ListObjectsV2Command({ + Bucket: this.config.bucket, + Prefix: prefix, + MaxKeys: maxKeys, + }); + + const response = await this.client.send(command); + + if (!response.Contents) { + return []; + } + + return response.Contents.map((obj) => ({ + key: obj.Key || '', + size: obj.Size || 0, + lastModified: obj.LastModified || new Date(), + etag: obj.ETag, + })); + } catch (error) { + logger.error(`Failed to list objects`, { prefix, error }); + throw new Error(`List objects failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // ============================================================================ + // URL Generation + // ============================================================================ + + getPublicUrl(key: string): string { + if (this.config.cdnUrl) { + return `${this.config.cdnUrl}/${key}`; + } + + if (this.config.provider === 'r2' && this.config.endpoint) { + // Cloudflare R2 public URL + return `${this.config.endpoint}/${this.config.bucket}/${key}`; + } + + // AWS S3 public URL + return `https://${this.config.bucket}.s3.${this.config.region}.amazonaws.com/${key}`; + } + + // ============================================================================ + // Helper Methods + // ============================================================================ + + generateKey(prefix: string, filename: string): string { + const timestamp = Date.now(); + const random = Math.random().toString(36).substring(7); + const sanitized = filename.replace(/[^a-zA-Z0-9.-]/g, '_'); + return `${prefix}/${timestamp}-${random}-${sanitized}`; + } +} + +// ============================================================================ +// Factory Function +// ============================================================================ + +export function createStorageService(config: StorageConfig): StorageService { + return new StorageService(config); +} + +// ============================================================================ +// Default Instance (from environment) +// ============================================================================ + +export const storageService = createStorageService({ + provider: (config.storage?.provider || 's3') as 's3' | 'r2', + bucket: config.storage?.bucket || '', + region: config.storage?.region || 'us-east-1', + endpoint: config.storage?.endpoint, + accessKeyId: config.storage?.accessKeyId || '', + secretAccessKey: config.storage?.secretAccessKey || '', + cdnUrl: config.storage?.cdnUrl, +});