diff --git a/api/services/filesystem/s3.py b/api/services/filesystem/s3.py index 1cbc7ffd..e6b99fa3 100644 --- a/api/services/filesystem/s3.py +++ b/api/services/filesystem/s3.py @@ -4,12 +4,6 @@ import aioboto3 from botocore.config import Config from botocore.exceptions import ClientError -from api.constants import ( - S3_ADDRESSING_STYLE, - S3_ENDPOINT_URL, - S3_SIGNATURE_VERSION, -) - from .base import BaseFileSystem @@ -30,27 +24,15 @@ class S3FileSystem(BaseFileSystem): bucket_name: Name of the S3 bucket region_name: AWS region name endpoint_url: Optional custom S3 endpoint (e.g. for MinIO/rustfs). - Defaults to ``S3_ENDPOINT_URL`` env var; ``None`` uses AWS. + ``None`` uses AWS's default endpoint resolution. signature_version: Optional botocore signature version (e.g. - ``"s3v4"``). Defaults to ``S3_SIGNATURE_VERSION`` env var; - ``None`` keeps botocore's default signing behavior. + ``"s3v4"``). ``None`` keeps botocore's default signing behavior. addressing_style: Optional S3 addressing style (``"path"`` / - ``"virtual"`` / ``"auto"``). Defaults to ``S3_ADDRESSING_STYLE`` - env var; ``None`` keeps botocore's default. + ``"virtual"`` / ``"auto"``). ``None`` keeps botocore's default. """ self.bucket_name = bucket_name self.region_name = region_name - self.endpoint_url = ( - endpoint_url if endpoint_url is not None else S3_ENDPOINT_URL - ) - signature_version = ( - signature_version - if signature_version is not None - else S3_SIGNATURE_VERSION - ) - addressing_style = ( - addressing_style if addressing_style is not None else S3_ADDRESSING_STYLE - ) + self.endpoint_url = endpoint_url self.session = aioboto3.Session() # Build a botocore Config only when an override is requested so that the @@ -77,9 +59,7 @@ class S3FileSystem(BaseFileSystem): async def acreate_file(self, file_path: str, content: BinaryIO) -> bool: try: - async with self.session.client( - "s3", **self._client_kwargs() - ) as s3_client: + async with self.session.client("s3", **self._client_kwargs()) as s3_client: await s3_client.put_object( Bucket=self.bucket_name, Key=file_path, Body=await content.read() ) @@ -89,9 +69,7 @@ class S3FileSystem(BaseFileSystem): async def aupload_file(self, local_path: str, destination_path: str) -> bool: try: - async with self.session.client( - "s3", **self._client_kwargs() - ) as s3_client: + async with self.session.client("s3", **self._client_kwargs()) as s3_client: await s3_client.upload_file( local_path, self.bucket_name, destination_path ) @@ -114,9 +92,7 @@ class S3FileSystem(BaseFileSystem): disposition on the response. """ try: - async with self.session.client( - "s3", **self._client_kwargs() - ) as s3_client: + async with self.session.client("s3", **self._client_kwargs()) as s3_client: params = {"Bucket": self.bucket_name, "Key": file_path} # Make artifacts viewable inline in the browser when requested @@ -155,9 +131,7 @@ class S3FileSystem(BaseFileSystem): async def aget_file_metadata(self, file_path: str) -> Optional[Dict[str, Any]]: """Get S3 object metadata.""" try: - async with self.session.client( - "s3", **self._client_kwargs() - ) as s3_client: + async with self.session.client("s3", **self._client_kwargs()) as s3_client: response = await s3_client.head_object( Bucket=self.bucket_name, Key=file_path ) @@ -181,9 +155,7 @@ class S3FileSystem(BaseFileSystem): ) -> Optional[str]: """Generate a presigned PUT URL for direct file upload.""" try: - async with self.session.client( - "s3", **self._client_kwargs() - ) as s3_client: + async with self.session.client("s3", **self._client_kwargs()) as s3_client: url = await s3_client.generate_presigned_url( "put_object", Params={ @@ -200,9 +172,7 @@ class S3FileSystem(BaseFileSystem): async def adownload_file(self, source_path: str, local_path: str) -> bool: """Download a file from S3 to local path.""" try: - async with self.session.client( - "s3", **self._client_kwargs() - ) as s3_client: + async with self.session.client("s3", **self._client_kwargs()) as s3_client: await s3_client.download_file(self.bucket_name, source_path, local_path) return True except ClientError: @@ -211,9 +181,7 @@ class S3FileSystem(BaseFileSystem): async def acopy_file(self, source_path: str, destination_path: str) -> bool: """Copy a file within S3 (server-side copy).""" try: - async with self.session.client( - "s3", **self._client_kwargs() - ) as s3_client: + async with self.session.client("s3", **self._client_kwargs()) as s3_client: await s3_client.copy_object( Bucket=self.bucket_name, Key=destination_path, diff --git a/api/services/storage.py b/api/services/storage.py index b24310b6..81bb2014 100644 --- a/api/services/storage.py +++ b/api/services/storage.py @@ -9,8 +9,11 @@ from api.constants import ( MINIO_PUBLIC_ENDPOINT, MINIO_SECRET_KEY, MINIO_SECURE, + S3_ADDRESSING_STYLE, S3_BUCKET, + S3_ENDPOINT_URL, S3_REGION, + S3_SIGNATURE_VERSION, ) from api.enums import Environment, StorageBackend @@ -57,7 +60,13 @@ def get_storage_for_backend(backend: str) -> BaseFileSystem: logger.info( f"Initializing {backend} storage with bucket '{bucket}' in region '{region}'" ) - return S3FileSystem(bucket, region) + return S3FileSystem( + bucket_name=bucket, + region_name=region, + endpoint_url=S3_ENDPOINT_URL, + signature_version=S3_SIGNATURE_VERSION, + addressing_style=S3_ADDRESSING_STYLE, + ) # Future backend implementations can be added here: # elif backend == StorageBackend.GCS: # Code 3 diff --git a/docker-compose.yaml b/docker-compose.yaml index 813351a4..352c4356 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -145,17 +145,27 @@ services: # Redis configuration (using containerized redis) REDIS_URL: "redis://:${REDIS_PASSWORD:-redissecret}@redis:6379" - # Storage configuration - using local MinIO - ENABLE_AWS_S3: "false" + # Storage configuration - bundled MinIO by default. Set ENABLE_AWS_S3=true + # in .env to make the API use AWS S3 or another S3-compatible server. + ENABLE_AWS_S3: "${ENABLE_AWS_S3:-false}" - # To use AWS S3 or any S3-compatible server (MinIO, rustfs, Ceph, ...) - # instead of the bundled MinIO, set ENABLE_AWS_S3 to "true" and provide: - # S3_BUCKET, S3_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY - # For a non-AWS S3-compatible server, also set: - # S3_ENDPOINT_URL e.g. https://s3.example.com - # S3_SIGNATURE_VERSION set "s3v4" if the server requires SigV4 (e.g. rustfs) - # S3_ADDRESSING_STYLE set "path" if the server / TLS cert requires path-style + # S3 backend configuration. Compose's .env file is used for interpolation, + # but those values are not automatically injected into containers, so pass + # the S3 settings through explicitly. + AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY:-}" + AWS_SESSION_TOKEN: "${AWS_SESSION_TOKEN:-}" + S3_BUCKET: "${S3_BUCKET:-}" + S3_REGION: "${S3_REGION:-us-east-1}" + + # For a non-AWS S3-compatible server, also set these in Compose's .env + # S3_ENDPOINT_URL e.g. https://s3.example.com + # S3_SIGNATURE_VERSION set "s3v4" if the server requires SigV4 (e.g. rustfs) + # S3_ADDRESSING_STYLE set "path" if the server / TLS cert requires path-style # The S3 backend issues real presigned URLs, so the bucket can stay private. + S3_ENDPOINT_URL: "${S3_ENDPOINT_URL:-}" + S3_SIGNATURE_VERSION: "${S3_SIGNATURE_VERSION:-}" + S3_ADDRESSING_STYLE: "${S3_ADDRESSING_STYLE:-}" # MinIO MINIO_ENDPOINT: "minio:9000"