apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-06-28 08:49:42 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: pass s3 compose settings into storage

Browse source
This commit is contained in:
Abhishek Kumar 2026-06-24 18:28:04 +05:30
parent 1e2a276a61
commit 978fb9c262
3 changed files with 40 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

54
api/services/filesystem/s3.py
View file

@ -4,12 +4,6 @@ import aioboto3
from botocore.config import Config from botocore.config import Config
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
from api.constants import (
S3_ADDRESSING_STYLE,
S3_ENDPOINT_URL,
S3_SIGNATURE_VERSION,
)
from .base import BaseFileSystem from .base import BaseFileSystem
@ -30,27 +24,15 @@ class S3FileSystem(BaseFileSystem):
bucket_name: Name of the S3 bucket bucket_name: Name of the S3 bucket
region_name: AWS region name region_name: AWS region name
endpoint_url: Optional custom S3 endpoint (e.g. for MinIO/rustfs). endpoint_url: Optional custom S3 endpoint (e.g. for MinIO/rustfs).
Defaults to ``S3_ENDPOINT_URL`` env var; ``None`` uses AWS. ``None`` uses AWS's default endpoint resolution.
signature_version: Optional botocore signature version (e.g. signature_version: Optional botocore signature version (e.g.
``"s3v4"``). Defaults to ``S3_SIGNATURE_VERSION`` env var; ``"s3v4"``). ``None`` keeps botocore's default signing behavior.
``None`` keeps botocore's default signing behavior.
addressing_style: Optional S3 addressing style (``"path"`` / addressing_style: Optional S3 addressing style (``"path"`` /
``"virtual"`` / ``"auto"``). Defaults to ``S3_ADDRESSING_STYLE`` ``"virtual"`` / ``"auto"``). ``None`` keeps botocore's default.
env var; ``None`` keeps botocore's default.
""" """
self.bucket_name = bucket_name self.bucket_name = bucket_name
self.region_name = region_name self.region_name = region_name
self.endpoint_url = ( self.endpoint_url = endpoint_url
endpoint_url if endpoint_url is not None else S3_ENDPOINT_URL
)
signature_version = (
signature_version
if signature_version is not None
else S3_SIGNATURE_VERSION
)
addressing_style = (
addressing_style if addressing_style is not None else S3_ADDRESSING_STYLE
)
self.session = aioboto3.Session() self.session = aioboto3.Session()
# Build a botocore Config only when an override is requested so that the # Build a botocore Config only when an override is requested so that the
@ -77,9 +59,7 @@ class S3FileSystem(BaseFileSystem):
async def acreate_file(self, file_path: str, content: BinaryIO) -> bool: async def acreate_file(self, file_path: str, content: BinaryIO) -> bool:
try: try:
async with self.session.client( async with self.session.client("s3", **self._client_kwargs()) as s3_client:
"s3", **self._client_kwargs()
) as s3_client:
await s3_client.put_object( await s3_client.put_object(
Bucket=self.bucket_name, Key=file_path, Body=await content.read() Bucket=self.bucket_name, Key=file_path, Body=await content.read()
) )
@ -89,9 +69,7 @@ class S3FileSystem(BaseFileSystem):
async def aupload_file(self, local_path: str, destination_path: str) -> bool: async def aupload_file(self, local_path: str, destination_path: str) -> bool:
try: try:
async with self.session.client( async with self.session.client("s3", **self._client_kwargs()) as s3_client:
"s3", **self._client_kwargs()
) as s3_client:
await s3_client.upload_file( await s3_client.upload_file(
local_path, self.bucket_name, destination_path local_path, self.bucket_name, destination_path
) )
@ -114,9 +92,7 @@ class S3FileSystem(BaseFileSystem):
disposition on the response. disposition on the response.
""" """
try: try:
async with self.session.client( async with self.session.client("s3", **self._client_kwargs()) as s3_client:
"s3", **self._client_kwargs()
) as s3_client:
params = {"Bucket": self.bucket_name, "Key": file_path} params = {"Bucket": self.bucket_name, "Key": file_path}
# Make artifacts viewable inline in the browser when requested # Make artifacts viewable inline in the browser when requested
@ -155,9 +131,7 @@ class S3FileSystem(BaseFileSystem):
async def aget_file_metadata(self, file_path: str) -> Optional[Dict[str, Any]]: async def aget_file_metadata(self, file_path: str) -> Optional[Dict[str, Any]]:
"""Get S3 object metadata.""" """Get S3 object metadata."""
try: try:
async with self.session.client( async with self.session.client("s3", **self._client_kwargs()) as s3_client:
"s3", **self._client_kwargs()
) as s3_client:
response = await s3_client.head_object( response = await s3_client.head_object(
Bucket=self.bucket_name, Key=file_path Bucket=self.bucket_name, Key=file_path
) )
@ -181,9 +155,7 @@ class S3FileSystem(BaseFileSystem):
) -> Optional[str]: ) -> Optional[str]:
"""Generate a presigned PUT URL for direct file upload.""" """Generate a presigned PUT URL for direct file upload."""
try: try:
async with self.session.client( async with self.session.client("s3", **self._client_kwargs()) as s3_client:
"s3", **self._client_kwargs()
) as s3_client:
url = await s3_client.generate_presigned_url( url = await s3_client.generate_presigned_url(
"put_object", "put_object",
Params={ Params={
@ -200,9 +172,7 @@ class S3FileSystem(BaseFileSystem):
async def adownload_file(self, source_path: str, local_path: str) -> bool: async def adownload_file(self, source_path: str, local_path: str) -> bool:
"""Download a file from S3 to local path.""" """Download a file from S3 to local path."""
try: try:
async with self.session.client( async with self.session.client("s3", **self._client_kwargs()) as s3_client:
"s3", **self._client_kwargs()
) as s3_client:
await s3_client.download_file(self.bucket_name, source_path, local_path) await s3_client.download_file(self.bucket_name, source_path, local_path)
return True return True
except ClientError: except ClientError:
@ -211,9 +181,7 @@ class S3FileSystem(BaseFileSystem):
async def acopy_file(self, source_path: str, destination_path: str) -> bool: async def acopy_file(self, source_path: str, destination_path: str) -> bool:
"""Copy a file within S3 (server-side copy).""" """Copy a file within S3 (server-side copy)."""
try: try:
async with self.session.client( async with self.session.client("s3", **self._client_kwargs()) as s3_client:
"s3", **self._client_kwargs()
) as s3_client:
await s3_client.copy_object( await s3_client.copy_object(
Bucket=self.bucket_name, Bucket=self.bucket_name,
Key=destination_path, Key=destination_path,

11
api/services/storage.py
View file

@ -9,8 +9,11 @@ from api.constants import (
MINIO_PUBLIC_ENDPOINT, MINIO_PUBLIC_ENDPOINT,
MINIO_SECRET_KEY, MINIO_SECRET_KEY,
MINIO_SECURE, MINIO_SECURE,
S3_ADDRESSING_STYLE,
S3_BUCKET, S3_BUCKET,
S3_ENDPOINT_URL,
S3_REGION, S3_REGION,
S3_SIGNATURE_VERSION,
) )
from api.enums import Environment, StorageBackend from api.enums import Environment, StorageBackend
@ -57,7 +60,13 @@ def get_storage_for_backend(backend: str) -> BaseFileSystem:
logger.info( logger.info(
f"Initializing {backend} storage with bucket '{bucket}' in region '{region}'" f"Initializing {backend} storage with bucket '{bucket}' in region '{region}'"
) )
return S3FileSystem(bucket, region) return S3FileSystem(
bucket_name=bucket,
region_name=region,
endpoint_url=S3_ENDPOINT_URL,
signature_version=S3_SIGNATURE_VERSION,
addressing_style=S3_ADDRESSING_STYLE,
)
# Future backend implementations can be added here: # Future backend implementations can be added here:
# elif backend == StorageBackend.GCS: # Code 3 # elif backend == StorageBackend.GCS: # Code 3

28
docker-compose.yaml
View file

@ -145,17 +145,27 @@ services:
# Redis configuration (using containerized redis) # Redis configuration (using containerized redis)
REDIS_URL: "redis://:${REDIS_PASSWORD:-redissecret}@redis:6379" REDIS_URL: "redis://:${REDIS_PASSWORD:-redissecret}@redis:6379"
# Storage configuration - using local MinIO # Storage configuration - bundled MinIO by default. Set ENABLE_AWS_S3=true
ENABLE_AWS_S3: "false" # in .env to make the API use AWS S3 or another S3-compatible server.
ENABLE_AWS_S3: "${ENABLE_AWS_S3:-false}"
# To use AWS S3 or any S3-compatible server (MinIO, rustfs, Ceph, ...) # S3 backend configuration. Compose's .env file is used for interpolation,
# instead of the bundled MinIO, set ENABLE_AWS_S3 to "true" and provide: # but those values are not automatically injected into containers, so pass
# S3_BUCKET, S3_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY # the S3 settings through explicitly.
# For a non-AWS S3-compatible server, also set: AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID:-}"
# S3_ENDPOINT_URL e.g. https://s3.example.com AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY:-}"
# S3_SIGNATURE_VERSION set "s3v4" if the server requires SigV4 (e.g. rustfs) AWS_SESSION_TOKEN: "${AWS_SESSION_TOKEN:-}"
# S3_ADDRESSING_STYLE set "path" if the server / TLS cert requires path-style S3_BUCKET: "${S3_BUCKET:-}"
S3_REGION: "${S3_REGION:-us-east-1}"
# For a non-AWS S3-compatible server, also set these in Compose's .env
# S3_ENDPOINT_URL e.g. https://s3.example.com
# S3_SIGNATURE_VERSION set "s3v4" if the server requires SigV4 (e.g. rustfs)
# S3_ADDRESSING_STYLE set "path" if the server / TLS cert requires path-style
# The S3 backend issues real presigned URLs, so the bucket can stay private. # The S3 backend issues real presigned URLs, so the bucket can stay private.
S3_ENDPOINT_URL: "${S3_ENDPOINT_URL:-}"
S3_SIGNATURE_VERSION: "${S3_SIGNATURE_VERSION:-}"
S3_ADDRESSING_STYLE: "${S3_ADDRESSING_STYLE:-}"
# MinIO # MinIO
MINIO_ENDPOINT: "minio:9000" MINIO_ENDPOINT: "minio:9000"