feat: add csv upload functionality for OSS (#29)

feat: add csv upload functionality
chore: remove redundant arq-worker from docker-compose

api/services/filesystem/base.py

@@ -33,13 +33,19 @@ class BaseFileSystem(ABC):
 
     @abstractmethod
     async def aget_signed_url(
-        self, file_path: str, expiration: int = 3600
+        self,
+        file_path: str,
+        expiration: int = 3600,
+        force_inline: bool = False,
+        use_internal_endpoint: bool = False,
     ) -> Optional[str]:
         """Generate a signed URL for temporary access to a file.
 
         Args:
             file_path: Path to the file
             expiration: URL expiration time in seconds (default: 1 hour)
+            force_inline: Force inline display (browser preview vs download)
+            use_internal_endpoint: Use internal endpoint (for container-to-container access)
 
         Returns:
             Optional[str]: Signed URL if successful, None otherwise
@@ -58,3 +64,24 @@ class BaseFileSystem(ABC):
             Contains: size, created_at, modified_at, etag, etc.
         """
         pass
+
+    @abstractmethod
+    async def aget_presigned_put_url(
+        self,
+        file_path: str,
+        expiration: int = 900,
+        content_type: str = "text/csv",
+        max_size: int = 10_485_760,
+    ) -> Optional[str]:
+        """Generate a presigned PUT URL for direct file upload.
+
+        Args:
+            file_path: Path where the file should be uploaded
+            expiration: URL expiration time in seconds (default: 15 minutes)
+            content_type: MIME type of the file (default: text/csv)
+            max_size: Maximum file size in bytes (default: 10MB)
+
+        Returns:
+            Optional[str]: Presigned PUT URL if successful, None otherwise
+        """
+        pass

api/services/filesystem/minio.py

@@ -48,17 +48,28 @@ class MinioFileSystem(BaseFileSystem):
             if not self.client.bucket_exists(self.bucket_name):
                 self.client.make_bucket(self.bucket_name)
 
-            # Set anonymous download policy for local development
-            # This allows unsigned URLs to work
+            # Set public read/write policy for local development
+            # This allows:
+            # 1. Anonymous downloads (s3:GetObject)
+            # 2. Anonymous uploads (s3:PutObject) - bypasses presigned URL signature issues
+            # 3. List bucket contents (s3:ListBucket) for debugging
+            # Note: This is set on every initialization to ensure policy is correct
+            # WARNING: Only use in local development, not production!
             policy = {
                 "Version": "2012-10-17",
                 "Statement": [
                     {
                         "Effect": "Allow",
                         "Principal": {"AWS": "*"},
-                        "Action": ["s3:GetObject"],
+                        "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
                         "Resource": [f"arn:aws:s3:::{self.bucket_name}/*"],
-                    }
+                    },
+                    {
+                        "Effect": "Allow",
+                        "Principal": {"AWS": "*"},
+                        "Action": ["s3:ListBucket"],
+                        "Resource": [f"arn:aws:s3:::{self.bucket_name}"],
+                    },
                 ],
             }
 
@@ -97,14 +108,19 @@ class MinioFileSystem(BaseFileSystem):
             return False
 
     async def aget_signed_url(
-        self, file_path: str, expiration: int = 3600, force_inline: bool = False
+        self,
+        file_path: str,
+        expiration: int = 3600,
+        force_inline: bool = False,
+        use_internal_endpoint: bool = False,
     ) -> Optional[str]:
         try:
             # For MinIO in local development, return unsigned URLs
             # This avoids signature mismatch issues when endpoint differs
             # MinIO must be configured to allow anonymous read access
             protocol = "https" if self.secure else "http"
-            url = f"{protocol}://{self.public_endpoint}/{self.bucket_name}/{file_path}"
+            endpoint = self.endpoint if use_internal_endpoint else self.public_endpoint
+            url = f"{protocol}://{endpoint}/{self.bucket_name}/{file_path}"
             return url
         except Exception as e:
             logger.error(f"Error generating MinIO URL: {e}")
@@ -128,3 +144,29 @@ class MinioFileSystem(BaseFileSystem):
             }
         except S3Error:
             return None
+
+    async def aget_presigned_put_url(
+        self,
+        file_path: str,
+        expiration: int = 900,
+        content_type: str = "text/csv",
+        max_size: int = 10_485_760,
+    ) -> Optional[str]:
+        """Generate an unsigned URL for direct file upload.
+
+        For local MinIO development with anonymous upload enabled, we return
+        a simple unsigned URL instead of a presigned URL. This avoids signature
+        mismatch issues when the internal endpoint (minio:9000) differs from
+        the public endpoint (localhost:9000).
+
+        The bucket policy allows anonymous s3:PutObject, so no signature is needed.
+        """
+        try:
+            # Return unsigned URL for anonymous upload
+            protocol = "https" if self.secure else "http"
+            url = f"{protocol}://{self.public_endpoint}/{self.bucket_name}/{file_path}"
+            logger.debug(f"Generated unsigned upload URL: {url}")
+            return url
+        except Exception as e:
+            logger.error(f"Error generating MinIO upload URL: {e}")
+            return None

api/services/filesystem/s3.py

@@ -45,7 +45,11 @@ class S3FileSystem(BaseFileSystem):
             return False
 
     async def aget_signed_url(
-        self, file_path: str, expiration: int = 3600, force_inline: bool = False
+        self,
+        file_path: str,
+        expiration: int = 3600,
+        force_inline: bool = False,
+        use_internal_endpoint: bool = False,
     ) -> Optional[str]:
         """Generate a presigned GET url for the given object.
 
@@ -97,3 +101,28 @@ class S3FileSystem(BaseFileSystem):
                 }
         except ClientError:
             return None
+
+    async def aget_presigned_put_url(
+        self,
+        file_path: str,
+        expiration: int = 900,
+        content_type: str = "text/csv",
+        max_size: int = 10_485_760,
+    ) -> Optional[str]:
+        """Generate a presigned PUT URL for direct file upload."""
+        try:
+            async with self.session.client(
+                "s3", region_name=self.region_name
+            ) as s3_client:
+                url = await s3_client.generate_presigned_url(
+                    "put_object",
+                    Params={
+                        "Bucket": self.bucket_name,
+                        "Key": file_path,
+                        "ContentType": content_type,
+                    },
+                    ExpiresIn=expiration,
+                )
+            return url
+        except ClientError:
+            return None