mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-12 20:45:20 +02:00
feat(etl-cache): resolve cache blob storage backend
This commit is contained in:
parent
d9b1b491e9
commit
217d040e9e
1 changed files with 46 additions and 0 deletions
46
surfsense_backend/app/etl_pipeline/cache/storage/backend.py
vendored
Normal file
46
surfsense_backend/app/etl_pipeline/cache/storage/backend.py
vendored
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
"""Resolve the storage backend for cache blobs: shared main store or a dedicated one."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
from app.file_storage.backends.base import StorageBackend
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def resolve_cache_backend() -> StorageBackend:
|
||||
from app.etl_pipeline.cache.settings import load_etl_cache_settings
|
||||
|
||||
settings = load_etl_cache_settings()
|
||||
|
||||
if not settings.storage_backend:
|
||||
from app.file_storage.factory import get_storage_backend
|
||||
|
||||
return get_storage_backend()
|
||||
|
||||
backend = settings.storage_backend.strip().lower()
|
||||
|
||||
if backend == "azure":
|
||||
from app.config import config
|
||||
|
||||
if not settings.storage_container:
|
||||
raise ValueError("ETL_CACHE_STORAGE_CONTAINER is required for azure cache.")
|
||||
if not config.AZURE_STORAGE_CONNECTION_STRING:
|
||||
raise ValueError(
|
||||
"AZURE_STORAGE_CONNECTION_STRING is required for azure cache."
|
||||
)
|
||||
from app.file_storage.backends.azure import AzureBlobBackend
|
||||
|
||||
return AzureBlobBackend(
|
||||
connection_string=config.AZURE_STORAGE_CONNECTION_STRING,
|
||||
container=settings.storage_container,
|
||||
)
|
||||
|
||||
if backend == "local":
|
||||
if not settings.storage_local_root:
|
||||
raise ValueError("ETL_CACHE_STORAGE_LOCAL_PATH is required for local cache.")
|
||||
from app.file_storage.backends.local import LocalFileBackend
|
||||
|
||||
return LocalFileBackend(settings.storage_local_root)
|
||||
|
||||
raise ValueError(f"Unknown ETL_CACHE_STORAGE_BACKEND: {settings.storage_backend!r}")
|
||||
Loading…
Add table
Add a link
Reference in a new issue