mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-05-05 05:12:36 +02:00
Change MinIO integration options in librarian to be more generic - to support a Garage integration (#594)
* Tweak object store parameters to be more generic for other S3-type store integration * Update librarian to have region & SSL params * Update MinIO migration tech spec
This commit is contained in:
cybermaggedon
GitHub
parent
5304f96fe6
commit
25563bae3c
5 changed files with 333 additions and 44 deletions
|
|
@ -14,29 +14,32 @@ class BlobStore:
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
minio_host, minio_access_key, minio_secret_key, bucket_name,
|
||||
endpoint, access_key, secret_key, bucket_name,
|
||||
use_ssl=False, region=None,
|
||||
):
|
||||
|
||||
|
||||
self.minio = Minio(
|
||||
endpoint = minio_host,
|
||||
access_key = minio_access_key,
|
||||
secret_key = minio_secret_key,
|
||||
secure = False,
|
||||
self.client = Minio(
|
||||
endpoint = endpoint,
|
||||
access_key = access_key,
|
||||
secret_key = secret_key,
|
||||
secure = use_ssl,
|
||||
region = region,
|
||||
)
|
||||
|
||||
self.bucket_name = bucket_name
|
||||
|
||||
logger.info("Connected to MinIO")
|
||||
protocol = "https" if use_ssl else "http"
|
||||
logger.info(f"Connected to S3-compatible storage at {protocol}://{endpoint}")
|
||||
|
||||
self.ensure_bucket()
|
||||
|
||||
def ensure_bucket(self):
|
||||
|
||||
# Make the bucket if it doesn't exist.
|
||||
found = self.minio.bucket_exists(bucket_name=self.bucket_name)
|
||||
found = self.client.bucket_exists(bucket_name=self.bucket_name)
|
||||
if not found:
|
||||
self.minio.make_bucket(bucket_name=self.bucket_name)
|
||||
self.client.make_bucket(bucket_name=self.bucket_name)
|
||||
logger.info(f"Created bucket {self.bucket_name}")
|
||||
else:
|
||||
logger.debug(f"Bucket {self.bucket_name} already exists")
|
||||
|
|
@ -44,7 +47,7 @@ class BlobStore:
|
|||
async def add(self, object_id, blob, kind):
|
||||
|
||||
# FIXME: Loop retry
|
||||
self.minio.put_object(
|
||||
self.client.put_object(
|
||||
bucket_name = self.bucket_name,
|
||||
object_name = "doc/" + str(object_id),
|
||||
length = len(blob),
|
||||
|
|
@ -57,7 +60,7 @@ class BlobStore:
|
|||
async def remove(self, object_id):
|
||||
|
||||
# FIXME: Loop retry
|
||||
self.minio.remove_object(
|
||||
self.client.remove_object(
|
||||
bucket_name = self.bucket_name,
|
||||
object_name = "doc/" + str(object_id),
|
||||
)
|
||||
|
|
@ -68,7 +71,7 @@ class BlobStore:
|
|||
async def get(self, object_id):
|
||||
|
||||
# FIXME: Loop retry
|
||||
resp = self.minio.get_object(
|
||||
resp = self.client.get_object(
|
||||
bucket_name = self.bucket_name,
|
||||
object_name = "doc/" + str(object_id),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,12 +17,14 @@ class Librarian:
|
|||
def __init__(
|
||||
self,
|
||||
cassandra_host, cassandra_username, cassandra_password,
|
||||
minio_host, minio_access_key, minio_secret_key,
|
||||
object_store_endpoint, object_store_access_key, object_store_secret_key,
|
||||
bucket_name, keyspace, load_document,
|
||||
object_store_use_ssl=False, object_store_region=None,
|
||||
):
|
||||
|
||||
self.blob_store = BlobStore(
|
||||
minio_host, minio_access_key, minio_secret_key, bucket_name
|
||||
object_store_endpoint, object_store_access_key, object_store_secret_key, bucket_name,
|
||||
use_ssl=object_store_use_ssl, region=object_store_region,
|
||||
)
|
||||
|
||||
self.table_store = LibraryTableStore(
|
||||
|
|
|
|||
|
|
@ -41,9 +41,11 @@ default_collection_response_queue = collection_response_queue
|
|||
default_config_request_queue = config_request_queue
|
||||
default_config_response_queue = config_response_queue
|
||||
|
||||
default_minio_host = "minio:9000"
|
||||
default_minio_access_key = "minioadmin"
|
||||
default_minio_secret_key = "minioadmin"
|
||||
default_object_store_endpoint = "ceph-rgw:7480"
|
||||
default_object_store_access_key = "object-user"
|
||||
default_object_store_secret_key = "object-password"
|
||||
default_object_store_use_ssl = False
|
||||
default_object_store_region = None
|
||||
default_cassandra_host = "cassandra"
|
||||
|
||||
bucket_name = "library"
|
||||
|
|
@ -80,14 +82,22 @@ class Processor(AsyncProcessor):
|
|||
"config_response_queue", default_config_response_queue
|
||||
)
|
||||
|
||||
minio_host = params.get("minio_host", default_minio_host)
|
||||
minio_access_key = params.get(
|
||||
"minio_access_key",
|
||||
default_minio_access_key
|
||||
object_store_endpoint = params.get("object_store_endpoint", default_object_store_endpoint)
|
||||
object_store_access_key = params.get(
|
||||
"object_store_access_key",
|
||||
default_object_store_access_key
|
||||
)
|
||||
minio_secret_key = params.get(
|
||||
"minio_secret_key",
|
||||
default_minio_secret_key
|
||||
object_store_secret_key = params.get(
|
||||
"object_store_secret_key",
|
||||
default_object_store_secret_key
|
||||
)
|
||||
object_store_use_ssl = params.get(
|
||||
"object_store_use_ssl",
|
||||
default_object_store_use_ssl
|
||||
)
|
||||
object_store_region = params.get(
|
||||
"object_store_region",
|
||||
default_object_store_region
|
||||
)
|
||||
|
||||
cassandra_host = params.get("cassandra_host")
|
||||
|
|
@ -113,8 +123,8 @@ class Processor(AsyncProcessor):
|
|||
"librarian_response_queue": librarian_response_queue,
|
||||
"collection_request_queue": collection_request_queue,
|
||||
"collection_response_queue": collection_response_queue,
|
||||
"minio_host": minio_host,
|
||||
"minio_access_key": minio_access_key,
|
||||
"object_store_endpoint": object_store_endpoint,
|
||||
"object_store_access_key": object_store_access_key,
|
||||
"cassandra_host": self.cassandra_host,
|
||||
"cassandra_username": self.cassandra_username,
|
||||
"cassandra_password": self.cassandra_password,
|
||||
|
|
@ -208,12 +218,14 @@ class Processor(AsyncProcessor):
|
|||
cassandra_host = self.cassandra_host,
|
||||
cassandra_username = self.cassandra_username,
|
||||
cassandra_password = self.cassandra_password,
|
||||
minio_host = minio_host,
|
||||
minio_access_key = minio_access_key,
|
||||
minio_secret_key = minio_secret_key,
|
||||
object_store_endpoint = object_store_endpoint,
|
||||
object_store_access_key = object_store_access_key,
|
||||
object_store_secret_key = object_store_secret_key,
|
||||
bucket_name = bucket_name,
|
||||
keyspace = keyspace,
|
||||
load_document = self.load_document,
|
||||
object_store_use_ssl = object_store_use_ssl,
|
||||
object_store_region = object_store_region,
|
||||
)
|
||||
|
||||
self.collection_manager = CollectionManager(
|
||||
|
|
@ -494,23 +506,36 @@ class Processor(AsyncProcessor):
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--minio-host',
|
||||
default=default_minio_host,
|
||||
help=f'Minio hostname (default: {default_minio_host})',
|
||||
'--object-store-endpoint',
|
||||
default=default_object_store_endpoint,
|
||||
help=f'Object storage endpoint (default: {default_object_store_endpoint})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--minio-access-key',
|
||||
default='minioadmin',
|
||||
help='Minio access key / username '
|
||||
f'(default: {default_minio_access_key})',
|
||||
'--object-store-access-key',
|
||||
default=default_object_store_access_key,
|
||||
help='Object storage access key / username '
|
||||
f'(default: {default_object_store_access_key})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--minio-secret-key',
|
||||
default='minioadmin',
|
||||
help='Minio secret key / password '
|
||||
f'(default: {default_minio_access_key})',
|
||||
'--object-store-secret-key',
|
||||
default=default_object_store_secret_key,
|
||||
help='Object storage secret key / password '
|
||||
f'(default: {default_object_store_secret_key})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--object-store-use-ssl',
|
||||
action='store_true',
|
||||
default=default_object_store_use_ssl,
|
||||
help=f'Use SSL/TLS for object storage connection (default: {default_object_store_use_ssl})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--object-store-region',
|
||||
default=default_object_store_region,
|
||||
help='Object storage region (optional)',
|
||||
)
|
||||
|
||||
add_cassandra_args(parser)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue