apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-20 10:41:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Integrate tg-init-pulsar with YAMLs

Browse source
This commit is contained in:
Cyber MacGeddon 2024-09-30 12:17:26 +01:00
parent aa2350e248
commit 4113066dd2
3 changed files with 31 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

43
scripts/tg-init-pulsar
View file

@ -8,13 +8,13 @@ import requests
import time import time
import argparse import argparse
default_pulsar_admin_url = "http://pulsar:8080/" default_pulsar_admin_url = "http://pulsar:8080"
def get_clusters(url): def get_clusters(url):
print("Get clusters...") print("Get clusters...", flush=True)
resp = requests.get(f"{url}admin/v2/clusters") resp = requests.get(f"{url}/admin/v2/clusters")
if resp.status_code != 200: raise RuntimeError("Could not fetch clusters") if resp.status_code != 200: raise RuntimeError("Could not fetch clusters")
@ -22,14 +22,14 @@ def get_clusters(url):
def ensure_tenant(url, tenant, clusters): def ensure_tenant(url, tenant, clusters):
resp = requests.get(f"{url}admin/v2/tenants/{tenant}") resp = requests.get(f"{url}/admin/v2/tenants/{tenant}")
if resp.status_code == 200: if resp.status_code == 200:
print(f"Tenant {tenant} already exists.") print(f"Tenant {tenant} already exists.", flush=True)
return return
resp = requests.put( resp = requests.put(
f"{url}admin/v2/tenants/{tenant}", f"{url}/admin/v2/tenants/{tenant}",
json={ json={
"adminRoles": [], "adminRoles": [],
"allowedClusters": clusters, "allowedClusters": clusters,
@ -37,30 +37,30 @@ def ensure_tenant(url, tenant, clusters):
) )
if resp.status_code != 204: if resp.status_code != 204:
print(resp.text) print(resp.text, flush=True)
raise RuntimeError("Tenant creation failed.") raise RuntimeError("Tenant creation failed.")
print(f"Tenant {tenant} created.") print(f"Tenant {tenant} created.", flush=True)
def ensure_namespace(url, tenant, namespace, config): def ensure_namespace(url, tenant, namespace, config):
resp = requests.get(f"{url}admin/v2/namespaces/{tenant}/{namespace}") resp = requests.get(f"{url}/admin/v2/namespaces/{tenant}/{namespace}")
if resp.status_code == 200: if resp.status_code == 200:
print(f"Namespace {tenant}/{namespace} already exists.") print(f"Namespace {tenant}/{namespace} already exists.", flush=True)
return return
resp = requests.put( resp = requests.put(
f"{url}admin/v2/namespaces/{tenant}/{namespace}", f"{url}/admin/v2/namespaces/{tenant}/{namespace}",
json=config, json=config,
) )
if resp.status_code != 204: if resp.status_code != 204:
print(resp.status_code) print(resp.status_code, flush=True)
print(resp.text) print(resp.text, flush=True)
raise RuntimeError(f"Namespace {tenant}/{namespace} creation failed.") raise RuntimeError(f"Namespace {tenant}/{namespace} creation failed.")
print(f"Namespace {tenant}/{namespace} created.") print(f"Namespace {tenant}/{namespace} created.", flush=True)
def init(url, tenant="tg"): def init(url, tenant="tg"):
@ -98,19 +98,22 @@ def main():
try: try:
print() print(flush=True)
print("Initialising...") print(
f"Initialising with Pulsar {args.pulsar_admin_url}...",
flush=True
)
init(args.pulsar_admin_url, "tg") init(args.pulsar_admin_url, "tg")
print("Initialisation complete.") print("Initialisation complete.", flush=True)
break break
except Exception as e: except Exception as e:
print("Exception:", e) print("Exception:", e, flush=True)
print("Sleeping...") print("Sleeping...", flush=True)
time.sleep(2) time.sleep(2)
print("Will retry...") print("Will retry...", flush=True)
main() main()

14
templates/components/pulsar.jsonnet
View file

@ -1,5 +1,6 @@
local base = import "base/base.jsonnet"; local base = import "base/base.jsonnet";
local images = import "values/images.jsonnet"; local images = import "values/images.jsonnet";
local url = import "values/url.jsonnet";
{ {
@ -14,7 +15,6 @@ local images = import "values/images.jsonnet";
engine.container("pulsar") engine.container("pulsar")
.with_image(images.pulsar) .with_image(images.pulsar)
.with_command(["bin/pulsar", "standalone"]) .with_command(["bin/pulsar", "standalone"])
// .with_command(["/bin/sh", "-c", "sleep 9999999"])
.with_environment({ .with_environment({
"PULSAR_MEM": "-Xms600M -Xmx600M" "PULSAR_MEM": "-Xms600M -Xmx600M"
}) })
@ -27,14 +27,14 @@ local images = import "values/images.jsonnet";
local adminContainer = local adminContainer =
engine.container("init-pulsar") engine.container("init-pulsar")
.with_image(images.pulsar) .with_image(images.trustgraph)
.with_command([ .with_command([
"sh", "tg-init-pulsar",
"-c", "-p",
"while true; do pulsar-admin --admin-url http://pulsar:8080 tenants create tg ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response ; pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response; sleep 20; done", url.pulsar_admin,
]) ])
.with_limits("1", "400M") .with_limits("1", "128M")
.with_reservations("0.1", "400M"); .with_reservations("0.1", "128M");
local containerSet = engine.containers( local containerSet = engine.containers(
"pulsar", "pulsar",

1
templates/values/url.jsonnet
View file

@ -1,5 +1,6 @@
{ {
pulsar: "pulsar://pulsar:6650", pulsar: "pulsar://pulsar:6650",
pulsar_admin: "http://pulsar:8080",
milvus: "http://milvus:19530", milvus: "http://milvus:19530",
qdrant: "http://qdrant:6333", qdrant: "http://qdrant:6333",
} }