apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-16 16:51:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Integrate tg-init-pulsar with YAMLs

Browse source
This commit is contained in:
Cyber MacGeddon 2024-09-30 12:17:26 +01:00
parent aa2350e248
commit 4113066dd2
3 changed files with 31 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

43
scripts/tg-init-pulsar
View file

@ -8,13 +8,13 @@ import requests
import time
import argparse
default_pulsar_admin_url = "http://pulsar:8080/"
default_pulsar_admin_url = "http://pulsar:8080"
def get_clusters(url):
print("Get clusters...")
print("Get clusters...", flush=True)
resp = requests.get(f"{url}admin/v2/clusters")
resp = requests.get(f"{url}/admin/v2/clusters")
if resp.status_code != 200: raise RuntimeError("Could not fetch clusters")
@ -22,14 +22,14 @@ def get_clusters(url):
def ensure_tenant(url, tenant, clusters):
resp = requests.get(f"{url}admin/v2/tenants/{tenant}")
resp = requests.get(f"{url}/admin/v2/tenants/{tenant}")
if resp.status_code == 200:
print(f"Tenant {tenant} already exists.")
print(f"Tenant {tenant} already exists.", flush=True)
return
resp = requests.put(
f"{url}admin/v2/tenants/{tenant}",
f"{url}/admin/v2/tenants/{tenant}",
json={
"adminRoles": [],
"allowedClusters": clusters,
@ -37,30 +37,30 @@ def ensure_tenant(url, tenant, clusters):
)
if resp.status_code != 204:
print(resp.text)
print(resp.text, flush=True)
raise RuntimeError("Tenant creation failed.")
print(f"Tenant {tenant} created.")
print(f"Tenant {tenant} created.", flush=True)
def ensure_namespace(url, tenant, namespace, config):
resp = requests.get(f"{url}admin/v2/namespaces/{tenant}/{namespace}")
resp = requests.get(f"{url}/admin/v2/namespaces/{tenant}/{namespace}")
if resp.status_code == 200:
print(f"Namespace {tenant}/{namespace} already exists.")
print(f"Namespace {tenant}/{namespace} already exists.", flush=True)
return
resp = requests.put(
f"{url}admin/v2/namespaces/{tenant}/{namespace}",
f"{url}/admin/v2/namespaces/{tenant}/{namespace}",
json=config,
)
if resp.status_code != 204:
print(resp.status_code)
print(resp.text)
print(resp.status_code, flush=True)
print(resp.text, flush=True)
raise RuntimeError(f"Namespace {tenant}/{namespace} creation failed.")
print(f"Namespace {tenant}/{namespace} created.")
print(f"Namespace {tenant}/{namespace} created.", flush=True)
def init(url, tenant="tg"):
@ -98,19 +98,22 @@ def main():
try:
print()
print("Initialising...")
print(flush=True)
print(
f"Initialising with Pulsar {args.pulsar_admin_url}...",
flush=True
)
init(args.pulsar_admin_url, "tg")
print("Initialisation complete.")
print("Initialisation complete.", flush=True)
break
except Exception as e:
print("Exception:", e)
print("Exception:", e, flush=True)
print("Sleeping...")
print("Sleeping...", flush=True)
time.sleep(2)
print("Will retry...")
print("Will retry...", flush=True)
main()

14
templates/components/pulsar.jsonnet
View file

@ -1,5 +1,6 @@
local base = import "base/base.jsonnet";
local images = import "values/images.jsonnet";
local url = import "values/url.jsonnet";
{
@ -14,7 +15,6 @@ local images = import "values/images.jsonnet";
engine.container("pulsar")
.with_image(images.pulsar)
.with_command(["bin/pulsar", "standalone"])
// .with_command(["/bin/sh", "-c", "sleep 9999999"])
.with_environment({
"PULSAR_MEM": "-Xms600M -Xmx600M"
})
@ -27,14 +27,14 @@ local images = import "values/images.jsonnet";
local adminContainer =
engine.container("init-pulsar")
.with_image(images.pulsar)
.with_image(images.trustgraph)
.with_command([
"sh",
"-c",
"while true; do pulsar-admin --admin-url http://pulsar:8080 tenants create tg ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response ; pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response; sleep 20; done",
"tg-init-pulsar",
"-p",
url.pulsar_admin,
])
.with_limits("1", "400M")
.with_reservations("0.1", "400M");
.with_limits("1", "128M")
.with_reservations("0.1", "128M");
local containerSet = engine.containers(
"pulsar",

1
templates/values/url.jsonnet
View file

@ -1,5 +1,6 @@
{
pulsar: "pulsar://pulsar:6650",
pulsar_admin: "http://pulsar:8080",
milvus: "http://milvus:19530",
qdrant: "http://qdrant:6333",
}