apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-05-12 08:42:37 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix remaining templates (#54)

Browse source
This commit is contained in:
cybermaggedon 2024-09-05 18:17:47 +01:00 • committed by GitHub
parent 6fb118ba95
commit f7429f2f9b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 328 additions and 304 deletions
Download patch file Download diff file Expand all files Collapse all files

4
templates/components/document-rag.jsonnet
View file

@ -10,7 +10,7 @@ local prompts = import "prompts/mixtral.jsonnet";
create:: function(engine)
local container =
engine.container("document-rag"")
engine.container("document-rag")
.with_image(images.trustgraph)
.with_command([
"document-rag",
@ -25,7 +25,7 @@ local prompts = import "prompts/mixtral.jsonnet";
.with_reservations("0.1", "128M");
local containerSet = engine.containers(
"document-rag"", [ container ]
"document-rag", [ container ]
);
engine.resources([

55
templates/components/embeddings-ollama.yaml
View file

@ -1,41 +1,40 @@
local base = import "base/base.jsonnet";
local images = import "values/images.jsonnet";
local url = import "values/url.jsonnet";
local prompts = import "prompt-template.jsonnet";
{
"embeddings-model":: "mxbai-embed-large",
"ollama-url":: "${OLLAMA_HOST}",
services +: {
embeddings +: {
create:: function(engine)
embeddings: base + {
image: images.trustgraph,
command: [
"embeddings-ollama",
"-p",
url.pulsar,
"-m",
$["embeddings-model"],
"-r",
$["ollama-url"],
],
deploy: {
resources: {
limits: {
cpus: '0.5',
memory: '128M'
},
reservations: {
cpus: '0.1',
memory: '128M'
}
}
},
},
local container =
engine.container("embeddings")
.with_image(images.trustgraph)
.with_command([
"embeddings-ollama",
"-p",
url.pulsar,
"-m",
$["embeddings-model"],
"-r",
$["ollama-url"],
])
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");
}
local containerSet = engine.containers(
"embeddings", [ container ]
);
} + prompts
engine.resources([
containerSet,
])
},
}

193
templates/components/milvus.jsonnet
View file

@ -1,104 +1,117 @@
local base = import "base/base.jsonnet";
local images = import "values/images.jsonnet";
local url = import "values/url.jsonnet";
local cassandra_hosts = "cassandra";
local milvus = import "stores/milvus.jsonnet";
milvus + {
services +: {
"store-graph-embeddings" +: {
create:: function(engine)
"store-graph-embeddings": base + {
image: images.trustgraph,
command: [
"ge-write-milvus",
"-p",
url.pulsar,
"-t",
url.milvus,
],
deploy: {
resources: {
limits: {
cpus: '0.5',
memory: '128M'
},
reservations: {
cpus: '0.1',
memory: '128M'
}
}
},
},
local container =
engine.container("store-graph-embeddings")
.with_image(images.trustgraph)
.with_command([
"ge-write-milvus",
"-p",
url.pulsar,
"-t",
url.milvus,
])
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");
"query-graph-embeddings": base + {
image: images.trustgraph,
command: [
"ge-query-milvus",
"-p",
url.pulsar,
"-t",
url.milvus,
],
deploy: {
resources: {
limits: {
cpus: '0.5',
memory: '128M'
},
reservations: {
cpus: '0.1',
memory: '128M'
}
}
},
},
local containerSet = engine.containers(
"store-graph-embeddings", [ container ]
);
// Document embeddings writer & query service.
"store-doc-embeddings": base + {
image: images.trustgraph,
command: [
"de-write-milvus",
"-p",
url.pulsar,
"-t",
url.milvus,
],
deploy: {
resources: {
limits: {
cpus: '0.5',
memory: '128M'
},
reservations: {
cpus: '0.1',
memory: '128M'
}
}
},
},
engine.resources([
containerSet,
])
},
"query-graph-embeddings" +: {
create:: function(engine)
local container =
engine.container("query-graph-embeddings")
.with_image(images.trustgraph)
.with_command([
"ge-query-milvus",
"-p",
url.pulsar,
"-t",
url.milvus,
])
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");
local containerSet = engine.containers(
"query-graph-embeddings", [ container ]
);
engine.resources([
containerSet,
])
},
"store-doc-embeddings" +: {
create:: function(engine)
local container =
engine.container("store-doc-embeddings")
.with_image(images.trustgraph)
.with_command([
"de-write-milvus",
"-p",
url.pulsar,
"-t",
url.milvus,
])
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");
local containerSet = engine.containers(
"store-doc-embeddings", [ container ]
);
engine.resources([
containerSet,
])
},
"query-doc-embeddings" +: {
create:: function(engine)
local container =
engine.container("query-doc-embeddings")
.with_image(images.trustgraph)
.with_command([
"de-query-milvus",
"-p",
url.pulsar,
"-t",
url.milvus,
])
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");
local containerSet = engine.containers(
"query-doc-embeddings", [ container ]
);
engine.resources([
containerSet,
])
"query-doc-embeddings": base + {
image: images.trustgraph,
command: [
"de-query-milvus",
"-p",
url.pulsar,
"-t",
url.milvus,
],
deploy: {
resources: {
limits: {
cpus: '0.5',
memory: '128M'
},
reservations: {
cpus: '0.1',
memory: '128M'
}
}
},
},
}

60
templates/components/pulsar-manager.jsonnet
View file

@ -1,33 +1,41 @@
local base = import "base/base.jsonnet";
local images = import "values/images.jsonnet";
{
// FIXME: Should persist *something*
volumes +: {
},
"pulsar" +: {
create:: function(engine)
// FIXME: Should persist something?
// local volume = engine.volume(...)
local container =
engine.container("pulsar")
.with_image(images.pulsar_manager)
.with_environment({
SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties",
})
.with_limits("0.5", "1.4G")
.with_reservations("0.1", "1.4G")
.with_port(9527, 9527, "api")
.with_port(7750, 7750, "api2");
local containerSet = engine.containers(
"pulsar", [ container ]
);
local service =
engine.service(containerSet)
.with_port(9527, 9527)
.with_port(7750, 7750);
engine.resources([
containerSet,
service,
])
services +: {
"pulsar-manager": base + {
image: images.pulsar_manager,
ports: [
"9527:9527",
"7750:7750",
],
environment: {
SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties",
},
deploy: {
resources: {
limits: {
cpus: '0.5',
memory: '1.4G'
},
reservations: {
cpus: '0.1',
memory: '1.4G'
}
}
},
},
}
}

82
templates/components/pulsar.jsonnet
View file

@ -7,52 +7,52 @@ local images = import "values/images.jsonnet";
create:: function(engine)
local confVolume = engine.volume("pulsar-conf").with_size("2G");
local dataVolume = engine.volume("pulsar-data").with_size("20G");
local confVolume = engine.volume("pulsar-conf").with_size("2G");
local dataVolume = engine.volume("pulsar-data").with_size("20G");
local container =
engine.container("pulsar")
.with_image(images.pulsar)
.with_command("bin/pulsar standalone")
.with_environment({
"PULSAR_MEM": "-Xms700M -Xmx700M"
})
.with_limits("1.0", "900M")
.with_reservations("0.5", "900M")
.with_volume_mount(confVolume, "/pulsar/conf")
.with_volume_mount(dataVolume, "/pulsar/data")
.with_port(6650, 6650, "bookie")
.with_port(8080, 8080, "http");
local container =
engine.container("pulsar")
.with_image(images.pulsar)
.with_command("bin/pulsar standalone")
.with_environment({
"PULSAR_MEM": "-Xms700M -Xmx700M"
})
.with_limits("1.0", "900M")
.with_reservations("0.5", "900M")
.with_volume_mount(confVolume, "/pulsar/conf")
.with_volume_mount(dataVolume, "/pulsar/data")
.with_port(6650, 6650, "bookie")
.with_port(8080, 8080, "http");
local adminContainer =
engine.container("init-pulsar")
.with_image(images.pulsar)
.with_command([
"sh",
"-c",
"pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response",
])
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");
local adminContainer =
engine.container("init-pulsar")
.with_image(images.pulsar)
.with_command([
"sh",
"-c",
"pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response",
])
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");
local containerSet = engine.containers(
"pulsar",
[
container, adminContainer
]
);
local containerSet = engine.containers(
"pulsar",
[
container, adminContainer
]
);
local service =
engine.service(containerSet)
.with_port(6650, 6650)
.with_port(8080, 8080);
local service =
engine.service(containerSet)
.with_port(6650, 6650)
.with_port(8080, 8080);
engine.resources([
confVolume,
dataVolume,
containerSet,
service,
])
engine.resources([
confVolume,
dataVolume,
containerSet,
service,
])
}

5
templates/stores/cassandra.jsonnet
View file

@ -24,9 +24,14 @@ local images = import "values/images.jsonnet";
"cassandra", [ container ]
);
local service =
engine.service(containerSet)
.with_port(9042, 9042);
engine.resources([
vol,
containerSet,
service,
])
},

233
templates/stores/milvus.jsonnet
View file

@ -3,128 +3,127 @@ local images = import "values/images.jsonnet";
{
volumes +: {
etcd: {},
"minio-data": {},
milvus: {},
etcd +: {
create:: function(engine)
local vol = engine.volume("etcd").with_size("20G");
local container =
engine.container("etcd")
.with_image(images.etcd)
.with_command([
"etcd",
"-advertise-client-urls=http://127.0.0.1:2379",
"-listen-client-urls",
"http://0.0.0.0:2379",
"--data-dir",
"/etcd",
])
.with_environment({
ETCD_AUTO_COMPACTION_MODE: "revision",
ETCD_AUTO_COMPACTION_RETENTION: "1000",
ETCD_QUOTA_BACKEND_BYTES: "4294967296",
ETCD_SNAPSHOT_COUNT: "50000"
})
.with_limits("1.0", "128M")
.with_reservations("0.25", "128M")
.with_port(2379, 2379, "api")
.with_volume_mount(vol, "/etcd");
local containerSet = engine.containers(
"etcd", [ container ]
);
local service =
engine.service(containerSet)
.with_port(2379, 2379);
engine.resources([
vol,
containerSet,
service,
])
},
services +: {
mino +: {
create:: function(engine)
etcd: base + {
image: images.etcd,
command: [
"etcd",
"-advertise-client-urls=http://127.0.0.1:2379",
"-listen-client-urls",
"http://0.0.0.0:2379",
"--data-dir",
"/etcd",
],
environment: {
ETCD_AUTO_COMPACTION_MODE: "revision",
ETCD_AUTO_COMPACTION_RETENTION: "1000",
ETCD_QUOTA_BACKEND_BYTES: "4294967296",
ETCD_SNAPSHOT_COUNT: "50000"
},
ports: [
{
src: 2379,
dest: 2379,
name: "api",
}
],
volumes: [
"etcd:/etcd"
],
deploy: {
resources: {
limits: {
cpus: '1.0',
memory: '128M'
},
reservations: {
cpus: '0.25',
memory: '128M'
}
},
},
},
local vol = engine.volume("minio-data").with_size("20G");
minio: base + {
image: images.minio,
command: [
"minio",
"server",
"/minio_data",
"--console-address",
":9001",
],
environment: {
MINIO_ROOT_USER: "minioadmin",
MINIO_ROOT_PASSWORD: "minioadmin",
},
ports: [
{
src: 9001,
dest: 9001,
name: "api",
}
],
volumes: [
"minio-data:/minio_data",
],
deploy: {
resources: {
limits: {
cpus: '0.5',
memory: '128M'
},
reservations: {
cpus: '0.25',
memory: '128M'
}
}
},
},
local container =
engine.container("minio")
.with_image(images.minio)
.with_command([
"minio",
"server",
"/minio_data",
"--console-address",
":9001",
])
.with_environment({
MINIO_ROOT_USER: "minioadmin",
MINIO_ROOT_PASSWORD: "minioadmin",
})
.with_limits("0.5", "128M")
.with_reservations("0.25", "128M")
.with_port(9001, 9001, "api")
.with_volume_mount(vol, "/minio_data");
milvus: base + {
image: images.milvus,
command: [
"milvus", "run", "standalone"
],
environment: {
ETCD_ENDPOINTS: "etcd:2379",
MINIO_ADDRESS: "minio:9000",
},
ports: [
{
src: 9091,
dest: 9091,
name: "api",
},
{
src: 19530,
dest: 19530,
name: "api2",
}
],
volumes: [
"milvus:/var/lib/milvus"
],
deploy: {
resources: {
limits: {
cpus: '1.0',
memory: '256M'
},
reservations: {
cpus: '0.5',
memory: '256M'
}
}
},
},
local containerSet = engine.containers(
"etcd", [ container ]
);
local service =
engine.service(containerSet)
.with_port(9001, 9001);
engine.resources([
vol,
containerSet,
service,
])
},
milvus +: {
create:: function(engine)
local vol = engine.volume("milvus").with_size("20G");
local container =
engine.container("milvus")
.with_image(images.milvus)
.with_command([
"milvus", "run", "standalone"
])
.with_environment({
ETCD_ENDPOINTS: "etcd:2379",
MINIO_ADDRESS: "minio:9000",
})
.with_limits("1.0", "256M")
.with_reservations("0.5", "256M")
.with_port(9091, 9091, "api")
.with_port(19530, 19530, "api2")
.with_volume_mount(vol, "/var/lib/milvus");
local containerSet = engine.containers(
"milvus", [ container ]
);
local service =
engine.service(containerSet)
.with_port(9091, 9091)
.with_port(19530, 19530);
engine.resources([
vol,
containerSet,
service,
])
},