- Added docker-compose-storage.yaml, just the infrastructure bits

- Tidied storage invocation
2026-07-05 19:32:11 +02:00 · 2024-08-21 12:14:45 +01:00 · 2024-08-21 12:14:45 +01:00 · e9298fbde9
commit e9298fbde9
parent b0fdb4f314
11 changed files with 291 additions and 121 deletions
--- a/6
+++ b/6
@ -1,6 +1,6 @@

 # VERSION=$(shell git describe | sed 's/^v//')
-VERSION=0.6.6
+VERSION=0.6.7

 DOCKER=podman

@ -33,7 +33,9 @@ set-version:
 # 	sed -i 's/trustgraph-flow:[0-9]*\.[0-9]*\.[0-9]*/trustgraph-flow:'${VERSION}'/' docker-compose*.yaml
 	echo '"${VERSION}"' > templates/components/version.jsonnet

-TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai openai-neo4j
+TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai \
+    openai-neo4j storage
+
 DCS=$(foreach template,${TEMPLATES},${template:%=docker-compose-%.yaml})

 update-templates: set-version ${DCS}
--- a/docker-compose-storage.yaml
+++ b/docker-compose-storage.yaml
@ -0,0 +1,115 @@
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}
--- a/setup.py
+++ b/setup.py
@ -4,7 +4,7 @@ import os
 with open("README.md", "r") as fh:
    long_description = fh.read()

-version = "0.6.6"
+version = "0.6.7"

 setuptools.setup(
    name="trustgraph",
--- a/templates/components/cassandra.jsonnet
+++ b/templates/components/cassandra.jsonnet
@ -2,21 +2,11 @@ local base = import "base.jsonnet";
 local images = import "images.jsonnet";
 local url = import "url.jsonnet";
 local cassandra_hosts = "cassandra";
-{
-    volumes +: {
-	cassandra: {},
-    },
-    services +: {
+local cassandra = import "stores/cassandra.jsonnet";

-	cassandra: base + {
-	    image: images.cassandra,
-	    ports: [
-		"9042:9042"
-	    ],
-	    volumes: [
-		"cassandra:/var/lib/cassandra"
-	    ],
-	},
+cassandra + {
+
+    services +: {

 	"store-triples": base + {
 	    image: images.trustgraph,
--- a/templates/components/milvus.jsonnet
+++ b/templates/components/milvus.jsonnet
@ -1,71 +1,36 @@
 local base = import "base.jsonnet";
 local images = import "images.jsonnet";
-{
-    volumes +: {
-	etcd: {},
-	"minio-data": {},
-	milvus: {},
-    },
+local url = import "url.jsonnet";
+local milvus = import "stores/milvus.jsonnet";
+
+milvus + {
+
    services +: {
-	etcd: base + {
-	    image: images.etcd,
+
+	"store-graph-embeddings": base + {
+	    image: images.trustgraph,
 	    command: [
-		"etcd",
-		"-advertise-client-urls=http://127.0.0.1:2379",
-		"-listen-client-urls",
-		"http://0.0.0.0:2379",
-		"--data-dir",
-		"/etcd",
-	    ],
-	    environment: {
-		ETCD_AUTO_COMPACTION_MODE: "revision",
-		ETCD_AUTO_COMPACTION_RETENTION: "1000",
-		ETCD_QUOTA_BACKEND_BYTES: "4294967296",
-		ETCD_SNAPSHOT_COUNT: "50000"
-	    },
-	    ports: [
-		"2379:2379",
-	    ],
-	    volumes: [
-		"etcd:/etcd"
+		"ge-write-milvus",
+		"-p",
+		url.pulsar,
+		"-t",
+		url.milvus,
 	    ],
 	},
-	minio: base + {
-	    image: images.minio,
+
+	"query-graph-embeddings": base + {
+	    image: images.trustgraph,
 	    command: [
-		"minio",
-		"server",
-		"/minio_data",
-		"--console-address",
-		":9001",
-	    ],
-	    environment: {
-		MINIO_ROOT_USER: "minioadmin",
-		MINIO_ROOT_PASSWORD: "minioadmin",
-	    },
-	    ports: [
-		"9001:9001",
-	    ],
-	    volumes: [
-		"minio-data:/minio_data",
-	    ],	
-	},
-	milvus: base + {
-	    image: images.milvus,
-	    command: [
-		"milvus", "run", "standalone"
-	    ],
-	    environment: {
-		ETCD_ENDPOINTS: "etcd:2379",
-		MINIO_ADDRESS: "minio:9000",
-	    },
-	    ports: [
-		"9091:9091",
-		"19530:19530",
-	    ],
-	    volumes: [
-		"milvus:/var/lib/milvus"
+		"ge-query-milvus",
+		"-p",
+		url.pulsar,
+		"-t",
+		url.milvus,
 	    ],
 	},
-    },
+
+    }
+
 }
+
+
--- a/templates/components/neo4j.jsonnet
+++ b/templates/components/neo4j.jsonnet
@ -1,31 +1,12 @@
 local base = import "base.jsonnet";
 local images = import "images.jsonnet";
 local url = import "url.jsonnet";
-{
+local neo4j = import "stores/neo4j.jsonnet";

-    volumes +: {
-	neo4j: {},
-    },
+neo4j + {

    services +: {

-	neo4j: base + {
-	    image: images.neo4j,
-	    ports: [
-		"7474:7474",
-                "7687:7687",
-	    ],
-	    environment: {
-		NEO4J_AUTH: "neo4j/password",
-//		NEO4J_server_bolt_listen__address: "0.0.0.0:7687",
-//		NEO4J_server_default__listen__address: "0.0.0.0",
-//		NEO4J_server_http_listen__address: "0.0.0.0:7474",
-	    },
-	    volumes: [
-		"neo4j:/data"
-	    ],
-	},
-
 	"query-triples": base + {
 	    image: images.trustgraph,
 	    command: [
--- a/templates/components/stores/cassandra.jsonnet
+++ b/templates/components/stores/cassandra.jsonnet
@ -0,0 +1,20 @@
+local base = import "../base.jsonnet";
+local images = import "../images.jsonnet";
+{
+    volumes +: {
+	cassandra: {},
+    },
+    services +: {
+
+	cassandra: base + {
+	    image: images.cassandra,
+	    ports: [
+		"9042:9042"
+	    ],
+	    volumes: [
+		"cassandra:/var/lib/cassandra"
+	    ],
+	},
+
+    },
+}
--- a/templates/components/stores/milvus.jsonnet
+++ b/templates/components/stores/milvus.jsonnet
@ -0,0 +1,79 @@
+local base = import "../base.jsonnet";
+local images = import "../images.jsonnet";
+
+{
+
+    volumes +: {
+	etcd: {},
+	"minio-data": {},
+	milvus: {},
+    },
+
+    services +: {
+
+	etcd: base + {
+	    image: images.etcd,
+	    command: [
+		"etcd",
+		"-advertise-client-urls=http://127.0.0.1:2379",
+		"-listen-client-urls",
+		"http://0.0.0.0:2379",
+		"--data-dir",
+		"/etcd",
+	    ],
+	    environment: {
+		ETCD_AUTO_COMPACTION_MODE: "revision",
+		ETCD_AUTO_COMPACTION_RETENTION: "1000",
+		ETCD_QUOTA_BACKEND_BYTES: "4294967296",
+		ETCD_SNAPSHOT_COUNT: "50000"
+	    },
+	    ports: [
+		"2379:2379",
+	    ],
+	    volumes: [
+		"etcd:/etcd"
+	    ],
+	},
+
+	minio: base + {
+	    image: images.minio,
+	    command: [
+		"minio",
+		"server",
+		"/minio_data",
+		"--console-address",
+		":9001",
+	    ],
+	    environment: {
+		MINIO_ROOT_USER: "minioadmin",
+		MINIO_ROOT_PASSWORD: "minioadmin",
+	    },
+	    ports: [
+		"9001:9001",
+	    ],
+	    volumes: [
+		"minio-data:/minio_data",
+	    ],	
+	},
+
+	milvus: base + {
+	    image: images.milvus,
+	    command: [
+		"milvus", "run", "standalone"
+	    ],
+	    environment: {
+		ETCD_ENDPOINTS: "etcd:2379",
+		MINIO_ADDRESS: "minio:9000",
+	    },
+	    ports: [
+		"9091:9091",
+		"19530:19530",
+	    ],
+	    volumes: [
+		"milvus:/var/lib/milvus"
+	    ],
+	},
+
+    },
+
+}
--- a/templates/components/stores/neo4j.jsonnet
+++ b/templates/components/stores/neo4j.jsonnet
@ -0,0 +1,30 @@
+local base = import "../base.jsonnet";
+local images = import "../images.jsonnet";
+{
+
+    volumes +: {
+	neo4j: {},
+    },
+
+    services +: {
+
+	neo4j: base + {
+	    image: images.neo4j,
+	    ports: [
+		"7474:7474",
+                "7687:7687",
+	    ],
+	    environment: {
+		NEO4J_AUTH: "neo4j/password",
+//		NEO4J_server_bolt_listen__address: "0.0.0.0:7687",
+//		NEO4J_server_default__listen__address: "0.0.0.0",
+//		NEO4J_server_http_listen__address: "0.0.0.0:7474",
+	    },
+	    volumes: [
+		"neo4j:/data"
+	    ],
+	},
+
+    },
+
+}
--- a/templates/components/trustgraph.jsonnet
+++ b/templates/components/trustgraph.jsonnet
@ -64,28 +64,6 @@ local url = import "url.jsonnet";
 	    ],
 	},

-	"store-graph-embeddings": base + {
-	    image: images.trustgraph,
-	    command: [
-		"ge-write-milvus",
-		"-p",
-		url.pulsar,
-		"-t",
-		url.milvus,
-	    ],
-	},
-
-	"query-graph-embeddings": base + {
-	    image: images.trustgraph,
-	    command: [
-		"ge-query-milvus",
-		"-p",
-		url.pulsar,
-		"-t",
-		url.milvus,
-	    ],
-	},
-
 	"graph-rag": base + {
 	    image: images.trustgraph,
 	    command: [
--- a/templates/docker-compose-storage.jsonnet
+++ b/templates/docker-compose-storage.jsonnet
@ -0,0 +1,10 @@
+
+local cassandra = import "components/stores/cassandra.jsonnet";
+local pulsar = import "components/pulsar.jsonnet";
+local milvus = import "components/stores/milvus.jsonnet";
+local grafana = import "components/grafana.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana;
+
+std.manifestYamlDoc(config)
+