diff --git a/Makefile b/Makefile index 78cf820b..957c1810 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # VERSION=$(shell git describe | sed 's/^v//') -VERSION=0.6.6 +VERSION=0.6.7 DOCKER=podman @@ -33,7 +33,9 @@ set-version: # sed -i 's/trustgraph-flow:[0-9]*\.[0-9]*\.[0-9]*/trustgraph-flow:'${VERSION}'/' docker-compose*.yaml echo '"${VERSION}"' > templates/components/version.jsonnet -TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai openai-neo4j +TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai \ + openai-neo4j storage + DCS=$(foreach template,${TEMPLATES},${template:%=docker-compose-%.yaml}) update-templates: set-version ${DCS} diff --git a/docker-compose-storage.yaml b/docker-compose-storage.yaml new file mode 100644 index 00000000..fc366799 --- /dev/null +++ b/docker-compose-storage.yaml @@ -0,0 +1,115 @@ +"services": + "cassandra": + "image": "docker.io/cassandra:4.1.5" + "ports": + - "9042:9042" + "restart": "on-failure:100" + "volumes": + - "cassandra:/var/lib/cassandra" + "etcd": + "command": + - "etcd" + - "-advertise-client-urls=http://127.0.0.1:2379" + - "-listen-client-urls" + - "http://0.0.0.0:2379" + - "--data-dir" + - "/etcd" + "environment": + "ETCD_AUTO_COMPACTION_MODE": "revision" + "ETCD_AUTO_COMPACTION_RETENTION": "1000" + "ETCD_QUOTA_BACKEND_BYTES": "4294967296" + "ETCD_SNAPSHOT_COUNT": "50000" + "image": "quay.io/coreos/etcd:v3.5.5" + "ports": + - "2379:2379" + "restart": "on-failure:100" + "volumes": + - "etcd:/etcd" + "grafana": + "environment": + "GF_ORG_NAME": "trustgraph.ai" + "image": "docker.io/grafana/grafana:10.0.0" + "ports": + - "3000:3000" + "restart": "on-failure:100" + "volumes": + - "grafana-storage:/var/lib/grafana" + - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml" + - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml" + - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json" + "init-pulsar": + "command": + - "sh" + - "-c" + - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response" + "depends_on": + "pulsar": + "condition": "service_started" + "image": "docker.io/apachepulsar/pulsar:3.3.0" + "restart": "on-failure:100" + "milvus": + "command": + - "milvus" + - "run" + - "standalone" + "environment": + "ETCD_ENDPOINTS": "etcd:2379" + "MINIO_ADDRESS": "minio:9000" + "image": "docker.io/milvusdb/milvus:v2.4.5" + "ports": + - "9091:9091" + - "19530:19530" + "restart": "on-failure:100" + "volumes": + - "milvus:/var/lib/milvus" + "minio": + "command": + - "minio" + - "server" + - "/minio_data" + - "--console-address" + - ":9001" + "environment": + "MINIO_ROOT_PASSWORD": "minioadmin" + "MINIO_ROOT_USER": "minioadmin" + "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z" + "ports": + - "9001:9001" + "restart": "on-failure:100" + "volumes": + - "minio-data:/minio_data" + "prometheus": + "image": "docker.io/prom/prometheus:v2.53.1" + "ports": + - "9090:9090" + "restart": "on-failure:100" + "volumes": + - "./prometheus:/etc/prometheus" + - "prometheus-data:/prometheus" + "pulsar": + "command": "bin/pulsar standalone" + "image": "docker.io/apachepulsar/pulsar:3.3.0" + "ports": + - "6650:6650" + - "8080:8080" + "restart": "on-failure:100" + "volumes": + - "pulsar-conf:/pulsar/conf" + - "pulsar-data:/pulsar/data" + "pulsar-manager": + "environment": + "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties" + "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0" + "ports": + - "9527:9527" + - "7750:7750" + "restart": "on-failure:100" +"volumes": + "cassandra": {} + "etcd": {} + "grafana-storage": {} + "milvus": {} + "minio-data": {} + "prometheus-data": {} + "pulsar-conf": {} + "pulsar-data": {} diff --git a/setup.py b/setup.py index 8729d3d7..6c57c144 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import os with open("README.md", "r") as fh: long_description = fh.read() -version = "0.6.6" +version = "0.6.7" setuptools.setup( name="trustgraph", diff --git a/templates/components/cassandra.jsonnet b/templates/components/cassandra.jsonnet index af9dd5f1..d1a7defc 100644 --- a/templates/components/cassandra.jsonnet +++ b/templates/components/cassandra.jsonnet @@ -2,21 +2,11 @@ local base = import "base.jsonnet"; local images = import "images.jsonnet"; local url = import "url.jsonnet"; local cassandra_hosts = "cassandra"; -{ - volumes +: { - cassandra: {}, - }, - services +: { +local cassandra = import "stores/cassandra.jsonnet"; - cassandra: base + { - image: images.cassandra, - ports: [ - "9042:9042" - ], - volumes: [ - "cassandra:/var/lib/cassandra" - ], - }, +cassandra + { + + services +: { "store-triples": base + { image: images.trustgraph, diff --git a/templates/components/milvus.jsonnet b/templates/components/milvus.jsonnet index e1b40e9a..141c708e 100644 --- a/templates/components/milvus.jsonnet +++ b/templates/components/milvus.jsonnet @@ -1,71 +1,36 @@ local base = import "base.jsonnet"; local images = import "images.jsonnet"; -{ - volumes +: { - etcd: {}, - "minio-data": {}, - milvus: {}, - }, +local url = import "url.jsonnet"; +local milvus = import "stores/milvus.jsonnet"; + +milvus + { + services +: { - etcd: base + { - image: images.etcd, + + "store-graph-embeddings": base + { + image: images.trustgraph, command: [ - "etcd", - "-advertise-client-urls=http://127.0.0.1:2379", - "-listen-client-urls", - "http://0.0.0.0:2379", - "--data-dir", - "/etcd", - ], - environment: { - ETCD_AUTO_COMPACTION_MODE: "revision", - ETCD_AUTO_COMPACTION_RETENTION: "1000", - ETCD_QUOTA_BACKEND_BYTES: "4294967296", - ETCD_SNAPSHOT_COUNT: "50000" - }, - ports: [ - "2379:2379", - ], - volumes: [ - "etcd:/etcd" + "ge-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, ], }, - minio: base + { - image: images.minio, + + "query-graph-embeddings": base + { + image: images.trustgraph, command: [ - "minio", - "server", - "/minio_data", - "--console-address", - ":9001", - ], - environment: { - MINIO_ROOT_USER: "minioadmin", - MINIO_ROOT_PASSWORD: "minioadmin", - }, - ports: [ - "9001:9001", - ], - volumes: [ - "minio-data:/minio_data", - ], - }, - milvus: base + { - image: images.milvus, - command: [ - "milvus", "run", "standalone" - ], - environment: { - ETCD_ENDPOINTS: "etcd:2379", - MINIO_ADDRESS: "minio:9000", - }, - ports: [ - "9091:9091", - "19530:19530", - ], - volumes: [ - "milvus:/var/lib/milvus" + "ge-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, ], }, - }, + + } + } + + diff --git a/templates/components/neo4j.jsonnet b/templates/components/neo4j.jsonnet index dd6670d2..ea635f9a 100644 --- a/templates/components/neo4j.jsonnet +++ b/templates/components/neo4j.jsonnet @@ -1,31 +1,12 @@ local base = import "base.jsonnet"; local images = import "images.jsonnet"; local url = import "url.jsonnet"; -{ +local neo4j = import "stores/neo4j.jsonnet"; - volumes +: { - neo4j: {}, - }, +neo4j + { services +: { - neo4j: base + { - image: images.neo4j, - ports: [ - "7474:7474", - "7687:7687", - ], - environment: { - NEO4J_AUTH: "neo4j/password", -// NEO4J_server_bolt_listen__address: "0.0.0.0:7687", -// NEO4J_server_default__listen__address: "0.0.0.0", -// NEO4J_server_http_listen__address: "0.0.0.0:7474", - }, - volumes: [ - "neo4j:/data" - ], - }, - "query-triples": base + { image: images.trustgraph, command: [ diff --git a/templates/components/stores/cassandra.jsonnet b/templates/components/stores/cassandra.jsonnet new file mode 100644 index 00000000..4bd1205b --- /dev/null +++ b/templates/components/stores/cassandra.jsonnet @@ -0,0 +1,20 @@ +local base = import "../base.jsonnet"; +local images = import "../images.jsonnet"; +{ + volumes +: { + cassandra: {}, + }, + services +: { + + cassandra: base + { + image: images.cassandra, + ports: [ + "9042:9042" + ], + volumes: [ + "cassandra:/var/lib/cassandra" + ], + }, + + }, +} diff --git a/templates/components/stores/milvus.jsonnet b/templates/components/stores/milvus.jsonnet new file mode 100644 index 00000000..b53f967d --- /dev/null +++ b/templates/components/stores/milvus.jsonnet @@ -0,0 +1,79 @@ +local base = import "../base.jsonnet"; +local images = import "../images.jsonnet"; + +{ + + volumes +: { + etcd: {}, + "minio-data": {}, + milvus: {}, + }, + + services +: { + + etcd: base + { + image: images.etcd, + command: [ + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls", + "http://0.0.0.0:2379", + "--data-dir", + "/etcd", + ], + environment: { + ETCD_AUTO_COMPACTION_MODE: "revision", + ETCD_AUTO_COMPACTION_RETENTION: "1000", + ETCD_QUOTA_BACKEND_BYTES: "4294967296", + ETCD_SNAPSHOT_COUNT: "50000" + }, + ports: [ + "2379:2379", + ], + volumes: [ + "etcd:/etcd" + ], + }, + + minio: base + { + image: images.minio, + command: [ + "minio", + "server", + "/minio_data", + "--console-address", + ":9001", + ], + environment: { + MINIO_ROOT_USER: "minioadmin", + MINIO_ROOT_PASSWORD: "minioadmin", + }, + ports: [ + "9001:9001", + ], + volumes: [ + "minio-data:/minio_data", + ], + }, + + milvus: base + { + image: images.milvus, + command: [ + "milvus", "run", "standalone" + ], + environment: { + ETCD_ENDPOINTS: "etcd:2379", + MINIO_ADDRESS: "minio:9000", + }, + ports: [ + "9091:9091", + "19530:19530", + ], + volumes: [ + "milvus:/var/lib/milvus" + ], + }, + + }, + +} diff --git a/templates/components/stores/neo4j.jsonnet b/templates/components/stores/neo4j.jsonnet new file mode 100644 index 00000000..6c37ecca --- /dev/null +++ b/templates/components/stores/neo4j.jsonnet @@ -0,0 +1,30 @@ +local base = import "../base.jsonnet"; +local images = import "../images.jsonnet"; +{ + + volumes +: { + neo4j: {}, + }, + + services +: { + + neo4j: base + { + image: images.neo4j, + ports: [ + "7474:7474", + "7687:7687", + ], + environment: { + NEO4J_AUTH: "neo4j/password", +// NEO4J_server_bolt_listen__address: "0.0.0.0:7687", +// NEO4J_server_default__listen__address: "0.0.0.0", +// NEO4J_server_http_listen__address: "0.0.0.0:7474", + }, + volumes: [ + "neo4j:/data" + ], + }, + + }, + +} diff --git a/templates/components/trustgraph.jsonnet b/templates/components/trustgraph.jsonnet index 0962c98f..ef84fa07 100644 --- a/templates/components/trustgraph.jsonnet +++ b/templates/components/trustgraph.jsonnet @@ -64,28 +64,6 @@ local url = import "url.jsonnet"; ], }, - "store-graph-embeddings": base + { - image: images.trustgraph, - command: [ - "ge-write-milvus", - "-p", - url.pulsar, - "-t", - url.milvus, - ], - }, - - "query-graph-embeddings": base + { - image: images.trustgraph, - command: [ - "ge-query-milvus", - "-p", - url.pulsar, - "-t", - url.milvus, - ], - }, - "graph-rag": base + { image: images.trustgraph, command: [ diff --git a/templates/docker-compose-storage.jsonnet b/templates/docker-compose-storage.jsonnet new file mode 100644 index 00000000..6ae5c0be --- /dev/null +++ b/templates/docker-compose-storage.jsonnet @@ -0,0 +1,10 @@ + +local cassandra = import "components/stores/cassandra.jsonnet"; +local pulsar = import "components/pulsar.jsonnet"; +local milvus = import "components/stores/milvus.jsonnet"; +local grafana = import "components/grafana.jsonnet"; + +local config = cassandra + pulsar + milvus + grafana; + +std.manifestYamlDoc(config) +