From 4ae4b65c2fb4f0aa834f28b26b765698a9b2a49a Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Wed, 14 Aug 2024 00:46:25 +0100 Subject: [PATCH] Update docker compose --- docker-compose-azure.yaml | 256 +++++++++++++++++++++++++++++++ docker-compose-openai-neo4j.yaml | 3 - processing-just-llm.yaml | 45 ------ processing.yaml | 45 ------ 4 files changed, 256 insertions(+), 93 deletions(-) delete mode 100644 processing-just-llm.yaml delete mode 100644 processing.yaml diff --git a/docker-compose-azure.yaml b/docker-compose-azure.yaml index e69de29b..98a25510 100644 --- a/docker-compose-azure.yaml +++ b/docker-compose-azure.yaml @@ -0,0 +1,256 @@ +"services": + "cassandra": + "image": "docker.io/cassandra:4.1.5" + "ports": + - "9042:9042" + "restart": "on-failure:100" + "volumes": + - "cassandra:/var/lib/cassandra" + "chunker": + "command": + - "chunker-recursive" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "embeddings": + "command": + - "embeddings-hf" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "etcd": + "command": + - "etcd" + - "-advertise-client-urls=http://127.0.0.1:2379" + - "-listen-client-urls" + - "http://0.0.0.0:2379" + - "--data-dir" + - "/etcd" + "environment": + "ETCD_AUTO_COMPACTION_MODE": "revision" + "ETCD_AUTO_COMPACTION_RETENTION": "1000" + "ETCD_QUOTA_BACKEND_BYTES": "4294967296" + "ETCD_SNAPSHOT_COUNT": "50000" + "image": "quay.io/coreos/etcd:v3.5.5" + "ports": + - "2379:2379" + "restart": "on-failure:100" + "volumes": + - "etcd:/etcd" + "grafana": + "environment": + "GF_ORG_NAME": "trustgraph.ai" + "image": "docker.io/grafana/grafana:10.0.0" + "ports": + - "3000:3000" + "restart": "on-failure:100" + "volumes": + - "grafana-storage:/var/lib/grafana" + - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml" + - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml" + - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json" + "graph-rag": + "command": + - "graph-rag" + - "-p" + - "pulsar://pulsar:6650" + - "--prompt-request-queue" + - "non-persistent://tg/request/prompt-rag" + - "--prompt-response-queue" + - "non-persistent://tg/response/prompt-rag-response" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "init-pulsar": + "command": + - "sh" + - "-c" + - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response" + "depends_on": + "pulsar": + "condition": "service_started" + "image": "docker.io/apachepulsar/pulsar:3.3.0" + "restart": "on-failure:100" + "kg-extract-definitions": + "command": + - "kg-extract-definitions" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "kg-extract-relationships": + "command": + - "kg-extract-relationships" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "milvus": + "command": + - "milvus" + - "run" + - "standalone" + "environment": + "ETCD_ENDPOINTS": "etcd:2379" + "MINIO_ADDRESS": "minio:9000" + "image": "docker.io/milvusdb/milvus:v2.4.5" + "ports": + - "9091:9091" + - "19530:19530" + "restart": "on-failure:100" + "volumes": + - "milvus:/var/lib/milvus" + "minio": + "command": + - "minio" + - "server" + - "/minio_data" + - "--console-address" + - ":9001" + "environment": + "MINIO_ROOT_PASSWORD": "minioadmin" + "MINIO_ROOT_USER": "minioadmin" + "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z" + "ports": + - "9001:9001" + "restart": "on-failure:100" + "volumes": + - "minio-data:/minio_data" + "pdf-decoder": + "command": + - "pdf-decoder" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "prometheus": + "image": "docker.io/prom/prometheus:v2.53.1" + "ports": + - "9090:9090" + "restart": "on-failure:100" + "volumes": + - "./prometheus:/etc/prometheus" + - "prometheus-data:/prometheus" + "prompt": + "command": + - "prompt-generic" + - "-p" + - "pulsar://pulsar:6650" + - "--text-completion-request-queue" + - "non-persistent://tg/request/text-completion" + - "--text-completion-response-queue" + - "non-persistent://tg/response/text-completion-response" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "prompt-rag": + "command": + - "prompt-generic" + - "-p" + - "pulsar://pulsar:6650" + - "-i" + - "non-persistent://tg/request/prompt-rag" + - "-o" + - "non-persistent://tg/response/prompt-rag-response" + - "--text-completion-request-queue" + - "non-persistent://tg/request/text-completion-rag" + - "--text-completion-response-queue" + - "non-persistent://tg/response/text-completion-rag-response" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "pulsar": + "command": "bin/pulsar standalone" + "image": "docker.io/apachepulsar/pulsar:3.3.0" + "ports": + - "6650:6650" + - "8080:8080" + "restart": "on-failure:100" + "volumes": + - "pulsar-conf:/pulsar/conf" + - "pulsar-data:/pulsar/data" + "pulsar-manager": + "environment": + "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties" + "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0" + "ports": + - "9527:9527" + - "7750:7750" + "restart": "on-failure:100" + "query-graph-embeddings": + "command": + - "ge-query-milvus" + - "-p" + - "pulsar://pulsar:6650" + - "-t" + - "http://milvus:19530" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "query-triples": + "command": + - "triples-query-cassandra" + - "-p" + - "pulsar://pulsar:6650" + - "-g" + - "cassandra" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "store-graph-embeddings": + "command": + - "ge-write-milvus" + - "-p" + - "pulsar://pulsar:6650" + - "-t" + - "http://milvus:19530" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "store-triples": + "command": + - "triples-write-cassandra" + - "-p" + - "pulsar://pulsar:6650" + - "-g" + - "cassandra" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "text-completion": + "command": + - "text-completion-azure" + - "-p" + - "pulsar://pulsar:6650" + - "-k" + - "${AZURE_TOKEN}" + - "-e" + - "${AZURE_ENDPOINT}" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "text-completion-rag": + "command": + - "text-completion-azure" + - "-p" + - "pulsar://pulsar:6650" + - "-k" + - "${AZURE_TOKEN}" + - "-e" + - "${AZURE_ENDPOINT}" + - "-i" + - "non-persistent://tg/request/text-completion-rag" + - "-o" + - "non-persistent://tg/response/text-completion-rag-response" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "vectorize": + "command": + - "embeddings-vectorize" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" +"volumes": + "cassandra": {} + "etcd": {} + "grafana-storage": {} + "milvus": {} + "minio-data": {} + "prometheus-data": {} + "pulsar-conf": {} + "pulsar-data": {} diff --git a/docker-compose-openai-neo4j.yaml b/docker-compose-openai-neo4j.yaml index 7b901c1e..4ef1c177 100644 --- a/docker-compose-openai-neo4j.yaml +++ b/docker-compose-openai-neo4j.yaml @@ -113,9 +113,6 @@ "neo4j": "environment": "NEO4J_AUTH": "neo4j/password" - "NEO4J_server_bolt_listen__address": "0.0.0.0:7687" - "NEO4J_server_default__listen__address": "0.0.0.0" - "NEO4J_server_http_listen__address": "0.0.0.0:7474" "image": "docker.io/neo4j:5.22.0-community-bullseye" "ports": - "7474:7474" diff --git a/processing-just-llm.yaml b/processing-just-llm.yaml deleted file mode 100644 index 817d6594..00000000 --- a/processing-just-llm.yaml +++ /dev/null @@ -1,45 +0,0 @@ - -services: - - pdf-decoder: - module: trustgraph.decoder.pdf - - chunker: - module: trustgraph.chunker.recursive - - vectorize: - module: trustgraph.embeddings.vectorize - -# embeddings: -# module: trustgraph.embeddings.hf - - # embeddings: - # module: trustgraph.embeddings.ollama - # parameters: - # ollama: http://ollama:11434 - - embeddings: - module: trustgraph.embeddings.hf - parameters: - model: mixedbread-ai/mxbai-embed-large-v1 - - kg-extract-definitions: - module: trustgraph.kg.extract_definitions - - kg-extract-relationships: - module: trustgraph.kg.extract_relationships - - vector-write: - module: trustgraph.vector.milvus_write - - graph-write: - module: trustgraph.graph.cassandra_write - - llm: - module: trustgraph.llm.ollama_text - parameters: - ollama: http://ollama:11434 - - graph-rag: - module: trustgraph.rag.graph - diff --git a/processing.yaml b/processing.yaml deleted file mode 100644 index 817d6594..00000000 --- a/processing.yaml +++ /dev/null @@ -1,45 +0,0 @@ - -services: - - pdf-decoder: - module: trustgraph.decoder.pdf - - chunker: - module: trustgraph.chunker.recursive - - vectorize: - module: trustgraph.embeddings.vectorize - -# embeddings: -# module: trustgraph.embeddings.hf - - # embeddings: - # module: trustgraph.embeddings.ollama - # parameters: - # ollama: http://ollama:11434 - - embeddings: - module: trustgraph.embeddings.hf - parameters: - model: mixedbread-ai/mxbai-embed-large-v1 - - kg-extract-definitions: - module: trustgraph.kg.extract_definitions - - kg-extract-relationships: - module: trustgraph.kg.extract_relationships - - vector-write: - module: trustgraph.vector.milvus_write - - graph-write: - module: trustgraph.graph.cassandra_write - - llm: - module: trustgraph.llm.ollama_text - parameters: - ollama: http://ollama:11434 - - graph-rag: - module: trustgraph.rag.graph -